diff --git a/frontend/.gitignore b/.gitignore similarity index 96% rename from frontend/.gitignore rename to .gitignore index 8f322f0..5bbb991 100644 --- a/frontend/.gitignore +++ b/.gitignore @@ -32,4 +32,4 @@ yarn-error.log* # typescript *.tsbuildinfo -next-env.d.ts +next-env.d.ts \ No newline at end of file diff --git a/frontend/LICENSE b/LICENSE similarity index 100% rename from frontend/LICENSE rename to LICENSE diff --git a/README.md b/README.md index fe7ae7a..9b4717a 100644 --- a/README.md +++ b/README.md @@ -1,36 +1,31 @@ -# Ayre - -> *"... I'll support you!"* - -Ayre is a Visual Question Answering project created by members of [this organization](https://github.com/projectayre) as part of our semester V projects and coursework during our Bachelor's degree. This is primarily a research based project, since Visual Question Answering with Sentiment Analysis is a significant research gap that we would like to address. This project will feature several components coming together as a complete full-stack AI project. - -## Planned/Implemented features [checklist] - -### AI - -All AI development to done using Torch, for better support with transformers. - -- [x] ~~Sentiment Analysis on images using integrating fuzzy logic.~~ -- [ ] NEW: Semantic Image Segmentation for Improved Understanding of the image. -- [ ] Visual Question Answering task using data from sentiment analysis. -- [ ] At least 1 Journal and 1 Conference paper based on novel approaches made here. -- [ ] MLOps pipelines that can be easily imported and deployed inside the backend. -- [ ] (Optional) Github Packages Integration - -### Frontend - -A simple, beautiful UI/UX with the following features: - -- [ ] Responsive Design for accessibility. -- [ ] Light/Dark themes. -- [ ] OAuth integration, with some of the benefits. -- [ ] Deployment on Vercel/Github Pages. -- [ ] (Optional) A Loading screen. - -### Backend - -- [ ] Minimal and fast backend created in Django. -- [ ] Database connectivity for live storage of queries and information. -- [ ] Modular design for improved CI/CD. -- [ ] Full cloud hosting/self-hosting of the backend. -- [ ] (Optional) Scalability Options. +# Next.js & NextUI Template + +This is a template for creating applications using Next.js 13 (app directory) and NextUI (v2). + +## Technologies Used + +- [Next.js 13](https://nextjs.org/docs/getting-started) +- [NextUI v2](https://nextui.org/) +- [Tailwind CSS](https://tailwindcss.com/) +- [Tailwind Variants](https://tailwind-variants.org) +- [TypeScript](https://www.typescriptlang.org/) +- [Framer Motion](https://www.framer.com/motion/) +- [next-themes](https://github.com/pacocoursey/next-themes) + +## How to Use + +### Install dependencies + +```bash +npm install +``` + +### Run the development server + +```bash +npm run dev +``` + +## License + +Licensed under the [MIT license](https://github.com/nextui-org/next-app-template/blob/main/LICENSE). \ No newline at end of file diff --git a/frontend/Scene.jsx b/Scene.jsx similarity index 100% rename from frontend/Scene.jsx rename to Scene.jsx diff --git a/frontend/app/blog/layout.tsx b/app/algorithm/layout.tsx similarity index 56% rename from frontend/app/blog/layout.tsx rename to app/algorithm/layout.tsx index 45deef1..54d5e13 100644 --- a/frontend/app/blog/layout.tsx +++ b/app/algorithm/layout.tsx @@ -1,13 +1,15 @@ -export default function BlogLayout({ +import "@/styles/globals.css"; + +export default function AlgorithmLayout({ children, }: { children: React.ReactNode; }) { return (
-
+
{children}
); -} +} \ No newline at end of file diff --git a/app/algorithm/page.tsx b/app/algorithm/page.tsx new file mode 100644 index 0000000..3415b01 --- /dev/null +++ b/app/algorithm/page.tsx @@ -0,0 +1,79 @@ +'use client;' + +import { Image, Spacer, Text } from "@nextui-org/react"; +import { title } from "@Ï/components/primitives"; + +export default function AlgorithmPage() { + return ( +
+ +
+

+ About Us +

+
+ +
+

+ Adpative Yielding Response Engine (AYRE) which is not only semantically attentive but also emotionally aware. + The engine is a Semantic-Attentive Visual Question Answering system with the capability of generating answers to questions based on the content of the image and the mood it portrays. +

+ +

+ Problem Statement +


+ +

+ Given an image, can our machine answer users’ questions in natural language? + If so, can it frame answers in a way that captures any abstract semantics behind just the image or the question? +

+ +

+ Methodology +

+ + NextUI hero Image + +

+ A modified vision transformer along with another modified U-NET model made into a seamless pipeline that will perform tasks of semantic segmentation and VQA in hopes that the segmented analysis is able to outperform SOTA methods. +


+ +

+ A modified U-Net segmentation model provides semantic information about each pixel in the image. + These color segments, as latent tensors, provide the value matrix for attention masks. + The segments can also be annotated on the fly to provide context to the user. +


+ + NextUI hero Image + +

+ Dual stream transformer Vision Language Processing model that uses self-attention on both visual and text features to maximize info-gain. The novelty lies in the attention heads; these are generated using image segmentation to ensure every label gets its own attention +


+ + NextUI hero Image + +

+ The AYRE sentiment analysis module is designed to interpret an image's emotional undertones through color analysis. The system maps dominant colors extracted from the image to corresponding emotions using color psychology. This process begins loading the image into the system, which imports the image and converts it to RGB format for accurate color representation. +


+ + NextUI hero Image +
+
+ ); +} \ No newline at end of file diff --git a/frontend/app/error.tsx b/app/error.tsx similarity index 100% rename from frontend/app/error.tsx rename to app/error.tsx diff --git a/frontend/app/layout.tsx b/app/layout.tsx similarity index 100% rename from frontend/app/layout.tsx rename to app/layout.tsx diff --git a/app/page.tsx b/app/page.tsx new file mode 100644 index 0000000..00b7e7c --- /dev/null +++ b/app/page.tsx @@ -0,0 +1,49 @@ +'use client'; + +// Import necessary modules +import { useState } from 'react'; +import { Divider, Spacer } from '@nextui-org/react'; +import { SketchfabModel } from '../components/SketchfabModel'; +import { AnswerCard } from '@/components/answer-card'; +import DragDrop from '@/components/drap-drop'; +import axios from 'axios'; // Add this import + +const responseData = { + // image: "https://example.com/image.jpg", + // description: "This is the description of the answer.", +}; + +export default function Home() { + const [predictionData, setPredictionData] = useState(responseData); + + const handlePrediction = (data) => { + // Check if the image URL is present in the response data + if (data.image) { + // If the image is present, set the image URL directly + data.image = URL.createObjectURL(data.image); + } + setPredictionData(data); + }; + + + return ( +
+
+
+ +
+ +
+ +
+
+ + + + +
+ +
+
+ ); +} \ No newline at end of file diff --git a/frontend/app/providers.tsx b/app/providers.tsx similarity index 100% rename from frontend/app/providers.tsx rename to app/providers.tsx diff --git a/app/response/new/page.tsx b/app/response/new/page.tsx new file mode 100644 index 0000000..489241e --- /dev/null +++ b/app/response/new/page.tsx @@ -0,0 +1,2 @@ +//in case a new page is required to be created after submission + diff --git a/frontend/components/SketchfabModel.tsx b/components/SketchfabModel.tsx similarity index 92% rename from frontend/components/SketchfabModel.tsx rename to components/SketchfabModel.tsx index 43e5225..c0c6d62 100644 --- a/frontend/components/SketchfabModel.tsx +++ b/components/SketchfabModel.tsx @@ -12,7 +12,7 @@ function MeshComponent() { const gltf = useLoader(GLTFLoader, fileUrl); useFrame(() => { - mesh.current.rotation.y += 0.01; + mesh.current.rotation.y += 0.005; }); diff --git a/components/answer-card.tsx b/components/answer-card.tsx new file mode 100644 index 0000000..69b8251 --- /dev/null +++ b/components/answer-card.tsx @@ -0,0 +1,66 @@ +'use client'; + +// Import necessary modules +import React, { useEffect, useState } from 'react'; +import { Card, CardHeader, CardBody, CardFooter, Divider, Image } from '@nextui-org/react'; + +export const AnswerCard = ({ responseData }) => { + const [displayedData, setDisplayedData] = useState(responseData); + + useEffect(() => { + console.log('AnswerCard received data:', responseData); + setDisplayedData(responseData); + }, [responseData]); + + useEffect(() => { + console.log('Updating displayed data:', displayedData); + }, [displayedData]); + + console.log('Image type:', typeof displayedData.image); + + return ( + + +
+

Ayre's response...

+
+ + + + + + {displayedData.image ? ( + // Render the image directly if it's a URL + typeof displayedData.image === 'string' ? ( + Image of the answer + ) : displayedData.image instanceof Blob ? ( + // Use Blob URL directly + Image of the answer + ) : ( +

Invalid image format

+ ) + ) : ( +

No image available

+ )} + + + + + +

{displayedData.prediction || 'Lorem ipsum dolor sit amet...'}

+ + + ); +}; diff --git a/components/drap-drop.tsx b/components/drap-drop.tsx new file mode 100644 index 0000000..f4e0614 --- /dev/null +++ b/components/drap-drop.tsx @@ -0,0 +1,88 @@ +'use client'; + +// Import necessary modules +import React, { useState } from 'react'; +import { FileUploader } from 'react-drag-drop-files'; +import { Button } from '@nextui-org/button'; +import { Spacer, Textarea } from '@nextui-org/react'; +import axios from 'axios'; // Add this import + +const fileTypes = ['jpg', 'jpeg', 'png']; + +function DragDrop({ onPrediction }) { + const [file, setFile] = useState(null); + const [description, setDescription] = useState(''); + + const handleChange = (file) => { + setFile(file); + }; + + const handleDescriptionChange = (event) => { + setDescription(event.target.value); + }; + + const handleButtonClick = async (event) => { + event.preventDefault(); + + if (!file || !description) { + return; + } + + const formData = new FormData(); + formData.append('image', file); + formData.append('query', description); + + try { + const response = await axios.post('https://singularly-inviting-cat.ngrok-free.app/predict/', formData); + + if (response.status === 200) { + const responseData = { + prediction: response.data.prediction, + time: response.data.time, + image: file, // Pass the File object directly + }; + + console.log('Prediction data:', responseData); + onPrediction(responseData); + } else { + console.error('Error submitting prediction data:', response.status, response.statusText); + } + } catch (error) { + console.error('Error sending POST request:', error); + } + }; + + return ( +
+