Expo Camera integration

.gitignore

@@ -1,5 +1,9 @@
 # Learn more https://docs.github.com/en/get-started/getting-started-with-git/ignoring-files
 
+ocr/__pycache__/
+ocr/imgs/*
+ocr/output/*
+
 # dependencies
 node_modules/
 
@@ -165,3 +169,7 @@ dist
 .yarn/build-state.yml
 .yarn/install-state.gz
 .pnp.*
+
+ocr/env/*
+
+ocr/.venv

README.md

@@ -1,19 +1,49 @@
-# housemates-groceries
+# green
+
+Groceries for housemates made simple.
+Maintain a grocery list, track items bought,
+and the split the bill automatically by scanning receipts.
+
+## Project Structure
+
+Green uses React Native/Expo for frontend, MongoDB for storage,
+and a Python FastAPI server for running OCR on receipts.
+
+Pages are in `app`. Adding a new file there will automatically create a new route.
+
+Python OCR server code is in `ocr/`. `test/` contains some sample images and debug OCR output.
+`imgs/` stores the files sent to the OCR server.
+`app.py` is the FastAPI server code, and `ocr.py` is the code to process a given image.
 
 ## Getting Started
 
-Download the "Expo Go" app
+### Expo and Node (Frontend)
+
+Download the "Expo Go" app on your phone.
 
-Install dependencies `npm install`
+Install dependencies: `npm install`
 
 Start Expo App: `npx expo start --tunnel`
 
+### Python OCR Server (Backend)
+
+Create virtual environment in the `ocr` folder: `cd ocr; virtualenv venv; source venv/bin/activate`
+
+Install dependencies: `python3 -m pip install -r requirements.txt`
+
+Start server: `python3 -m uvicorn app:app --reload`
+
+## Testing & Scripts
+
+You can test the OCR output by:
+
+- Add a new receipt image into `ocr/test`
+- Run `scripts/test_receipt.sh --receipt_name=<your receipt image name>`
+
+You can clear the OCR images and debug output with `./scripts/clear_images.sh`
+
 ## Important Links
 
 [Expo Docs](https://docs.expo.dev/tutorial/create-your-first-app/)
 [NativeWind/Tailwind with Expo](https://www.nativewind.dev/quick-starts/expo)
 [Clerk with Expo](https://clerk.com/docs/references/expo/overview)
-
-## Project Structure
-
-Pages are in `app`. Adding a new file there will automatically create a new route.

app.json

@@ -37,7 +37,13 @@
     },
     "owner": "labrador-housemates",
     "plugins": [
-      "expo-router"
+      "expo-router",
+      [
+        "expo-camera",
+        {
+          "cameraPermission": "Allow Green to access your camera"
+        }
+      ]
     ],
     "scheme": "app-scheme"
   }

app/index.tsx

@@ -14,6 +14,13 @@ export default function Page() {
     <View className="flex-1 items-center padding-24">
       <View className="flex-1 justify-center max-w-4xl mx-auto">
         <Text className="text-4xl font-bold self-center">Welcome to Green</Text>
+        <Link href="/scan" asChild>
+          <TouchableOpacity
+            className="bg-gray-500 hover:bg-gray-600 mt-14 py-2.5 px-4 w-1/3 self-center rounded-lg"
+          >
+            <Text className="text-white text-center self-center"> Autumn's Camera Button!</Text>
+          </TouchableOpacity>
+        </Link>
         <Link href="/signup" asChild>
           <TouchableOpacity 
               className="bg-gray-500 hover:bg-gray-600 mt-14 py-2.5 px-4 w-1/3 self-center rounded-lg"

app/scan.tsx

@@ -1,7 +1,171 @@
+
+import { CameraView, CameraType, useCameraPermissions } from 'expo-camera';
+import React, { useRef } from 'react';
+import { useState } from 'react';
+import { Button, StyleSheet, Text, TouchableOpacity, View, Image } from 'react-native';
+
 export default function Page() {
-
-    // TODO: Implement the scan page
-    // Camera access and integration with OCR
-
-    return;
-}
+  const [facing, setFacing] = useState<CameraType>('back');
+  const [permission, requestPermission] = useCameraPermissions();
+  const [imageUri, setImageUri] = useState(null);
+  const cameraRef = useRef(null);
+  const [receiptLines, setReceiptLines] = useState([]);
+
+  let RECEIPT_API_URL = 'http://127.0.0.1:8000/receiptLines';
+
+  if (!permission) {
+    // Camera permissions are still loading.
+    return <View />;
+  }
+
+  if (!permission.granted) {
+    // Camera permissions are not granted yet.
+    return (
+      <View style={styles.container}>
+        <Text style={styles.message}>We need your permission to show the camera</Text>
+        <Button onPress={requestPermission} title="grant permission" />
+      </View>
+    );
+  }
+
+  function toggleCameraFacing() {
+    setFacing(current => (current === 'back' ? 'front' : 'back'));
+  }
+
+  async function takePicture() {
+    if (cameraRef.current) {
+      const options = { quality: 0.5, base64: true, exif: true };
+      const photo = await cameraRef.current.takePictureAsync(options);
+      setImageUri(photo.uri);
+
+      fetch(RECEIPT_API_URL, {
+        method: 'POST',
+        mode: 'cors',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          "image": photo.base64
+        }),
+      }).then((response) => {
+        console.log(response);
+        // receipt lines
+        response.json();
+      }
+      );
+      // .then((data) => {
+      //   console.log(data);
+      //   setReceiptLines(data)
+      // });
+    }
+  }
+
+
+  return (
+    <View style={styles.container}>
+      {imageUri ? <Image source={{ uri: imageUri }} style={{ width: 500, height: 500 }} /> :
+        <CameraView ref={cameraRef} style={styles.camera} facing={facing}>
+          <View style={styles.buttonContainer}>
+            <TouchableOpacity style={styles.button} onPress={toggleCameraFacing}>
+              <Text style={styles.text}>Flip Camera</Text>
+            </TouchableOpacity>
+            <TouchableOpacity style={styles.button} onPress={takePicture}>
+              <Text style={styles.text}>Take Picture</Text>
+            </TouchableOpacity>
+          </View>
+        </CameraView>}
+    </View>
+  );
+}
+
+const styles = StyleSheet.create({
+  container: {
+    flex: 1,
+    justifyContent: 'center',
+  },
+  message: {
+    textAlign: 'center',
+    paddingBottom: 10,
+  },
+  camera: {
+    flex: 1,
+  },
+  buttonContainer: {
+    flex: 1,
+    flexDirection: 'row',
+    backgroundColor: 'transparent',
+    margin: 64,
+  },
+  button: {
+    flex: 1,
+    alignSelf: 'flex-end',
+    alignItems: 'center',
+  },
+  text: {
+    fontSize: 24,
+    fontWeight: 'bold',
+    color: 'white',
+  },
+});
+
+
+// import { Button, Text, TouchableOpacity, View } from "react-native";
+// import React, { useState, useEffect } from 'react';
+
+// import { Camera } from 'expo-camera';
+// import { CameraType } from "expo-camera/build/legacy/Camera.types";
+
+// export default function Page() {
+//     const [type, setType] = useState(CameraType.back);
+//     const [permission, requestPermission] = Camera.useCameraPermissions();
+//     const [camera, setCamera] = useState(null);
+
+//     const [receiptLines, setReceiptLines] = useState([]);
+
+//     let RECEIPT_API_URL = 'http://127.0.0.1:8000/receiptLines';
+
+//     // if (!permission) ...
+
+//     // if (!permission.granted) ...
+
+//     function toggleCameraType() {
+//       setType(current => (current === CameraType.back ? CameraType.front : CameraType.back));
+//     }
+
+//     function takePicture() {
+//       if (camera) {
+//         camera.takePictureAsync({onPictureSaved: (data) => {
+//           fetch(RECEIPT_API_URL, {
+//             method: 'POST',
+//             mode: 'cors',
+//             headers: {
+//               'Content-Type': 'application/json',
+//             },
+//             body: JSON.stringify({
+//               "image": data.base64
+//             }),
+//           }).then((response) =>
+//             // receipt lines
+//             response.json()
+//           ).then((data) => {
+//             console.log(data);
+//             setReceiptLines(data)
+//           });
+//         }});
+//       }
+
+//     }
+
+//     return (
+//     <View className="flex-1 justify-center">
+//         <Camera className="flex-1" type={type} ref={(ref) => {setCamera(ref);}}>
+//           <View className="flex-1 flex-row bg-transparent m-64">
+//               <Button title="Take Picture" onPress={takePicture}/>
+//               <TouchableOpacity className="flex-1 self-end items-center" onPress={toggleCameraType}>
+//               <Text className="text-2xl font-bold text-white">Flip Camera</Text>
+//               </TouchableOpacity>
+//           </View>
+//         </Camera>
+//     </View>
+//     );
+// }

ocr/.flaskenv

@@ -0,0 +1,2 @@
+FLASK_APP=app.py
+FLASK_ENV=development

ocr/app.py

@@ -0,0 +1,40 @@
+import base64
+from datetime import datetime
+from io import BytesIO
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+import json
+from ocr import scan_receipt
+from pathlib import Path
+from PIL import Image
+from pydantic import BaseModel
+
+app = FastAPI()
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"]
+)
+
+class ReceiptLinesArgs(BaseModel):
+    image: str
+
+@app.post("/receiptLines")
+def scan_receipt_image(args: ReceiptLinesArgs):
+    image_url = args.image
+
+    # strip the metadata from the front of the image URI
+    assert image_url.startswith("data:image/png;base64,")
+    image_b64 = image_url.split(",")[1]
+
+    image_data = base64.b64decode(image_b64)
+    image = Image.open(BytesIO(image_data))
+
+    image_path = Path(f"imgs") / f"{datetime.now().strftime('%Y%m%d%H%M%S')}.png"
+    image.save(image_path)
+
+    return json.dumps(
+        scan_receipt(str(image_path), debug=True)
+    )