Merge pull request #30 from MetaCell/feature/CELE-78

Feature/cele 78

.github/workflows/pytest.yml

@@ -3,7 +3,7 @@ name: Python Tests
 on: [push, pull_request]
 
 jobs:
-  pytest:
+  pytest-visualizer:
     runs-on: ubuntu-latest
     defaults:
       run:
@@ -19,6 +19,25 @@ jobs:
           python -m pip install --upgrade pip
           pip install -r requirements.txt
           pip install -r requirements-dev.txt
+      - name: Test with pytest
+        run: |
+          python -m pytest
+  pytest-ingestion:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./ingestion
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Install dependencies and dev dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pytest
+          pip install .
       - name: Test with pytest
         run: |
           python -m pytest

applications/visualizer/api/openapi.json

@@ -416,7 +416,14 @@
             "type": "string"
           },
           "emData": {
-            "$ref": "#/components/schemas/EMData"
+            "anyOf": [
+              {
+                "$ref": "#/components/schemas/EMData"
+              },
+              {
+                "type": "null"
+              }
+            ]
           },
           "collection": {
             "maxLength": 20,
@@ -473,33 +480,78 @@
       },
       "EMData": {
         "properties": {
-          "min_zoom": {
-            "title": "Min Zoom",
+          "minZoom": {
+            "title": "Minzoom",
             "type": "integer"
           },
-          "max_zoom": {
-            "title": "Max Zoom",
+          "maxZoom": {
+            "title": "Maxzoom",
             "type": "integer"
           },
-          "nb_slices": {
-            "title": "Nb Slices",
+          "nbSlices": {
+            "title": "Nbslices",
             "type": "integer"
           },
-          "resource_url": {
-            "title": "Resource Url",
+          "tileSize": {
+            "maxItems": 2,
+            "minItems": 2,
+            "prefixItems": [
+              {
+                "type": "integer"
+              },
+              {
+                "type": "integer"
+              }
+            ],
+            "title": "Tilesize",
+            "type": "array"
+          },
+          "sliceRange": {
+            "maxItems": 2,
+            "minItems": 2,
+            "prefixItems": [
+              {
+                "type": "integer"
+              },
+              {
+                "type": "integer"
+              }
+            ],
+            "title": "Slicerange",
+            "type": "array"
+          },
+          "resourceUrl": {
+            "title": "Resourceurl",
             "type": "string"
           },
-          "segmentation_url": {
-            "title": "Segmentation Url",
+          "segmentationUrl": {
+            "title": "Segmentationurl",
             "type": "string"
+          },
+          "segmentationSize": {
+            "maxItems": 2,
+            "minItems": 2,
+            "prefixItems": [
+              {
+                "type": "integer"
+              },
+              {
+                "type": "integer"
+              }
+            ],
+            "title": "Segmentationsize",
+            "type": "array"
           }
         },
         "required": [
-          "min_zoom",
-          "max_zoom",
-          "nb_slices",
-          "resource_url",
-          "segmentation_url"
+          "minZoom",
+          "maxZoom",
+          "nbSlices",
+          "tileSize",
+          "sliceRange",
+          "resourceUrl",
+          "segmentationUrl",
+          "segmentationSize"
         ],
         "title": "EMData",
         "type": "object"

applications/visualizer/api/openapi.yaml

@@ -47,7 +47,9 @@ components:
           title: Description
           type: string
         emData:
-          $ref: '#/components/schemas/EMData'
+          anyOf:
+          - $ref: '#/components/schemas/EMData'
+          - type: 'null'
         id:
           title: Id
           type: string
@@ -82,27 +84,54 @@ components:
       type: object
     EMData:
       properties:
-        max_zoom:
-          title: Max Zoom
+        maxZoom:
+          title: Maxzoom
           type: integer
-        min_zoom:
-          title: Min Zoom
+        minZoom:
+          title: Minzoom
           type: integer
-        nb_slices:
-          title: Nb Slices
+        nbSlices:
+          title: Nbslices
           type: integer
-        resource_url:
-          title: Resource Url
+        resourceUrl:
+          title: Resourceurl
           type: string
-        segmentation_url:
-          title: Segmentation Url
+        segmentationSize:
+          maxItems: 2
+          minItems: 2
+          prefixItems:
+          - type: integer
+          - type: integer
+          title: Segmentationsize
+          type: array
+        segmentationUrl:
+          title: Segmentationurl
           type: string
+        sliceRange:
+          maxItems: 2
+          minItems: 2
+          prefixItems:
+          - type: integer
+          - type: integer
+          title: Slicerange
+          type: array
+        tileSize:
+          maxItems: 2
+          minItems: 2
+          prefixItems:
+          - type: integer
+          - type: integer
+          title: Tilesize
+          type: array
       required:
-      - min_zoom
-      - max_zoom
-      - nb_slices
-      - resource_url
-      - segmentation_url
+      - minZoom
+      - maxZoom
+      - nbSlices
+      - tileSize
+      - sliceRange
+      - resourceUrl
+      - segmentationUrl
+      - segmentationSize
       title: EMData
       type: object
     ErrorMessage:

applications/visualizer/backend/api/api.py

@@ -1,13 +1,15 @@
 from collections import defaultdict
 from typing import Iterable, Optional
 
-from ninja import NinjaAPI, Router, Schema, Query
+from ninja import NinjaAPI, Router, Query, Schema
 from ninja.pagination import paginate, PageNumberPagination
 from django.shortcuts import aget_object_or_404
 from django.db.models import Q
 from django.db.models.manager import BaseManager
 from django.conf import settings
 
+from .utils import fetch_dataset_metadata, to_list
+
 from .schemas import Dataset, EMData, Neuron, Connection
 from .models import (
     Dataset as DatasetModel,
@@ -21,10 +23,6 @@ class ErrorMessage(Schema):
     detail: str
 
 
-async def to_list(q):
-    return [x async for x in q]
-
-
 class CElegansAPI(NinjaAPI):
 
     def get_openapi_operation_id(self, operation):
@@ -57,17 +55,7 @@ def annotate_dataset(datasets: Iterable[DatasetModel]):
                 dataset=dataset_id
             )
         )
-        dataset.em_data = EMData(  # type: ignore
-            min_zoom=0,
-            max_zoom=0,
-            nb_slices=0,
-            # resource_url=settings.DATASET_EMDATA_URL_FORMAT.format(dataset=dataset_id),
-            # segmentation_url=settings.DATASET_EMDATA_SEGMENTATION_URL_FORMAT.format(
-            # dataset=dataset_id
-            # ),
-            resource_url=settings.DATASET_EMDATA_URL_FORMAT,
-            segmentation_url=settings.DATASET_EMDATA_SEGMENTATION_URL_FORMAT,
-        )
+        dataset.em_data = fetch_dataset_metadata(dataset_id)  # type: ignore
 
 
 @api.get("/datasets", response=list[Dataset], tags=["datasets"])

applications/visualizer/backend/api/management/commands/populatedb.py

@@ -1,18 +1,20 @@
 from django.core.management.base import BaseCommand, CommandError
 from pathlib import Path
 from api.populatedb import populate_functions
+from django.conf import settings
 
 
 class Command(BaseCommand):
     help = "Transform data from raw-directory and populate DB"
 
-    def add_arguments(self, parser):
-        parser.add_argument("raw_folder", type=str)
-
     def handle(self, *args, **options):
-        folder = Path(options["raw_folder"])
-        if not folder.exists():
-            raise CommandError(f"Folder {folder} does not exist")
+        try:
+            downloader = settings.RAW_DB_DATA_DOWNLOADER()
+            folder = downloader.pull_files()
+        except Exception as e:
+            raise CommandError(
+                f'An error occured while pulling the files from the bucket or the local file system: "{e}"'
+            )
 
         for func in populate_functions:
             func(

applications/visualizer/backend/api/populatedb.py

@@ -18,6 +18,14 @@ def translate(translation_map, data):
     return data
 
 
+def clear_db(_, print, print_success):
+    print("Cleaning all the entries from the tables...")
+    for table in (Synapse, Connection, Annotation, Neuron, Dataset):
+        print(f"  . removing entries from {table.__name__}...", ending="")
+        table.objects.all().delete()
+        print_success("\t[OK]")
+
+
 def populate_datasets(path, print, print_success):
     print("Populate Dataset table...", ending="")
     raw_datasets = path / "datasets.json"
@@ -30,7 +38,7 @@ def populate_datasets(path, print, print_success):
         ],
         ignore_conflicts=True,
     )
-    print_success("\t[OK]")
+    print_success("\t\t[OK]")
 
 
 def populate_neurons(path, print, print_success):
@@ -53,7 +61,7 @@ def populate_neurons(path, print, print_success):
         neurons.append(Neuron(**data))
 
     Neuron.objects.bulk_create(neurons, ignore_conflicts=True)
-    print_success("\t[OK]")
+    print_success("\t\t[OK]")
 
 
 def combine_annotations(annotation_path: Path, print):
@@ -121,7 +129,7 @@ def populate_annotations(path, print, print_success):
     Annotation.objects.bulk_create(
         [Annotation(**data) for data in annotations], ignore_conflicts=True
     )
-    print_success("\t\t[OK]")
+    print_success("\t\t\t[OK]")
 
 
 def combine_connections(connections_path: Path, print):
@@ -238,7 +246,7 @@ def populate_connections(path, print, print_success):
         [Connection(**connection) for connection in connections.values()],
         ignore_conflicts=True,
     )
-    print_success("\t\t[OK]")
+    print_success("\t\t\t[OK]")
 
 
 def populate_synapses(_, print, print_success):
@@ -251,10 +259,11 @@ def populate_synapses(_, print, print_success):
         synapse_objects.append(Synapse(**synapse))
 
     Synapse.objects.bulk_create(synapse_objects, ignore_conflicts=True)
-    print_success("\t\t[OK]")
+    print_success("\t\t\t[OK]")
 
 
 populate_functions = [
+    clear_db,
     populate_datasets,
     populate_neurons,
     populate_annotations,

applications/visualizer/backend/api/schemas.py

@@ -21,24 +21,21 @@ class Config(Schema.Config):
         populate_by_name = True
 
 
-class Artifact(Schema): ...
-
-
-class Model3D(Artifact): ...
-
-
-class EMData(Artifact):
+class EMData(BilingualSchema):
     min_zoom: int
     max_zoom: int
     nb_slices: int
+    tile_size: tuple[int, int]
+    slice_range: tuple[int, int]
     resource_url: str
     segmentation_url: str
+    segmentation_size: tuple[int, int]
 
 
 class Dataset(ModelSchema, BilingualSchema):
     id: str
     neuron3D_url: str
-    em_data: EMData
+    em_data: EMData | None
 
     class Meta:
         model = DatasetModel