Merge remote-tracking branch 'origin/main' into webui-dataset

.github/workflows/auth-ci.yml

@@ -30,7 +30,7 @@ jobs:
       run: |
         sudo apt-get install -y wget
         wget -O chrome.deb https://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_126.0.6478.126-1_amd64.deb
-        sudo dpkg -i chrome.deb
+        sudo apt update && sudo apt install ./chrome.deb -y --allow-downgrades
         rm chrome.deb
 
     - name: Install dependencies
@@ -46,6 +46,10 @@ jobs:
       working-directory: ./server
       run: cp .env.local.online-auth .env
 
+    - name: Run postgresql server in background
+      working-directory: ./server
+      run: sh run_dev_postgresql.sh && sleep 6
+
     - name: Run django server in background with generated certs
       working-directory: ./server
       run: sh setup-dev-server.sh & sleep 6

.github/workflows/docker-ci.yml

@@ -45,7 +45,7 @@ jobs:
 
     - name: Set server environment vars
       working-directory: ./server
-      run: cp .env.local.local-auth .env
+      run: cp .env.local.local-auth.sqlite .env
 
     - name: Generate SSL certificate
       working-directory: ./server

.github/workflows/local-ci.yml

@@ -47,6 +47,10 @@ jobs:
       working-directory: ./server
       run: cp .env.local.local-auth .env
 
+    - name: Run postgresql server in background
+      working-directory: ./server
+      run: sh run_dev_postgresql.sh && sleep 6
+
     - name: Run django server in background with generated certs
       working-directory: ./server
       run: sh setup-dev-server.sh & sleep 6

.github/workflows/unittests.yml

@@ -42,6 +42,9 @@ jobs:
     - name: Set server environment vars
       working-directory: ./server
       run: cp .env.local.local-auth .env
+    - name: Run postgresql server in background
+      working-directory: ./server
+      run: sh run_dev_postgresql.sh && sleep 6
     - name: Run migrations
       working-directory: ./server
       run: python manage.py migrate

cli/medperf/commands/dataset/prepare.py

@@ -260,6 +260,13 @@ def run_statistics(self):
             self.dataset.unmark_as_ready()
             raise e
 
+        with open(self.out_statistics_path) as f:
+            stats = yaml.safe_load(f)
+
+        if stats is None:
+            self.dataset.unmark_as_ready()
+            raise ExecutionError("Statistics file is empty.")
+
         self.ui.print("> Statistics complete")
 
     def mark_dataset_as_ready(self):

cli/medperf/tests/commands/dataset/test_prepare.py

@@ -56,7 +56,7 @@ def test_get_prep_cube_downloads_cube_file(mocker, data_preparation, cube):
 
 @pytest.mark.parametrize("dataset_for_test", [False, True])
 def test_prepare_with_test_data_doesnt_send_reports(
-        mocker, data_preparation, dataset_for_test, cube, comms, fs
+    mocker, data_preparation, dataset_for_test, cube, comms, fs
 ):
     # Arrange
     data_preparation.dataset.for_test = dataset_for_test
@@ -86,7 +86,9 @@ def test_prepare_runs_then_stops_report_handler(
     mocker.patch.object(data_preparation.dataset, "write")
     mocked_obs = mocker.create_autospec(spec=Observer)
     mocker.patch(PATCH_REGISTER.format("Observer"), side_effect=mocked_obs)
-    gen_report_spy = mocker.patch(PATCH_REGISTER.format("DataPreparation._DataPreparation__generate_report_dict"))
+    gen_report_spy = mocker.patch(
+        PATCH_REGISTER.format("DataPreparation._DataPreparation__generate_report_dict")
+    )
 
     # Act
     data_preparation.run_prepare()
@@ -208,11 +210,14 @@ def _failure_run(*args, **kwargs):
 
 @pytest.mark.parametrize("metadata_specified", [False, True])
 def test_statistics_checks_metadata_path(
-    mocker, data_preparation, metadata_specified, cube
+    mocker, data_preparation, metadata_specified, cube, fs
 ):
     # Arrange
     spy = mocker.patch.object(cube, "run")
     data_preparation.metadata_specified = metadata_specified
+    data_preparation.out_statistics_path = "test.yaml"
+    fs.create_file(data_preparation.out_statistics_path, contents="")
+    mocker.patch("yaml.safe_load", return_value={})
 
     # Act
     data_preparation.run_statistics()
@@ -224,6 +229,23 @@ def test_statistics_checks_metadata_path(
         assert "metadata_path" not in spy.call_args.kwargs.keys()
 
 
+def test_preparation_fails_if_statistics_is_none(mocker, data_preparation, cube, fs):
+
+    # Arrange
+    unmark_spy = mocker.patch.object(data_preparation.dataset, "unmark_as_ready")
+    mocker.patch.object(cube, "run")
+    data_preparation.out_statistics_path = "test.yaml"
+    fs.create_file(data_preparation.out_statistics_path, contents="")
+    mocker.patch("yaml.safe_load", return_value=None)
+
+    # Act
+    with pytest.raises(ExecutionError):
+        data_preparation.run_statistics()
+
+    # Assert
+    unmark_spy.assert_called_once()
+
+
 def test_dataset_is_updated_after_report_sending(mocker, data_preparation, comms):
     # Arrange
     send_spy = mocker.patch.object(comms, "update_dataset")

docs/getting_started/setup.md

@@ -21,7 +21,7 @@ For this tutorial, you should spawn a local MedPerf server for the MedPerf clien
 
     ```bash
     cd server
-    cp .env.local.local-auth .env
+    cp .env.local.local-auth.sqlite .env
     sh setup-dev-server.sh
     ```
 

examples/BraTS/data_prep/mlcube/mlcube.yaml

@@ -8,7 +8,7 @@ platform:
 
 docker:
   # Image name.
-  image: mlcommons/fets_data-prep
+  image: mlcommons/fets_data-prep-v2
   # Docker build context relative to $MLCUBE_ROOT. Default is `build`.
   build_context: "../project"
   # Docker file name within docker build context, default is `Dockerfile`.

examples/BraTS/data_prep/project/sanity_check.py

@@ -17,6 +17,7 @@ def check_subject_validity(subject_dir):
     subject_valid = True
     strings_to_check = [
         "_t1.nii.gz",
+        "_t1c.nii.gz",
         "_t1ce.nii.gz",
         "_t2.nii.gz",
         "_flair.nii.gz",

scripts/monitor/rano_monitor/utils.py

@@ -227,6 +227,56 @@ def to_local_path(mlcube_path: str, local_parent_path: str):
     return os.path.normpath(os.path.join(local_parent_path, mlcube_path))
 
 
+def package_segmentations(tar, row, data_path, labels_path):
+    brainscans = get_tumor_review_paths(row.name, data_path, labels_path)[:-2]
+    id, tp = row.name.split("|")
+    tar_path = os.path.join("review_cases", id, tp)
+    reviewed_path = os.path.join("review_cases", id, tp, "finalized")
+    reviewed_dir = tarfile.TarInfo(name=reviewed_path)
+    reviewed_dir.type = tarfile.DIRTYPE
+    reviewed_dir.mode = 0o755
+    tar.addfile(reviewed_dir)
+    tar.add(labels_path, tar_path)
+
+    brainscan_path = os.path.join("review_cases", id, tp, "brain_scans")
+    for brainscan in brainscans:
+        brainscan_target_path = os.path.join(
+            brainscan_path, os.path.basename(brainscan)
+        )
+        tar.add(brainscan, brainscan_target_path)
+
+
+def package_brain_masks(tar, row, data_path, labels_path):
+    id, tp = row.name.split("|")
+    brain_mask_filename = "brainMask_fused.nii.gz"
+    tar_path = os.path.join("review_cases", id, tp)
+    base_path = os.path.join(labels_path, "..")
+    brain_mask_path = os.path.join(base_path, brain_mask_filename)
+    brain_mask_tar_path = os.path.join(tar_path, brain_mask_filename)
+    if os.path.exists(brain_mask_path):
+        tar.add(brain_mask_path, brain_mask_tar_path)
+
+    rawscan_path = os.path.join("review_cases", id, tp, "raw_scans")
+    rawscans = get_brain_review_paths(row.name, labels_path)[:-1]
+    for rawscan in rawscans:
+        rawscan_target_path = os.path.join(
+            rawscan_path, os.path.basename(rawscan)
+        )
+        tar.add(rawscan, rawscan_target_path)
+
+
+def package_summary_imgs(tar, row, data_path, labels_path):
+    id, tp = row.name.split("|")
+    base_path = os.path.join(labels_path, "..")
+    tar_path = os.path.join("review_cases", id, tp)
+    for file in os.listdir(base_path):
+        if not file.endswith(".png"):
+            continue
+        img_path = os.path.join(base_path, file)
+        img_tar_path = os.path.join(tar_path, file)
+        tar.add(img_path, img_tar_path)
+
+
 def package_review_cases(report: pd.DataFrame, dset_path: str):
     review_cases = report[
         (MANUAL_REVIEW_STAGE <= abs(report["status"]))
@@ -236,49 +286,24 @@ def package_review_cases(report: pd.DataFrame, dset_path: str):
         for i, row in review_cases.iterrows():
             data_path = to_local_path(row["data_path"], dset_path)
             labels_path = to_local_path(row["labels_path"], dset_path)
-            brainscans = get_tumor_review_paths(row.name, data_path, labels_path)[:-2]
-            rawscans = get_brain_review_paths(row.name, labels_path)[:-1]
-            base_path = os.path.join(labels_path, "..")
+            if os.path.isfile(labels_path):
+                labels_path = os.path.dirname(labels_path)
+                labels_path = os.path.join(labels_path, "..")
 
             # Add tumor segmentations
-            id, tp = row.name.split("|")
-            tar_path = os.path.join("review_cases", id, tp)
-            reviewed_path = os.path.join("review_cases", id, tp, "finalized")
-            reviewed_dir = tarfile.TarInfo(name=reviewed_path)
-            reviewed_dir.type = tarfile.DIRTYPE
-            reviewed_dir.mode = 0o755
-            tar.addfile(reviewed_dir)
-            tar.add(labels_path, tar_path)
-
-            brainscan_path = os.path.join("review_cases", id, tp, "brain_scans")
-            for brainscan in brainscans:
-                brainscan_target_path = os.path.join(
-                    brainscan_path, os.path.basename(brainscan)
-                )
-                tar.add(brainscan, brainscan_target_path)
+            package_segmentations(tar, row, data_path, labels_path)
 
             # Add brain mask
-            brain_mask_filename = "brainMask_fused.nii.gz"
-            brain_mask_path = os.path.join(base_path, brain_mask_filename)
-            brain_mask_tar_path = os.path.join(tar_path, brain_mask_filename)
-            if os.path.exists(brain_mask_path):
-                tar.add(brain_mask_path, brain_mask_tar_path)
-
-            # Add raw scans
-            rawscan_path = os.path.join("review_cases", id, tp, "raw_scans")
-            for rawscan in rawscans:
-                rawscan_target_path = os.path.join(
-                    rawscan_path, os.path.basename(rawscan)
-                )
-                tar.add(rawscan, rawscan_target_path)
+            try:
+                package_brain_masks(tar, row, data_path, labels_path)
+            except FileNotFoundError:
+                pass
 
             # Add summary images
-            for file in os.listdir(base_path):
-                if not file.endswith(".png"):
-                    continue
-                img_path = os.path.join(base_path, file)
-                img_tar_path = os.path.join(tar_path, file)
-                tar.add(img_path, img_tar_path)
+            try:
+                package_summary_imgs(tar, row, data_path, labels_path)
+            except FileNotFoundError:
+                pass
 
 
 def get_tar_identified_masks(file):

scripts/monitor/setup.py

@@ -9,7 +9,7 @@
 
 setup(
     name="rano-monitor",
-    version="0.0.2",
+    version="0.0.3",
     description="TUI for monitoring medperf datasets",
     url="https://github.com/mlcommons/medperf",
     author="MLCommons",

server/.env.local.local-auth

@@ -4,7 +4,7 @@
 
 DEBUG=True
 SECRET_KEY=I_AM_A_DUMMY_KEY_CHANGE_ME
-DATABASE_URL=sqlite:///db.sqlite3
+DATABASE_URL=postgres://devuser:[email protected]/devdb
 SUPERUSER_USERNAME=admin
 SUPERUSER_PASSWORD=admin
 ALLOWED_HOSTS=*

server/.env.local.local-auth.sqlite

@@ -0,0 +1,23 @@
+##############################################################
+############## ALERT: DO NOT USE FOR PRODUCTION ##############
+##############################################################
+
+DEBUG=True
+SECRET_KEY=I_AM_A_DUMMY_KEY_CHANGE_ME
+DATABASE_URL=sqlite:///db.sqlite3
+SUPERUSER_USERNAME=admin
+SUPERUSER_PASSWORD=admin
+ALLOWED_HOSTS=*
+
+#Valid deployment environments are local, gcp-ci, gcp-prod(case-sensitive)
+DEPLOY_ENV=local
+
+#Production settings when deployed in GCP 
+CORS_ALLOWED_ORIGINS=
+GS_BUCKET_NAME=
+
+#Auth configuration
+AUTH_AUDIENCE=https://localhost-localdev/
+AUTH_ISSUER=https://localhost:8000/
+AUTH_JWK_URL=
+AUTH_VERIFYING_KEY="-----BEGIN PUBLIC KEY-----\nMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAtKO1SzU6N/sZTJmYNk0C\n/5XbK8eWfcKX2HxFl7fr0V++wrXXGsMs9A8hQEbVWtgYbWaOSkXN0ojmcUt1NFcb\nSPYLmOK/oUXVASEbuZAdIi+ByQ1EnIIAmYSKjRBDUQM8wc73Z9AvrjnhrvEHyrIN\nKyXeLnaCKj/r0s5sQA85SngnCWQbZsRQyHysfsQLwguG0SKFF9EfdNJiaoD8lLBo\nqvUQIYi8MXuVAB7O5EomJoZJe7KEeemsLhCnjTlKHcumjnAiRy5Y0rL6aFXgQkg0\nY4NWxMbsIWAplzh2qCs2jEd88mAUJnHkMzeOKhb1Q+tcmg6ZG6GmwT9fujsOjYrn\na/RTx83B1rRVRHHBFsEP4/ctVf2VdARz+RO+mIh5yZsPiqmRSKpHfbKgnkBpQlAj\nwVrzP9HYT11EXGFesLKRt6Oin0I5FkJ1Ji4w680XjeyZ4KInMY87OvQtltIyrZI9\nR9uY9EnpISGYch6kxbVw0GzdQdP/0mUnYlIeWwyvsXsWB/b3pZ9BiQuCMtlxoWlk\naRjWk9dWIZKFL2uhgeNeY5Wh3Qx9EFx8hnz9ohdaNBPB5BNO2qI61NedFrjYN9LF\nSfcGL7iATU1JQS4rDisnyjDikkTHL9B1u6sMrTsoaqi9Dl5b0gC8RnPVnJItasMN\n9HcW8Pfo2Ava4ler7oU47jUCAwEAAQ==\n-----END PUBLIC KEY-----"

server/.env.local.online-auth

@@ -4,7 +4,7 @@
 
 DEBUG=True
 SECRET_KEY=I_AM_A_DUMMY_KEY_CHANGE_ME
-DATABASE_URL=sqlite:///db.sqlite3
+DATABASE_URL=postgres://devuser:[email protected]/devdb
 SUPERUSER_USERNAME=admin
 SUPERUSER_PASSWORD=admin
 ALLOWED_HOSTS=*

server/dataset/migrations/0005_alter_dataset_generated_metadata.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.11 on 2024-11-29 16:23
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("dataset", "0004_auto_20231211_1827"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="dataset",
+            name="generated_metadata",
+            field=models.JSONField(blank=True, default=dict),
+        ),
+    ]

server/dataset/models.py

@@ -27,7 +27,7 @@ class Dataset(models.Model):
     state = models.CharField(
         choices=DATASET_STATE, max_length=100, default="DEVELOPMENT"
     )
-    generated_metadata = models.JSONField(default=dict, blank=True, null=True)
+    generated_metadata = models.JSONField(default=dict, blank=True)
     user_metadata = models.JSONField(default=dict, blank=True, null=True)
     report = models.JSONField(default=dict, blank=True, null=True)
     created_at = models.DateTimeField(auto_now_add=True)