Merge pull request #211 from MetaCell/develop

Release 1.1.0

applications/common/deploy/values-prod.yaml

@@ -1,4 +1,5 @@
 harness:
   env:
     - name: SENTRY_DSN
-      value: https://[email protected]/9
+      value: https://57b1ca8a70634782bd6c90dd119eeab5@o4506739169951744.ingest.sentry.io/4506758817382404
+

applications/pgadmin/deploy/charts/values.yaml

@@ -18,9 +18,11 @@ pgadmin:
 
   ## pgadmin config, Any custom environment variabls. Settings in config.py can be overriden with an environment variable using the prefix: PGADMIN_CONFIG_
   ## eg turn off enhanced cookie protection for default AKS, loadbalancer installation
-  #config:
-  #  - name: "PGADMIN_CONFIG_ENHANCED_COOKIE_PROTECTION"
-  #    value: "False"
+  config:
+    - name: "PGADMIN_CONFIG_ENHANCED_COOKIE_PROTECTION"
+      value: "False"
+    - name: "PGADMIN_CONFIG_WTF_CSRF_CHECK_DEFAULT"
+      value: "False"
 
 ## Enable persistence using Persistent Volume Claims
 ## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/

applications/portal/Dockerfile

@@ -1,7 +1,6 @@
-ARG CLOUDHARNESS_FRONTEND_BUILD
 ARG CLOUDHARNESS_DJANGO
 
-FROM $CLOUDHARNESS_FRONTEND_BUILD as frontend
+FROM node:20 as frontend
 
 ENV APP_DIR=/app
 
@@ -17,6 +16,10 @@ RUN yarn build
 
 FROM $CLOUDHARNESS_DJANGO
 
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    postgresql-client \
+    && rm -rf /var/lib/apt/lists/*
+
 WORKDIR ${APP_DIR}
 RUN mkdir -p ${APP_DIR}/static/www
 

applications/portal/README.md

@@ -31,7 +31,7 @@ See [here](backend/README.md#Develop)
 
 ### Frontend
 
-Backend code is inside the *frontend* directory.
+Frontend code is inside the *frontend* directory.
 
 Frontend is by default generated as a React web application, but no constraint about this specific technology.
 
@@ -111,7 +111,7 @@ The `KUBERNETES_SERVICE_HOST` switch will activate the creation of the keycloak
 
 The base command to run tests is `python manage.py test`.
 
-To run tests locally eed to add configure environmental variables to get the 
+To run tests locally you need to add configure environmental variables to get the 
 correct configuration and a working configure instance of the postgres database running.
 
 If you already have a deployment with the database, first forward it:

applications/portal/api/openapi.yaml

@@ -55,6 +55,16 @@ paths:
                         format: date-time
                         type: string
                     in: query
+                -
+                    examples:
+                        query parameter added as - status=rejected:
+                            value: /api/antibodies?page=1&size=100&status=rejected
+                    name: status
+                    description: 'Add a status to filter the query - CURATED, REJECTED, QUEUE. '
+                    schema:
+                        type: string
+                    in: query
+                    required: false
             responses:
                 '200':
                     content:
@@ -369,6 +379,16 @@ paths:
                     type: integer
                 in: path
                 required: true
+    /antibodies/export:
+        summary: Exports all antobodies in csv format
+        get:
+            tags:
+                - antibody
+            responses:
+                '200':
+                    content:
+                        text/csv: {}
+                    description: All antibodies in csv format
 components:
     schemas:
         FilterRequest:
@@ -739,3 +759,19 @@ components:
             bearerFormat: JWT
             type: http
             x-bearerInfoFunc: cloudharness.auth.decode_token
+tags:
+    -
+        name: antibody
+        description: ''
+    -
+        name: search
+        description: ''
+    -
+        name: api
+        description: ''
+    -
+        name: test
+        description: ''
+    -
+        name: ingest
+        description: ''

applications/portal/backend/README.md

@@ -10,6 +10,17 @@ This example uses the [Django](https://www.djangoproject.com/) library on top of
 ## Requirements
 Python >= 3
 
+Cloudharness libraries (check also dockerfile chain)
+
+```
+pip install -e cloud-harness/libraries/models
+pip install -e cloud-harness/libraries/cloudharness-common
+pip install -r cloud-harness/infrastructure/common-images/cloudharness-fastapi/libraries/fastapi/requirements.txt 
+pip install -e cloud-harness/infrastructure/common-images/cloudharness-django/libraries/cloudharness-django
+pip install -r applications/portal/backend/requirements.txt
+pip install -e applications/portal/backend
+```
+
 ## Local backend development
 ```
 # store the accounts api admin password on the local disk

applications/portal/backend/api/admin.py

@@ -12,7 +12,7 @@
 from django.utils.encoding import smart_str
 from django.utils.html import escape, format_html, format_html_join, mark_safe
 from django.utils.text import format_lazy
-from import_export.admin import ImportMixin
+from import_export.admin import ImportExportModelAdmin
 from keycloak.exceptions import KeycloakGetError
 from cloudharness_django.models import Member
 
@@ -100,7 +100,7 @@ class AntibodyFilesAdmin(admin.TabularInline):
     extra = 1
 
 @admin.register(Antibody)
-class AntibodyAdmin(ImportMixin, admin.ModelAdmin):
+class AntibodyAdmin(ImportExportModelAdmin):
 
     change_form_template = "admin/antibody_change_form.html"
 

applications/portal/backend/api/controllers/antibody_controller.py

@@ -8,7 +8,7 @@
 
 from fastapi import HTTPException
 from fastapi.encoders import jsonable_encoder
-from fastapi.responses import JSONResponse
+from fastapi.responses import JSONResponse, RedirectResponse
 
 from api.services import antibody_service
 from api.utilities.exceptions import AntibodyDataException
@@ -17,8 +17,7 @@
 from openapi.models import UpdateAntibody as UpdateAntibodyDTO
 from openapi.models import Antibody as AntibodyDTO
 
-
-def get_antibodies(page: int, size: int, updated_from: datetime, updated_to: datetime) -> PaginatedAntibodies:
+def get_antibodies(page: int, size: int, updated_from: datetime, updated_to: datetime, status=str) -> PaginatedAntibodies:
     if page is None:
         page = 1
     if size is None:
@@ -28,7 +27,7 @@ def get_antibodies(page: int, size: int, updated_from: datetime, updated_to: dat
     if size < 1:
         raise HTTPException(status_code=400, detail="Size must be greater than 0")
     try:
-        return antibody_service.get_antibodies(int(page), int(size), updated_from, updated_to)
+        return antibody_service.get_antibodies(int(page), int(size), updated_from, updated_to, status)
     except ValueError:
         raise HTTPException(status_code=400, detail="Page and size must be integers")
 
@@ -80,3 +79,15 @@ def get_by_accession(accession_number: int) -> AntibodyDTO:
         return antibody_service.get_antibody_by_accession(accession_number)
     except Antibody.DoesNotExist as e:
         raise HTTPException(status_code=404, detail=e.message)
+
+
+def get_antibodies_export():
+    from api.services.export_service import generate_antibodies_csv_file
+    fname = "static/www/antibodies_export.csv"
+
+    # check if file exists and it is created within 24 hours
+    # if not, generate a new file
+    if not os.path.exists(fname) or (datetime.now() - datetime.fromtimestamp(os.path.getmtime(fname))).days > 1:
+        generate_antibodies_csv_file(fname)
+    return RedirectResponse("/" + fname)
+    # return FileResponse(fname, filename="antibodies_export.csv")

applications/portal/backend/api/migrations/0008_auto_20240215_0459.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.9 on 2024-02-15 12:59
+
+from django.db import migrations, connection
+from cloudharness import log
+from api.repositories.maintainance import rechunk_catalog_number
+
+def chunk_catalog_number(apps, schema_editor):
+    Antibody = apps.get_model('api', 'Antibody')
+    rechunk_catalog_number(Antibody)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("api", "0007_alter_antibody_comments"),
+    ]
+    operations = [
+        migrations.RunPython(chunk_catalog_number)
+    ]

applications/portal/backend/api/migrations/0009_auto_20240220_0451.py

@@ -0,0 +1,72 @@
+# Generated by Django 4.2.10 on 2024-02-20 12:51
+
+import api.models
+import django.contrib.postgres.indexes
+import django.contrib.postgres.search
+from django.db import migrations, models
+import django.db.models.deletion
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("api", "0008_auto_20240215_0459"),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            sql="""
+            DROP MATERIALIZED VIEW IF EXISTS antibody_search;
+            CREATE MATERIALIZED VIEW antibody_search AS 
+            SELECT ix, 
+            (
+                setweight(to_tsvector('english'::regconfig, (((
+                COALESCE(ab_name, ''::text) || ' '::text) || 
+                        COALESCE(clone_id, ''::text) || ' '::text)) 
+            ), 'A'::"char") ||
+                
+                setweight(to_tsvector('english'::regconfig, (((((((((((((((((((((((((
+                    COALESCE(api_vendor.vendor, ''::text) || ' '::text) || 
+                    COALESCE(api_specie.name, ''::text) || ' '::text) || 
+                    COALESCE(target_subregion, ''::text) || ' '::text) || 
+                        COALESCE(clonality, ''::text)) || ' '::text) || 
+                    COALESCE(target_modification, ''::text)) || ' '::text) || 
+                    COALESCE(epitope, ''::character varying)::text) || ' '::text) || 
+                    COALESCE(product_isotype, ''::character varying)::text) || ' '::text) || 
+					COALESCE(ab_target, ''::character varying)::text) || ' '::text) ||
+					COALESCE(ab_target_entrez_gid, ''::character varying)::text) || ' '::text) ||
+					COALESCE(uniprot_id, ''::character varying)::text) || ' '::text) ||
+					COALESCE(product_isotype, ''::character varying)::text) || ' '::text) ||
+                COALESCE(product_conjugate, ''::text) || 
+                COALESCE(product_form, ''::character varying)::text) || ' '::text) || 
+                COALESCE(target_species_raw, ''::character varying)::text) || ' '::text) || 
+                COALESCE(kit_contents, ''::character varying)::text) || ' '::text)), 'C'::"char") ||
+            setweight(to_tsvector('english'::regconfig, (((
+                COALESCE(comments, ''::text) || ' '::text) || 
+                        COALESCE(curator_comment, ''::text) || ' '::text)) 
+            ), 'D'::"char")
+
+            ) AS search_vector,
+            defining_citation,
+            disc_date,
+            status
+            FROM api_antibody 
+            LEFT JOIN api_vendor ON api_vendor.id = api_antibody.vendor_id
+            LEFT JOIN api_specie ON api_specie.id = api_antibody.source_organism_id
+            """,
+            reverse_sql = '''
+            DROP MATERIALIZED VIEW antibody_search;
+            '''
+        ),
+        migrations.RunSQL(
+            sql="""CREATE UNIQUE INDEX IF NOT EXISTS antibody_search_idx
+                    ON antibody_search
+                    (ix);
+                    """,
+        ),
+        migrations.AddIndex(
+            model_name="antibodysearch",
+            index=django.contrib.postgres.indexes.GinIndex(
+                fields=["search_vector"], name="antibody_search_fts_idx"
+            ),
+        )
+    ]

applications/portal/backend/api/models.py

@@ -277,8 +277,10 @@ def import_save(self):
         if first_save:
             super().save()
             self._handle_duplicates()
+
         if self.catalog_num:
-            self.catalog_num_search = catalog_number_chunked(self.catalog_num)
+            self.catalog_num_search = catalog_number_chunked(self.catalog_num, self.cat_alt)
+
         if not self.accession or not self.ab_id:
             self._generate_automatic_attributes()
         super().save()
@@ -296,7 +298,7 @@ def save(self, *args, update_search=True, **kwargs):
         super(Antibody, self).save(*args, **kwargs)
 
         if self.catalog_num:
-            self.catalog_num_search = catalog_number_chunked(self.catalog_num)
+            self.catalog_num_search = catalog_number_chunked(self.catalog_num, self.cat_alt)
         self._handle_duplicates(*args, **kwargs)
         self._generate_related_fields(*args, **kwargs)
         self._synchronize_target_species(old_species)

applications/portal/backend/api/repositories/maintainance.py

@@ -1,5 +1,7 @@
 from django.db import connection
 from cloudharness import log
+from api.utilities.functions import catalog_number_chunked
+
 
 def refresh_search_view():
     import threading
@@ -8,4 +10,21 @@ def refresh_search_view_thread():
         with connection.cursor() as cursor:
             cursor.execute("REFRESH MATERIALIZED VIEW CONCURRENTLY antibody_search;")
         connection.commit()
-    threading.Thread(target=refresh_search_view_thread).start()
+    threading.Thread(target=refresh_search_view_thread).start()
+
+
+def rechunk_catalog_number(Antibody_model):
+    i = 0
+    from api.models import STATUS
+    with connection.cursor() as cursor:
+        for antibody in Antibody_model.objects.filter(status=STATUS.CURATED).values('ix', 'catalog_num_search', 'catalog_num', 'cat_alt'):
+            i = i + 1
+            if i % 10000 == 0:
+               print("Migrated", i)
+            new_catalog_number_chunked = catalog_number_chunked(antibody['catalog_num'], antibody['cat_alt'])
+            if new_catalog_number_chunked != antibody['catalog_num_search']:
+
+                try:
+                    cursor.execute(f"UPDATE api_antibody SET catalog_num_search = '{new_catalog_number_chunked}' WHERE ix={antibody['ix']};") 
+                except Exception as e:
+                    log.exception("`%s` `%s` %s`", antibody['catalog_num_search'], antibody['catalog_num'], antibody['cat_alt'])

applications/portal/backend/api/repositories/search_repository.py

@@ -64,7 +64,8 @@ def fts_antibodies(page: int = 0, size: int = settings.LIMIT_NUM_RESULTS, search
     # search only allows alphanumeric characters and spaces
 
     if might_be_catalog_number(search):
-        cat_search = fts_by_catalog_number(re.sub(r'[^\w\s]', '', search), page, size)
+        cat_search = fts_by_catalog_number(search, page, size)
+
 
         if cat_search:
             return cat_search

applications/portal/backend/api/services/antibody_service.py

@@ -10,9 +10,9 @@
 from api.utilities.exceptions import DuplicatedAntibody, AntibodyDataException
 from cloudharness import log
 from openapi.models import AddAntibody as AddAntibodyDTO
-from openapi.models import UpdateAntibody as UpdateAntibodyDTO
+from openapi.models import UpdateAntibody as UpdateAntibodyDTO, Status
 from openapi.models import Antibody as AntibodyDTO, PaginatedAntibodies
-
+from api.utilities.functions import check_if_status_exists_or_curated
 antibody_mapper = AntibodyMapper()
 
 
@@ -25,9 +25,9 @@ def search_antibodies_by_catalog(search: str, page: int = 1, size: int = 50,
     return PaginatedAntibodies(page=int(page), totalElements=p.count, items=items)
 
 
-def get_antibodies(page: int = 1, size: int = 50, date_from: datetime = None, date_to: datetime = None) -> PaginatedAntibodies:
+def get_antibodies(page: int = 1, size: int = 50, date_from: datetime = None, date_to: datetime = None, status:str=None) -> PaginatedAntibodies:
     try:
-        query = Antibody.objects.filter(status=STATUS.CURATED)
+        query = Antibody.objects.filter(status=check_if_status_exists_or_curated(status))
         if date_from:
             query = query.filter(lastedit_time__gte=date_from)
         if date_to:
@@ -53,9 +53,6 @@ def create_antibody(body: AddAntibodyDTO, userid: str) -> AntibodyDTO:
     antibody.uid = userid
     antibody.save()
 
-    if antibody.get_duplicate():
-        raise DuplicatedAntibody(antibody_mapper.to_dto(antibody))
-
     return antibody_mapper.to_dto(antibody)