Merge pull request #257 from MetaCell/release/1.4.0

Release/1.4.0

.github/workflows/trivy-analysis.yml

@@ -3,7 +3,7 @@
 # separate terms of service, privacy policy, and support
 # documentation.
 
-name: build
+name: Trivy analysis
 
 on:
   push:

applications/accounts-api/backend/accounts_api/services/user_service.py

@@ -163,4 +163,3 @@ def associate_orcid_id(userid: str, orcid: str):
 
 def validate_orcid_id( orcid: str) -> bool:
     return re.match(r'^https://orcid.org/\d{4}-\d{4}-\d{4}-\d{3}[0-9X]$', orcid)
-

applications/portal/api/openapi.yaml

@@ -60,7 +60,7 @@ paths:
                         query parameter added as - status=rejected:
                             value: /api/antibodies?page=1&size=100&status=rejected
                     name: status
-                    description: 'Add a status to filter the query - CURATED, REJECTED, QUEUE. '
+                    description: 'Add a status to filter the query - CURATED, REJECTED, QUEUE, UNDER_REVIEW. '
                     schema:
                         type: string
                     in: query
@@ -389,6 +389,20 @@ paths:
             operationId: filterAntibodies
             summary: Search on Antibodies with custom filters
             description: Gets a list of `Antibody` entities related with the body parameters
+    /antibodies/export/admin:
+        summary: Exports all antobodies in csv format
+        get:
+            tags:
+                - antibody
+            responses:
+                '200':
+                    content:
+                        text/csv: {}
+                    description: All antibodies in csv format
+            security:
+                -
+                    bearerAuth: []
+                    cookieAuth: []
 components:
     schemas:
         KeyValuePair:
@@ -423,81 +437,6 @@ components:
                     type: array
                     items:
                         $ref: '#/components/schemas/Antibody'
-        Antibody:
-            description: The data type associated with the antibody resource
-            type: object
-            allOf:
-                -
-                    required: []
-                    type: object
-                    properties:
-                        accession:
-                            description: >
-                                Thus value is the same as the Antibody identifier for newly added antibodies,
-                                different if antibody records have been consolidated or are not unique.
-                            type: string
-                        status:
-                            description: |
-                                Can include: curated, rejected, queue
-                            enum:
-                                - CURATED
-                                - REJECTED
-                                - QUEUE
-                            type: string
-                        feedback:
-                            description: Feedback to the submitted stored here
-                            type: string
-                        abId:
-                            description: Antibody identifier
-                            type: string
-                        catAlt:
-                            description: >-
-                                The alternative catalog numbers for this product, delimited by comma, e.g.,
-                                9101S, 9101P, 9191L
-                            type: string
-                        curateTime:
-                            format: date-time
-                            description: Unix time stamp when the row was last updated
-                            type: string
-                        curatorComment:
-                            description: |
-                                Curator comment about this reagent
-                            type: string
-                        discDate:
-                            description: The date on which the antibody product was found to be discontinued
-                            type: string
-                        insertTime:
-                            format: date-time
-                            description: Unix time stamp when the row was inserted.
-                            type: string
-                        targetModification:
-                            description: Any modification to the target protein
-                            type: string
-                        targetSubregion:
-                            description: The subregion of the target protein that the epitope is contained in
-                            type: string
-                        vendorId:
-                            description: ''
-                            type: integer
-                        lastEditTime:
-                            format: date-time
-                            description: ''
-                            type: string
-                        ix:
-                            description: ''
-                            type: integer
-                        showLink:
-                            description: ''
-                            type: boolean
-                        vendorUrl:
-                            description: ''
-                            type: array
-                            items:
-                                type: string
-                -
-                    $ref: '#/components/schemas/AbstractAntibody'
-                -
-                    $ref: '#/components/schemas/AntibodyCoreId'
         AbstractAntibody:
             description: The common fields between all REST operations for the antibody resource.
             required: []
@@ -617,6 +556,9 @@ components:
                 abTargetUniprotId:
                     description: ''
                     type: string
+                numOfCitation:
+                    description: Number of citation for an Antibody - kept track using cronjob from scicrunch.
+                    type: integer
         DataInfo:
             description: Information about the data in the system
             required:
@@ -816,6 +758,81 @@ components:
                 - size
                 - search
             type: string
+        Antibody:
+            description: The data type associated with the antibody resource
+            type: object
+            allOf:
+                -
+                    required: []
+                    type: object
+                    properties:
+                        accession:
+                            description: >
+                                Thus value is the same as the Antibody identifier for newly added antibodies,
+                                different if antibody records have been consolidated or are not unique.
+                            type: string
+                        status:
+                            description: 'Can include: curated, rejected, queue, under_review'
+                            enum:
+                                - CURATED
+                                - REJECTED
+                                - QUEUE
+                                - UNDER_REVIEW
+                            type: string
+                        feedback:
+                            description: Feedback to the submitted stored here
+                            type: string
+                        abId:
+                            description: Antibody identifier
+                            type: string
+                        catAlt:
+                            description: >-
+                                The alternative catalog numbers for this product, delimited by comma, e.g.,
+                                9101S, 9101P, 9191L
+                            type: string
+                        curateTime:
+                            format: date-time
+                            description: Unix time stamp when the row was last updated
+                            type: string
+                        curatorComment:
+                            description: |
+                                Curator comment about this reagent
+                            type: string
+                        discDate:
+                            description: The date on which the antibody product was found to be discontinued
+                            type: string
+                        insertTime:
+                            format: date-time
+                            description: Unix time stamp when the row was inserted.
+                            type: string
+                        targetModification:
+                            description: Any modification to the target protein
+                            type: string
+                        targetSubregion:
+                            description: The subregion of the target protein that the epitope is contained in
+                            type: string
+                        vendorId:
+                            description: ''
+                            type: integer
+                        lastEditTime:
+                            format: date-time
+                            description: ''
+                            type: string
+                        ix:
+                            description: ''
+                            type: integer
+                        showLink:
+                            description: ''
+                            type: boolean
+                        vendorUrl:
+                            description: ''
+                            type: array
+                            items:
+                                type: string
+                -
+                    $ref: '#/components/schemas/AbstractAntibody'
+                -
+                    $ref: '#/components/schemas/AntibodyCoreId'
     responses:
         PaginatedAntibodies:
             content:

applications/portal/api/templates/main.jinja2

@@ -92,10 +92,10 @@ if os.environ.get('KUBERNETES_SERVICE_HOST', None):
 
 
 # enable the Bearer Authentication
-#security = APIKeyCookie(name="kc-access") # This works but having issues with dev server
-security = HTTPBearer()
+# security = APIKeyCookie(name="kc-access") # This works but having issues with dev server
+# security = HTTPBearer()
 
-async def has_access(credentials: HTTPBasicCredentials = Depends(security)):
+async def has_access():
     """
     Function that is used to validate the token in the case that it requires it
     """

applications/portal/backend/README.md

@@ -28,6 +28,11 @@ pip install -e applications/portal/backend
 mkdir -p /opt/cloudharness/resources/auth/
 kubectl -n areg get secrets accounts -o yaml|grep api_user_password|cut -d " " -f 4|base64 -d > /opt/cloudharness/resources/auth/api_user_password
 
+# store the Scicrunch API_KEY on the local disk
+mkdir -p /opt/cloudharness/resources/secrets/
+kubectl -n areg get secrets scicrunch-api-key -o yaml|grep scicrunch-api-key|cut -d " " -f 4|base64 -d > /opt/cloudharness/resources/secrets/scicrunch-api-key
+
+
 # Make the cloudharness application configuration available on your local machine
 cp deployment/helm/values.yaml /opt/cloudharness/resources/allvalues.yaml
 ```

applications/portal/backend/api/admin.py

@@ -102,7 +102,7 @@ class AntibodyFilesAdmin(admin.TabularInline):
 
 # not in fields - catalog_num_search 
 antibody_fields_shown = (
-    "ab_name", "ab_id", "accession", "commercial_type", "catalog_num", "cat_alt", "vendor", 
+    "ab_name", "ab_id", "accession", "commercial_type", "catalog_num", "cat_alt", "citation", "vendor", 
     "url","ab_target", "entrez_id", "uniprot_id", "target_species_raw", "subregion", 
     "modifications", "epitope", "source_organism", "clonality", "clone_id", "product_isotype",
     "product_conjugate", "defining_citation", "product_form", "comments",
@@ -123,7 +123,7 @@ class AntibodyAdmin(ImportExportModelAdmin):
 
     # list display settings
     list_filter = ("status",)
-    list_display = (id_with_ab, "accession", "ab_name", "submitter_name", "status", "vendor", "catalog_num", "insert_time")
+    list_display = (id_with_ab, "accession", "ab_name", "submitter_name", "citation", "status", "vendor", "catalog_num", "insert_time")
     search_fields = ("ab_id", "ab_name", "catalog_num")
 
     # the following - maintains the order of the fields

applications/portal/backend/api/controllers/antibody_controller.py

@@ -3,15 +3,14 @@
 import os
 
 from cloudharness import log
-from api.models import Antibody
-from api.services.user_service import UnrecognizedUser, get_current_user_id
-
+from api.models import Antibody, STATUS
+from api.services.user_service import UnrecognizedUser, get_current_user_id, check_if_user_is_admin
 
 from fastapi import HTTPException
 from fastapi.encoders import jsonable_encoder
 from fastapi.responses import JSONResponse, RedirectResponse
 
-from api.services import antibody_service
+from api.services import antibody_service, filesystem_service
 from api.utilities.exceptions import AntibodyDataException
 
 from openapi.models import AddAntibody as AddAntibodyDTO, PaginatedAntibodies
@@ -27,8 +26,11 @@ def get_antibodies(page: int, size: int, updated_from: datetime, updated_to: dat
         raise HTTPException(status_code=400, detail="Pages start at 1")
     if size < 1:
         raise HTTPException(status_code=400, detail="Size must be greater than 0")
-    if size > 100:
-        raise HTTPException(status_code=400, detail="Size must be less than 100")
+    if page * size > 500:
+        try:
+            get_current_user_id()
+        except UnrecognizedUser:
+            raise HTTPException(status_code=401, detail="Request not allowed")
     try:
         return antibody_service.get_antibodies(int(page), int(size), updated_from, updated_to, status)
     except ValueError:
@@ -87,7 +89,7 @@ def get_by_accession(accession_number: int) -> AntibodyDTO:
     try:
         return antibody_service.get_antibody_by_accession(accession_number)
     except Antibody.DoesNotExist as e:
-        raise HTTPException(status_code=404, detail=e.message)
+        raise HTTPException(status_code=404, detail="Antibody not found")
 
 
 def get_antibodies_export():
@@ -96,7 +98,27 @@ def get_antibodies_export():
 
     # check if file exists and it is created within 24 hours
     # if not, generate a new file
-    if not os.path.exists(fname) or (datetime.now() - datetime.fromtimestamp(os.path.getmtime(fname))).days > 1:
+    if filesystem_service.check_if_file_exists_and_recent(fname):
         generate_antibodies_csv_file(fname)
     return RedirectResponse("/" + fname)
-    # return FileResponse(fname, filename="antibodies_export.csv")
+    # return FileResponse(fname, filename="antibodies_export.csv")
+
+
+
+def get_antibodies_export_admin():
+    """
+    Export all fields of all antibodies to a CSV file - Only for admin users
+    """
+    try:
+        is_admin = check_if_user_is_admin()
+    except Exception as e:
+        raise HTTPException(status_code=401, detail="Unauthorized: Only admin users can access this endpoint")
+    from api.services.export_service import generate_all_antibodies_fields_to_csv
+    fname = "static/www/antibodies_admin_export.csv"
+
+    if filesystem_service.check_if_file_exists_and_recent(fname) and is_admin:
+        generate_all_antibodies_fields_to_csv(fname)
+    return RedirectResponse("/" + fname)
+
+
+

applications/portal/backend/api/controllers/ingest_controller.py

@@ -3,6 +3,9 @@
 from api.services.workflow_service import execute_ingestion_workflow
 from openapi.models import IngestRequest
 from api.services.keycloak_service import KeycloakService
+from api.models import Antibody, STATUS
+from api.utilities.exceptions import AntibodyDoesNotExist
+
 
 def ingest(body: IngestRequest):
     auth = KeycloakService()
@@ -15,3 +18,6 @@ def ingest(body: IngestRequest):
                                 detail="You are not authorized to ingest data.")
     execute_ingestion_workflow(body.driveLinkOrId, body.hot)
     return status.HTTP_200_OK
+
+
+

applications/portal/backend/api/controllers/search_controller.py

@@ -2,13 +2,25 @@
 from fastapi import HTTPException
 
 from api.services import search_service, antibody_service
+from api.services.user_service import UnrecognizedUser, get_current_user_id
 from openapi.models import FilterRequest, PaginatedAntibodies
 from openapi.models import Antibody as AntibodyDTO
 
 
 def fts_antibodies(page: int = 1, size: int = 50, search: str = '') -> PaginatedAntibodies:
-    if size > 100:
-        raise HTTPException(status_code=400, detail="Size must be less than 100")
+    if page is None:
+        page = 1
+    if size is None:
+        size = 50
+    if page < 1:
+        raise HTTPException(status_code=400, detail="Pages start at 1")
+    if size < 1:
+        raise HTTPException(status_code=400, detail="Size must be greater than 0")
+    if page * size > 500:
+        try:
+            get_current_user_id()
+        except UnrecognizedUser:
+            raise HTTPException(status_code=401, detail="Request not allowed")
     if search.startswith("AB_"):
         try:
             a = antibody_service.get_antibody(int(search.replace("AB_", "")), accession=int(search.replace("AB_", "")))
@@ -18,7 +30,7 @@ def fts_antibodies(page: int = 1, size: int = 50, search: str = '') -> Paginated
     return search_service.fts_antibodies(page, size, search)
 
 
-def filter_antibodies(filter_request: FilterRequest) -> PaginatedAntibodies:
-    if filter_request.size > 100:
+def filter_antibodies(body: FilterRequest) -> PaginatedAntibodies:
+    if body.size > 100:
         raise HTTPException(status_code=400, detail="Size must be less than 100")
-    return search_service.filter_antibodies(filter_request)
+    return search_service.filter_antibodies(body)