Merge pull request #248 from MetaCell/release/1.3.0

Release/1.3.0

.github/pull_request_template.md

@@ -0,0 +1,40 @@
+Closes [AREG-X](https://metacell.atlassian.net/browse/AREG-X)
+
+# Implemented solution
+
+... 
+
+# How to test this PR
+
+...
+
+# Sanity checks:
+- [ ] The pull request is explicitly linked to the relevant issue(s)
+- [ ] The issue is well described: clearly states the problem and the general proposed solution(s)
+- [ ] In this PR it is explicitly stated how to test the current change
+- [ ] The labels in the issue set the scope and the type of issue (bug, feature, etc.)
+- [ ] The relevant components are indicated in the issue (if any)
+- [ ] All the automated test checks are passing
+- [ ] All the linked issues are included in one Sprint
+- [ ] All the linked issues are in the Review state
+- [ ] All the linked issues are assigned
+
+# Breaking changes (select one):
+- [ ] The present changes do not change the preexisting api in any way
+- [ ] This PR and the issue are tagged as a `breaking-change` and the migration procedure is well described [above](#implemented-solution)
+
+# Possible deployment updates issues (select one):
+- [ ] There is no reason why deployments based on CloudHarness may break after the current update
+- [ ] This PR and the issue are tagged as `alert:deployment`
+
+### Test coverage (select one):
+- [ ] Tests for the relevant cases are included in this pr
+- [ ] The changes included in this pr are out of the current test coverage scope
+
+### Documentation (select one):
+- [ ] The documentation has been updated to match the current changes
+- [ ] The changes included in this PR are out of the current documentation scope
+
+### Nice to have (if relevant):
+- [ ] Screenshots of the changes
+- [ ] Explanatory video/animated gif

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,70 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ develop, master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ develop ]
+  schedule:
+    - cron: '22 12 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'javascript', 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://git.io/codeql-language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/trivy-analysis.yml

@@ -0,0 +1,56 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: build
+
+on:
+  push:
+    branches: [ develop, master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ develop ]
+  schedule:
+    - cron: '30 12 * * 5'
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    permissions:
+      contents: read # for actions/checkout to fetch code
+      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
+    name: Build
+    runs-on: "ubuntu-20.04"
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Checkout cloudharness
+        uses: actions/checkout@v3
+        with:
+          repository: 'MetaCell/cloud-harness'
+          path: 'cloud-harness'
+
+      - name: Build an image from Dockerfile
+        run: |
+          ls
+          ls cloud-harness/infrastructure/base-images
+          DOCKER_BUILDKIT=1 docker build -t cloudharness-base-debian:${{ github.sha }} cloud-harness -f ./cloud-harness/infrastructure/base-images/cloudharness-base-debian/Dockerfile
+          ls ./cloud-harness/infrastructure/common-images/cloudharness-django/Dockerfile
+          DOCKER_BUILDKIT=1 docker build -t cloudharness-django:${{ github.sha }} --build-arg CLOUDHARNESS_BASE_DEBIAN=cloudharness-base-debian:${{ github.sha }} ./cloud-harness/infrastructure/common-images/cloudharness-django
+          DOCKER_BUILDKIT=1 docker build -t portal:${{ github.sha }}  --build-arg CLOUDHARNESS_DJANGO=cloudharness-django:${{ github.sha }} ./applications/portal
+
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: 'portal:${{ github.sha }}'
+          format: 'sarif'
+          output: 'trivy-results.sarif'
+          severity: 'CRITICAL,HIGH'
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: 'trivy-results.sarif'

applications/accounts-api/backend/requirements.txt

@@ -1,10 +1,4 @@
-connexion[swagger-ui] >= 2.6.0; python_version>="3.6"
-# 2.3 is the last version that supports python 3.4-3.5
-connexion[swagger-ui] <= 2.3.0; python_version=="3.5" or python_version=="3.4"
-# connexion requires werkzeug but connexion < 2.4.0 does not install werkzeug
-# we must peg werkzeug versions below to fix connexion
-# https://github.com/zalando/connexion/pull/1044
-werkzeug == 0.16.1; python_version=="3.5" or python_version=="3.4"
+connexion[swagger-ui] >= 2.6.0;
 swagger-ui-bundle >= 0.0.2
 python_dateutil >= 2.6.0
 setuptools >= 21.0.0

applications/portal/backend/api/admin.py

@@ -99,6 +99,18 @@ class AntibodyFilesAdmin(admin.TabularInline):
     exclude = ("uploader_uid", 'filehash', 'timestamp', 'display_name')
     extra = 1
 
+
+# not in fields - catalog_num_search 
+antibody_fields_shown = (
+    "ab_name", "ab_id", "accession", "commercial_type", "catalog_num", "cat_alt", "vendor", 
+    "url","ab_target", "entrez_id", "uniprot_id", "target_species_raw", "subregion", 
+    "modifications", "epitope", "source_organism", "clonality", "clone_id", "product_isotype",
+    "product_conjugate", "defining_citation", "product_form", "comments",
+    "kit_contents", "feedback", "curator_comment", "disc_date", "status", "show_link",
+    # also in the read-only fields
+    "uid", "uid_legacy", "insert_time", "lastedit_time", "curate_time",
+)
+
 @admin.register(Antibody)
 class AntibodyAdmin(ImportExportModelAdmin):
 
@@ -111,13 +123,13 @@ class AntibodyAdmin(ImportExportModelAdmin):
 
     # list display settings
     list_filter = ("status",)
-    list_display = (id_with_ab, "ab_name", "submitter_name", "status", "vendor", "catalog_num", "accession", "insert_time")
+    list_display = (id_with_ab, "accession", "ab_name", "submitter_name", "status", "vendor", "catalog_num", "insert_time")
     search_fields = ("ab_id", "ab_name", "catalog_num")
 
-    # Edit form settings
-    exclude= ("catalog_num_search",)
+    # the following - maintains the order of the fields
+    fields = antibody_fields_shown
 
-    inlines = [TargetSpeciesInlineAdmin, AntibodyFilesAdmin]
+    inlines = [TargetSpeciesInlineAdmin, AntibodyFilesAdmin, ApplicationsInlineAdmin]
 
     readonly_fields = (
         "submitter_name",

applications/portal/backend/api/mappers/antibody_mapper.py

@@ -22,10 +22,11 @@
 def extract_base_url(url):
     return urlsplit(url).hostname
 
-@cache
+
 def get_vendor_domains(vendor_id):
-    return [vd.base_url for vd in VendorDomain.objects.filter(
-                        vendor_id=vendor_id)]
+    return [vd.base_url for vd in VendorDomain.objects.filter(vendor_id=vendor_id, status=STATUS.CURATED)]
+
+
 class AntibodyMapper(IDAOMapper):
 
     def from_dto(self, dto: AntibodyDTO) -> Antibody:
@@ -35,6 +36,7 @@ def from_dto(self, dto: AntibodyDTO) -> Antibody:
             ab = Antibody()
             ab.ab_id = 0
 
+
         if dto.abTarget:
             # antigen_symbol = dto.abTarget
             # del dto.abTarget
@@ -50,7 +52,10 @@ def from_dto(self, dto: AntibodyDTO) -> Antibody:
                 log.info("Adding specie: %s", specie_name)
 
         if dto.url or dto.vendorName:
-            ab.set_vendor_from_name_url(url=dto.url, name=dto.vendorName)
+            ab.set_vendor_from_name_url(
+                url=dto.url, name=dto.vendorName, 
+                commercial_type=dto.commercialType.value if dto.commercialType else None
+            )
         else:
             raise AntibodyDataException(
                 "Either vendor url or name is mandatory", 'url/name', None)
@@ -128,8 +133,7 @@ def to_dto(self, dao: Antibody) -> AntibodyDTO:
             ab.sourceOrganism = dao.source_organism.name
         if dao.species and not ab.targetSpecies:
             ab.targetSpecies = [s.name for s in dao.species.all()]
-
-
+
         ab.url = get_url_if_permitted(dao)
 
         ab.showLink = dao.show_link if dao.show_link is not None else (dao.vendor and dao.vendor.show_link)

applications/portal/backend/api/migrations/0011_remove_antibodysearch_citations_and_more.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.9 on 2024-04-21 11:30
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0010_auto_20240315_0451'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='vendor',
+            name='commercial_type',
+            field=models.CharField(choices=[('commercial', 'commercial'), ('personal', 'personal'), ('non-profit', 'non-profit'), ('other', 'other')], db_index=True, default='commercial', max_length=32, null=True),
+        ),
+    ]