Clean up backend tests

.github/actions/deploy-service/action.yaml

@@ -14,6 +14,16 @@ inputs:
     description: To which cloud cluster to deploy
     required: true
 
+  prioritized:
+    description: Is the service listening on prioritized queues?
+    required: false
+    default: 'false'
+
+  cloud_env:
+    description: Which cloud environment are we deploying to?
+    required: false
+    default: 'default'
+
 runs:
   using: composite
   steps:
@@ -25,10 +35,38 @@ runs:
         AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
         AWS_REGION: ${{ env.AWS_REGION }}
 
-    - name: Deploy image
+    - name: Deploy image (non prioritized)
+      if: inputs.prioritized == 'false'
       shell: bash
       run: kubectl set image deployments/${{ inputs.service }}-dpl ${{ inputs.service }}=${{ inputs.image }}
 
+    - name: Deploy image (prioritized - production)
+      if: inputs.prioritized == 'true' && inputs.cloud_env == 'prod'
+      shell: bash
+      run: |
+        kubectl set image deployments/${{ inputs.service }}-system-dpl ${{ inputs.service }}-system=${{ inputs.image }}
+        kubectl set image deployments/${{ inputs.service }}-normal-dpl ${{ inputs.service }}-normal=${{ inputs.image }}
+        kubectl set image deployments/${{ inputs.service }}-high-dpl ${{ inputs.service }}-high=${{ inputs.image }}
+        kubectl set image deployments/${{ inputs.service }}-urgent-dpl ${{ inputs.service }}-urgent=${{ inputs.image }}
+
+    - name: Deploy image (prioritized - lfx production)
+      if: inputs.prioritized == 'true' && inputs.cloud_env == 'lfx_prod'
+      shell: bash
+      run: |
+        kubectl set image deployments/${{ inputs.service }}-system-dpl ${{ inputs.service }}-system=${{ inputs.image }}
+        kubectl set image deployments/${{ inputs.service }}-normal-dpl ${{ inputs.service }}-normal=${{ inputs.image }}
+        kubectl set image deployments/${{ inputs.service }}-high-dpl ${{ inputs.service }}-high=${{ inputs.image }}
+
+    - name: Deploy image (prioritized - staging)
+      if: inputs.prioritized == 'true' && inputs.cloud_env == 'staging'
+      shell: bash
+      run: kubectl set image deployments/${{ inputs.service }}-normal-dpl ${{ inputs.service }}-normal=${{ inputs.image }}
+
+    - name: Deploy image (prioritized - lfx staging)
+      if: inputs.prioritized == 'true' && inputs.cloud_env == 'lfx_staging'
+      shell: bash
+      run: kubectl set image deployments/${{ inputs.service }}-normal-dpl ${{ inputs.service }}-normal=${{ inputs.image }}
+
     - uses: ./.github/actions/slack-notify
       with:
         message: 'Service *${{ inputs.service }}* was just deployed using docker image `${{ inputs.image }}`'

.github/workflows/CI-node.yaml

@@ -59,50 +59,3 @@ jobs:
       - name: Check app linting, format and typescript
         run: ./scripts/lint_apps.sh
 
-  tests-main:
-    needs: lint-format
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash
-        working-directory: ./backend
-
-    steps:
-      - name: Check out repository code
-        uses: actions/checkout@v2
-
-      - name: Setup Node
-        uses: actions/setup-node@v3
-        with:
-          node-version: 16
-
-      - name: Install dependencies
-        run: cd .. && corepack enable && pnpm i --frozen-lockfile
-
-      - name: Run tests
-        working-directory: ./backend
-        run: SERVICE=test pnpm test -- --testPathIgnorePatterns=serverless
-
-  tests-serverless:
-    needs: lint-format
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash
-        working-directory: ./backend
-
-    steps:
-      - name: Check out repository code
-        uses: actions/checkout@v2
-
-      - name: Setup Node
-        uses: actions/setup-node@v3
-        with:
-          node-version: 16
-
-      - name: Install dependencies
-        run: cd .. && corepack enable && pnpm i --frozen-lockfile
-
-      - name: Run tests
-        working-directory: ./backend
-        run: SERVICE=test pnpm test -- --testPathPattern="serverless\/"

.github/workflows/lf-production-deploy-new.yaml

@@ -8,9 +8,9 @@ on:
         required: true
         type: boolean
       deploy_search_sync_api:
-          description: Deploy search-sync-api service?
-          required: true
-          type: boolean
+        description: Deploy search-sync-api service?
+        required: true
+        type: boolean
       deploy_integration_sync_worker:
         description: Deploy integration-sync-worker service?
         required: true
@@ -64,7 +64,7 @@ jobs:
       - name: Set docker image output
         id: image
         run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
-  
+
   build-and-push-search-sync-api:
     runs-on: ubuntu-latest
     if: ${{ inputs.deploy_search_sync_api }}
@@ -208,6 +208,8 @@ jobs:
           service: search-sync-worker
           image: ${{ needs.build-and-push-search-sync-worker.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: lfx_prod
+          prioritized: true
 
   deploy-search-sync-api:
     needs: build-and-push-search-sync-api
@@ -244,6 +246,8 @@ jobs:
           service: integration-sync-worker
           image: ${{ needs.build-and-push-integration-sync-worker.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: lfx_prod
+          prioritized: true
 
   deploy-webhook-api:
     needs: build-and-push-webhook-api

.github/workflows/lf-production-deploy-original.yaml

@@ -237,6 +237,8 @@ jobs:
           service: nodejs-worker
           image: ${{ needs.build-and-push-backend.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: lfx_prod
+          prioritized: true
 
   deploy-discord-ws:
     needs: build-and-push-backend
@@ -291,6 +293,8 @@ jobs:
           service: integration-run-worker
           image: ${{ needs.build-and-push-integration-run-worker.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: lfx_prod
+          prioritized: true
 
   deploy-integration-stream-worker:
     needs: build-and-push-integration-stream-worker
@@ -309,6 +313,8 @@ jobs:
           service: integration-stream-worker
           image: ${{ needs.build-and-push-integration-stream-worker.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: lfx_prod
+          prioritized: true
 
   deploy-integration-data-worker:
     needs: build-and-push-integration-data-worker
@@ -327,6 +333,8 @@ jobs:
           service: integration-data-worker
           image: ${{ needs.build-and-push-integration-data-worker.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: lfx_prod
+          prioritized: true
 
   deploy-data-sink-worker:
     needs: build-and-push-data-sink-worker
@@ -345,6 +353,8 @@ jobs:
           service: data-sink-worker
           image: ${{ needs.build-and-push-data-sink-worker.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: lfx_prod
+          prioritized: true
 
   deploy-frontend:
     needs: build-and-push-frontend

.github/workflows/lf-staging-deploy-backend.yaml

@@ -76,6 +76,8 @@ jobs:
           service: nodejs-worker
           image: ${{ needs.build-and-push.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: lfx_staging
+          prioritized: true
 
   deploy-job-generator:
     needs: build-and-push

.github/workflows/lf-staging-deploy-data-sink-worker.yaml

@@ -58,3 +58,5 @@ jobs:
           service: data-sink-worker
           image: ${{ needs.build-and-push.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: lfx_staging
+          prioritized: true

.github/workflows/lf-staging-deploy-integration-data-worker.yaml

@@ -58,3 +58,5 @@ jobs:
           service: integration-data-worker
           image: ${{ needs.build-and-push.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: lfx_staging
+          prioritized: true

.github/workflows/lf-staging-deploy-integration-run-worker.yaml

@@ -58,3 +58,5 @@ jobs:
           service: integration-run-worker
           image: ${{ needs.build-and-push.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: lfx_staging
+          prioritized: true

.github/workflows/lf-staging-deploy-integration-stream-worker.yaml

@@ -58,3 +58,5 @@ jobs:
           service: integration-stream-worker
           image: ${{ needs.build-and-push.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: lfx_staging
+          prioritized: true

.github/workflows/lf-staging-deploy-search-sync-worker.yaml

@@ -58,3 +58,5 @@ jobs:
           service: search-sync-worker
           image: ${{ needs.build-and-push.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: lfx_staging
+          prioritized: true

.github/workflows/production-deploy-new.yaml

@@ -64,7 +64,7 @@ jobs:
       - name: Set docker image output
         id: image
         run: echo "IMAGE=${{ steps.image-builder.outputs.image }}" >> $GITHUB_OUTPUT
-  
+
   build-and-push-search-sync-api:
     runs-on: ubuntu-latest
     if: ${{ inputs.deploy_search_sync_api }}
@@ -208,6 +208,8 @@ jobs:
           service: search-sync-worker
           image: ${{ needs.build-and-push-search-sync-worker.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: prod
+          prioritized: true
 
   deploy-search-sync-api:
     needs: build-and-push-search-sync-api
@@ -244,6 +246,8 @@ jobs:
           service: integration-sync-worker
           image: ${{ needs.build-and-push-integration-sync-worker.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: prod
+          prioritized: true
 
   deploy-webhook-api:
     needs: build-and-push-webhook-api

.github/workflows/production-deploy-original.yaml

@@ -237,6 +237,8 @@ jobs:
           service: nodejs-worker
           image: ${{ needs.build-and-push-backend.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: prod
+          prioritized: true
 
   deploy-discord-ws:
     needs: build-and-push-backend
@@ -291,6 +293,8 @@ jobs:
           service: integration-run-worker
           image: ${{ needs.build-and-push-integration-run-worker.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: prod
+          prioritized: true
 
   deploy-integration-stream-worker:
     needs: build-and-push-integration-stream-worker
@@ -309,6 +313,8 @@ jobs:
           service: integration-stream-worker
           image: ${{ needs.build-and-push-integration-stream-worker.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: prod
+          prioritized: true
 
   deploy-integration-data-worker:
     needs: build-and-push-integration-data-worker
@@ -327,6 +333,8 @@ jobs:
           service: integration-data-worker
           image: ${{ needs.build-and-push-integration-data-worker.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: prod
+          prioritized: true
 
   deploy-data-sink-worker:
     needs: build-and-push-data-sink-worker
@@ -345,6 +353,8 @@ jobs:
           service: data-sink-worker
           image: ${{ needs.build-and-push-data-sink-worker.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: prod
+          prioritized: true
 
   deploy-frontend:
     needs: build-and-push-frontend

.github/workflows/staging-deploy-backend.yaml

@@ -76,6 +76,8 @@ jobs:
           service: nodejs-worker
           image: ${{ needs.build-and-push.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          prioritized: true
+          cloud_env: staging
 
   deploy-job-generator:
     needs: build-and-push

.github/workflows/staging-deploy-data-sink-worker.yaml

@@ -58,3 +58,5 @@ jobs:
           service: data-sink-worker
           image: ${{ needs.build-and-push.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: staging
+          prioritized: true

.github/workflows/staging-deploy-integration-data-worker.yaml

@@ -58,3 +58,5 @@ jobs:
           service: integration-data-worker
           image: ${{ needs.build-and-push.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: staging
+          prioritized: true

.github/workflows/staging-deploy-integration-run-worker.yaml

@@ -58,3 +58,5 @@ jobs:
           service: integration-run-worker
           image: ${{ needs.build-and-push.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: staging
+          prioritized: true

.github/workflows/staging-deploy-integration-stream-worker.yaml

@@ -58,3 +58,5 @@ jobs:
           service: integration-stream-worker
           image: ${{ needs.build-and-push.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: staging
+          prioritized: true

.github/workflows/staging-deploy-integration-sync-worker.yaml

@@ -58,3 +58,5 @@ jobs:
           service: integration-sync-worker
           image: ${{ needs.build-and-push.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: staging
+          prioritized: true

.github/workflows/staging-deploy-search-sync-worker.yaml

@@ -58,3 +58,5 @@ jobs:
           service: search-sync-worker
           image: ${{ needs.build-and-push.outputs.image }}
           cluster: ${{ env.CROWD_CLUSTER }}
+          cloud_env: staging
+          prioritized: true

.gitignore

@@ -51,4 +51,4 @@ docker/volume
 
 services/libs/*/dist
 
-**/.cubestore
+**/.cubestore

backend/.env.dist.composed

@@ -1,8 +1,7 @@
 # SQS settings
 CROWD_SQS_HOST="sqs"
 CROWD_SQS_ENDPOINT=http://sqs:9324
-CROWD_SQS_NODEJS_WORKER_QUEUE="http://sqs:9324/000000000000/nodejs-worker.fifo"
-CROWD_SQS_NODEJS_WORKER_DELAYABLE_QUEUE=http://sqs:9324/000000000000/nodejs-worker
+CROWD_SQS_NODEJS_WORKER_QUEUE="http://sqs:9324/000000000000/nodejs-worker-normal.fifo"
 CROWD_SQS_PYTHON_WORKER_QUEUE="http://sqs:9324/000000000000/python-worker.fifo"
 
 # Redis settings

backend/.env.dist.local

@@ -2,6 +2,7 @@
 KUBE_MODE=1
 CROWD_EDITION=community
 TENANT_MODE=multi
+QUEUE_PRIORITY_LEVEL=normal
 
 # API settings
 CROWD_API_URL=https://localhost/api
@@ -15,7 +16,7 @@ CROWD_SQS_HOST=localhost
 CROWD_SQS_PORT=9324
 CROWD_SQS_ENDPOINT=http://localhost:9324
 CROWD_SQS_NODEJS_WORKER_QUEUE=http://localhost:9324/000000000000/nodejs-worker.fifo
-CROWD_SQS_NODEJS_WORKER_DELAYABLE_QUEUE=http://localhost:9324/000000000000/nodejs-worker
+CROWD_SQS_NODEJS_WORKER_PRIORITY_QUEUE=http://localhost:9324/000000000000/nodejs-worker-normal.fifo
 CROWD_SQS_PYTHON_WORKER_QUEUE=http://localhost:9324/000000000000/python-worker.fifo
 CROWD_SQS_AWS_ACCOUNT_ID=000000000000
 CROWD_SQS_AWS_ACCESS_KEY_ID=x

backend/config/custom-environment-variables.json

@@ -17,7 +17,7 @@
     "host": "CROWD_SQS_HOST",
     "port": "CROWD_SQS_PORT",
     "nodejsWorkerQueue": "CROWD_SQS_NODEJS_WORKER_QUEUE",
-    "nodejsWorkerDelayableQueue": "CROWD_SQS_NODEJS_WORKER_DELAYABLE_QUEUE",
+    "nodejsWorkerPriorityQueue": "CROWD_SQS_NODEJS_WORKER_PRIORITY_QUEUE",
     "integrationRunWorkerQueue": "CROWD_SQS_INTEGRATION_RUN_WORKER_QUEUE",
     "pythonWorkerQueue": "CROWD_SQS_PYTHON_WORKER_QUEUE",
     "aws": {