build: update to use turborepo, yarn 3, node 20 and volta (DX-868) (#485

)

### Brief description. What is this change?

Update to yarn3, node 20 and use volta to control the node version locally

### Checklist

- [ ] Breaking changes have been communicated, including:
    - New required environment variables
    - Renaming of interfaces (API routes, request/response interface, etc)
- [ ] New environment variables have [been deployed](https://www.notion.so/voiceflow/Add-Environment-Variables-be1b0136479f45f1adece7995a7adbfb)
- [ ] Appropriate tests have been written
    - Bug fixes are accompanied by an updated or new test
    - New features are accompanied by a new test

Co-authored-by: Ben Teichman <[email protected]>

.circleci/.gitignore

@@ -0,0 +1 @@
+continue_config.yml

.circleci/config.yml

@@ -1,25 +1,63 @@
 version: 2.1
 
+# This exposes the parameters to be overridden in the generated pipeline
+parameters:
+  ssh-fingerprint:
+    type: string
+    default: '93:f8:51:53:5e:bd:75:0e:29:24:4d:6a:3f:ef:e1:4a'
+
 # this allows you to use CircleCI's dynamic configuration feature
 setup: true
 
 orbs:
-  path-filtering: circleci/[email protected]
-
-# Reusable YAML chunks
-defaults:
-  tag_filters: &tag_filters
-    tags:
-      only: /^@voiceflow/.*@[0-9]*(\.[0-9]*)*$/
+  vfcommon: voiceflow/[email protected]
+  gomplate: xavientois/[email protected]
 
 workflows:
   generate-config:
+    when:
+      not:
+        equal: [scheduled_pipeline, << pipeline.trigger_source >>]
     jobs:
-      - path-filtering/filter:
+      - gomplate/render-config:
+          context: dev-test
+          executor: vfcommon/node-executor
+          pre-steps:
+            - checkout
+            - vfcommon/set-env-name: # On bors branches, we will be running the e2e tests in a dev environment
+                target_env_var: &env_name ENV_NAME
+            - vfcommon/set-json-string-from-env:
+                field: e2e_env_name
+                value-env-var: *env_name
+            - vfcommon/get_changed_files
+            - vfcommon/set-service-data-json:
+                service-directories: packages
+            - vfcommon/set-if-file-changed:
+                file-patterns: package.json yarn.lock libs/.* types/.* meta/.*
+                target-env-var: &build_all build_all_services
+            - vfcommon/set-json-boolean-from-env:
+                field: *build_all
+                value-env-var: *build_all
+            - vfcommon/set-pr-number
+            - vfcommon/set-pr-branch:
+                target_env_var: &pr_branch PR_BRANCH
+            - vfcommon/set-json-string-from-env:
+                field: pr_branch
+                value-env-var: *pr_branch
+            - vfcommon/set-json-string:
+                field: branch
+                # HACK: this cannot be empty, so we pass both branch and tag (only one of them will be non-empty)
+                # TODO: Fix this once https://discuss.circleci.com/t/empty-string-for-parameter-breaks-config/47695 has a solution
+                value: '<< pipeline.git.branch >><< pipeline.git.tag >>'
+            - vfcommon/set-json-string:
+                field: ecr_url
+                value: 168387678261.dkr.ecr.us-east-1.amazonaws.com
+            - vfcommon/set-json-string-from-env:
+                field: common_orb_version
+                value-env-var: COMMON_ORB_VERSION
+            - run: cat values.json
+          contexts: values.json
           filters:
-            <<: *tag_filters
-          base-revision: master
-          config-path: .circleci/continue-config.yml
-          mapping: |
-            package.json build-all true
-            yarn.lock build-all true
+            branches:
+              ignore:
+                - /.*\.tmp/

.circleci/continue_config.yml.tmpl

@@ -0,0 +1,123 @@
+version: 2.1
+
+parameters:
+  k8s-namespace:
+    type: string
+    default: 'voiceflow' # This is usually voiceflow
+  ssh-fingerprint:
+    type: string
+    default: '93:f8:51:53:5e:bd:75:0e:29:24:4d:6a:3f:ef:e1:4a'
+
+orbs:
+  vfcommon: voiceflow/common@{{ .values.common_orb_version }}
+  jira: circleci/[email protected]
+
+{{- $borsBranches := coll.Slice "trying" "staging" }}
+{{- $masterProdBranches := coll.Slice "master" "prod" }}
+
+{{/* isMasterProd denotes whether we are on a master/production branch */}}
+{{- $isMasterProd := has $masterProdBranches .values.branch }}
+
+{{/* isBors denotes whether we are on a master/production branch */}}
+{{- $isBors := has $borsBranches .values.branch }}
+
+{{- $buildAll := or $isBors (conv.ToBool .values.build_all_services)  }}
+
+{{/* Only update modified services or all if we are building all services */}}
+{{- $modifiedServices := coll.Slice }}
+{{- range $service := .values.services }}
+  {{- if or $buildAll $service.modified }}
+    {{- $modifiedServices = coll.Append $service $modifiedServices }}
+  {{- end }}
+{{- end }}
+
+{{/* buildAny denotes whether the list of services to build is non-empty */}}
+{{- $buildAny := not (not $modifiedServices) }}
+
+
+# Reusable YAML chunks
+defaults:
+  bors_branches_filters: &bors_branches_filters
+    branches:
+      only:
+        {{- range $borsBranches }}
+        - {{ . }}
+        {{- end }}
+
+  slack-fail-post-step: &slack-fail-post-step
+    post-steps:
+      - vfcommon/notify_slack:
+          channel: dev_general
+          event: fail
+          mentions: '@eng_cxd'
+          template: basic_fail_1
+          branch_pattern: master
+
+workflows:
+  test-and-release:
+    jobs:
+      - vfcommon/install_and_build:
+          <<: *slack-fail-post-step
+          executor: vfcommon/node-large-executor-node-20
+          context: dev-test
+          avoid_post_install_scripts: false
+          attach_workspace: true
+          name: build
+          force_execute: true
+          run_in_container: false
+          package: all
+          package_folder: "{apps,libs}"
+          paths_to_cache: build
+          {{- if .values.pr_branch }}
+          cache_branch: {{ .values.pr_branch | quote }}
+          {{- end }}
+          monorepo_engine: "turborepo"
+          post-steps:
+            - jira/notify
+
+      {{- $testJobs := (coll.Slice "unit_tests" "lint_report" "dependency_tests")}}
+      {{ range $testJob := $testJobs }}
+      - vfcommon/monorepo_{{ $testJob }}:
+          <<: *slack-fail-post-step
+          context: dev-test
+          executor: vfcommon/node-large-executor-node-20
+          name: {{ strings.KebabCase $testJob }}
+          run_on_root: {{ $.values.build_all_services }}
+          requires:
+            - build
+          filters:
+            branches:
+              ignore:
+                - production
+                - /^break-glass.*$/
+        {{- end }}
+
+      {{- if $buildAny }}
+      - vfcommon/sonarcloud_scan:
+          <<: *slack-fail-post-step
+          context: dev-test
+          name: sonarcloud-scan
+          filters:
+            branches:
+              ignore:
+                - production
+                - /^break-glass.*$/
+      {{- end }}
+
+
+      - vfcommon/monorepo_release:
+          <<: *slack-fail-post-step
+          executor: vfcommon/node-executor-node-20
+          sentry_project: creator-app
+          avoid_post_install_scripts: false
+          release_engine: "lite"
+          ssh_key: << pipeline.parameters.ssh-fingerprint >>
+          context: dev-test
+          requires:
+            {{- range $testJob := $testJobs }}
+            - {{ strings.KebabCase $testJob }}
+            {{- end }}
+            - build
+          filters:
+            branches:
+              only: master

.circleci/values.test.json

@@ -0,0 +1,7 @@
+{
+  "e2e_env_name": "e2e-libs-0000000",
+  "services": [],
+  "build_all_services": false,
+  "branch": "trying",
+  "ecr_url": "168387678261.dkr.ecr.voiceflow.amazonaws.com"
+}

.gitignore

@@ -111,3 +111,12 @@ nyc_*
 .idea/
 
 packages/**/yarn.lock
+
+# Yarn
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions