tests/ci: combine acceptance/integration tests (#4210)

tests/ci: combine acceptence/integration tests The separation of running acceptance tests and other bazel-based integration tests is not necessary for any technical reason. Combining these into a single "thing" makes it easier for contributors run all tests locally. The organizational separation persists, the acceptance tests remain in their separate directory. In .bazelrc, combine the test configs `integration_go` and `acceptance_all` to `integration_all` and rename the corresponding make target to `test-integration`. Generate the steps buildkite steps for all bazel integration tests in one go instead of treating acceptance separate from other tests. This adds the previously missing file_transfer demo to CI. Organize buildkite steps into groups and streamline the pipeline config: * Use yaml anchors for recurring snippets. * Split Lint into separate step. * Enable streamed test output for acceptance tests. * Remove unneeded (?) unzipping of bazel testlogs outputs.zip (the content is there anyway).
scionproto · Jun 13, 2022 · b836870 · b836870
1 parent fa7aebc
commit b836870
Show file tree

Hide file tree

Showing 7 changed files with 112 additions and 174 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -23,7 +23,6 @@ test:unit --test_tag_filters=unit
 test:unit_all --config=unit //...
 
 test:integration --test_tag_filters=integration,-lint
-test:integration_go --config=integration -- //... -//acceptance/... -//demo/... -//tools/cryptoplayground/...
-test:acceptance_all --config=integration //acceptance/... //demo/... //tools/cryptoplayground/...
+test:integration_all --config=integration //...
 
 test:lint --test_tag_filters=lint --test_summary=terse --noshow_progress --experimental_convenience_symlinks=ignore //...
diff --git a/.buildkite/pipeline.sh b/.buildkite/pipeline.sh
@@ -1,11 +1,5 @@
 #!/bin/bash
 
-# Following values are useful when debugging the CI.
-# Modify this value to run only a single test in the CI.
-export SINGLE_TEST=
-# Modify this value to run each step multiple times.
-export PARALLELISM=1
-
 set -eou pipefail
 
 # Following values are useful when debugging the CI.
@@ -17,4 +11,4 @@ export PARALLELISM=1
 . ./.buildkite/pipeline_lib.sh
 
 cat .buildkite/pipeline.yml
-gen_bazel_test_steps //acceptance
+gen_bazel_test_steps
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -1,33 +1,35 @@
 env:
   GOPROXY: "http://localhost:3200|https://proxy.golang.org|direct"
 steps:
-  - label: "build scion code :bazel:"
+  - label: "Build :bazel:"
     if: build.message !~ /\[doc\]/
     command:
       - bazel build --verbose_failures --announce_rc //:all
       - bazel run --verbose_failures //docker:prod //docker:test
     key: build
-    retry:
+    retry: &automatic-retry
       automatic:
         - exit_status: -1 # Agent was lost
         - exit_status: 255 # Forced agent shutdown
     timeout_in_minutes: 10
   - wait
-  - label: "UT: unit tests & lint :bazel::bash:"
+  - label: "Unit Tests :bazel:"
     if: build.message !~ /\[doc\]/
     command:
-      - bazel test --config=race --config=unit_all --cache_test_results=${BAZEL_CACHE_TEST_RESULTS:-auto}
-      - make lint
-      - make golangci-lint
+      - bazel test --config=race --config=unit_all
     key: unit_tests
     artifact_paths:
       - "artifacts.out/**/*"
-    retry:
-      automatic:
-        - exit_status: -1 # Agent was lost
-        - exit_status: 255 # Forced agent shutdown
+    retry: *automatic-retry
     timeout_in_minutes: 20
-  - label: "check generated"
+  - label: "Lint :bash:"
+    command:
+      - make lint
+      - make golangci-lint
+    key: lint
+    retry: *automatic-retry
+    timeout_in_minutes: 20
+  - label: "Check Generated :bash:"
     if: build.message !~ /\[doc\]/
     command:
       - echo "--- go_deps.bzl"
@@ -57,118 +59,61 @@ steps:
       - ./tools/update_testdata.sh
     timeout_in_minutes: 20
     key: check_generated
-    retry:
-      automatic:
-        - exit_status: -1 # Agent was lost
-        - exit_status: 255 # Forced agent shutdown
-  - label: "Lint"
-    if: build.message =~ /\[doc\]/
-    command: make lint
-    key: lint
-    timeout_in_minutes: 20
-    retry:
-      automatic:
-        - exit_status: -1 # Agent was lost
-        - exit_status: 255 # Forced agent shutdown
-  - label: "IT: :bazel:"
-    if: build.message !~ /\[doc\]/
-    command:
-      - bazel test --config=integration_go
-    artifact_paths:
-      - "artifacts.out/**/*"
-    timeout_in_minutes: 15
-    key: anapaya_integration_tests
-    retry:
-      automatic:
-        - exit_status: -1 # Agent was lost
-        - exit_status: 255 # Forced agent shutdown
-  - label: "E2E: default :go: :man_in_business_suit_levitating: (scion, ping)"
-    if: build.message !~ /\[doc\]/
-    parallelism: "${SCION_TESTING_FLAKE_PARALLELISM-1}"
-    command:
-      - echo "--- build"
-      - make
-      - echo "--- start topology"
-      - ./scion.sh topology -c topology/default.topo
-      - ./scion.sh run
-      - tools/await-connectivity
-      - ./bin/scion_integration || ( echo "^^^ +++" && false )
-      - ./bin/end2end_integration || ( echo "^^^ +++" && false )
-    plugins:
-      - scionproto/metahook#v0.3.0:
-          post-command: |
-            echo "--- Shutting down SCION topology"
-            ./scion.sh stop
-            echo "SCION topology successfully shut down"
-    artifact_paths:
-      - "artifacts.out/**/*"
-    timeout_in_minutes: 15
-    key: e2e_integration_tests_v2
-    retry:
-      automatic:
-        - exit_status: -1 # Agent was lost
-        - exit_status: 255 # Forced agent shutdown
-  - label: "E2E: failing links :go: :man_in_business_suit_levitating:"
-    if: build.message !~ /\[doc\]/
-    parallelism: "${SCION_TESTING_FLAKE_PARALLELISM-1}"
-    command:
-      - echo "--- build"
-      - make
-      - echo "--- start topology"
-      - ./scion.sh topology -c topology/default-no-peers.topo
-      - ./scion.sh run
-      - tools/await-connectivity
-      - ./bin/end2end_integration || ( echo "^^^ +++" && false )
-      - ./tools/integration/revocation_test.sh
-    plugins:
-      - scionproto/metahook#v0.3.0:
-          post-command: |
-            echo "--- Shutting down SCION topology"
-            ./scion.sh stop
-            echo "SCION topology successfully shut down"
-    artifact_paths:
-      - "artifacts.out/**/*"
-    timeout_in_minutes: 15
-    key: e2e_revocation_test_v2
-    retry:
-      automatic:
-        - exit_status: -1 # Agent was lost
-        - exit_status: 255 # Forced agent shutdown
-  - label: "E2E: default :go: :docker: (ping)"
+    retry: *automatic-retry
+  - group: "End to End"
+    key: e2e
     if: build.message !~ /\[doc\]/
-    parallelism: "${SCION_TESTING_FLAKE_PARALLELISM-1}"
-    command:
-      - echo "--- build"
-      - make build docker-images
-      - echo "--- start topology"
-      - ./scion.sh topology -d
-      - ./scion.sh run
-      - tools/await-connectivity
-      - echo "--- run tests"
-      - ./bin/end2end_integration -d || ( echo "^^^ +++" && false )
-    plugins:
-      - scionproto/metahook#v0.3.0:
-          post-command: |
-            echo "--- Shutting down SCION topology"
-            ./scion.sh stop
-            echo "SCION topology successfully shut down"
-    artifact_paths:
-      - "artifacts.out/**/*"
-    timeout_in_minutes: 15
-    key: docker_integration_e2e_default
-    retry:
-      automatic:
-        - exit_status: -1 # Agent was lost
-        - exit_status: 255 # Forced agent shutdown
-  - label: "TRC ceremony :bazel:"
-    if: build.message !~ /\[doc\]/
-    command:
-      - bazel test //tools/cryptoplayground:trc_ceremony_test //tools/cryptoplayground:trc_ceremony_sensitive_test
-    timeout_in_minutes: 15
-    key: trc_ceremony_test
-    artifact_paths:
-      - "artifacts.out/**/*"
-    retry:
-      automatic:
-        - exit_status: -1 # Agent was lost
-        - exit_status: 255 # Forced agent shutdown
+    steps:
+    - label: "E2E: default :man_in_business_suit_levitating: (scion, ping)"
+      command:
+        - echo "--- build"
+        - make
+        - echo "--- start topology"
+        - ./scion.sh topology -c topology/default.topo
+        - ./scion.sh run
+        - tools/await-connectivity
+        - ./bin/scion_integration || ( echo "^^^ +++" && false )
+        - ./bin/end2end_integration || ( echo "^^^ +++" && false )
+      plugins: &shutdown-scion-post-command
+        - scionproto/metahook#v0.3.0:
+            post-command: |
+              echo "--- Shutting down SCION topology"
+              ./scion.sh stop
+              echo "SCION topology successfully shut down"
+      artifact_paths:
+        - "artifacts.out/**/*"
+      timeout_in_minutes: 15
+      key: e2e_integration_tests_v2
+      retry: *automatic-retry
+    - label: "E2E: failing links :man_in_business_suit_levitating:"
+      command:
+        - echo "--- build"
+        - make
+        - echo "--- start topology"
+        - ./scion.sh topology -c topology/default-no-peers.topo
+        - ./scion.sh run
+        - tools/await-connectivity
+        - ./bin/end2end_integration || ( echo "^^^ +++" && false )
+        - ./tools/integration/revocation_test.sh
+      plugins: *shutdown-scion-post-command
+      artifact_paths:
+        - "artifacts.out/**/*"
+      timeout_in_minutes: 15
+      key: e2e_revocation_test_v2
+      retry: *automatic-retry
+    - label: "E2E: default :docker: (ping)"
+      command:
+        - echo "--- build"
+        - make build docker-images
+        - echo "--- start topology"
+        - ./scion.sh topology -d
+        - ./scion.sh run
+        - tools/await-connectivity
+        - echo "--- run tests"
+        - ./bin/end2end_integration -d || ( echo "^^^ +++" && false )
+      plugins: *shutdown-scion-post-command
+      artifact_paths:
+        - "artifacts.out/**/*"
+      timeout_in_minutes: 15
+      key: docker_integration_e2e_default
+      retry: *automatic-retry
diff --git a/.buildkite/pipeline_lib.sh b/.buildkite/pipeline_lib.sh
@@ -2,23 +2,22 @@
 # args:
 #   -1: the bazel directory in which the tests are.
 gen_bazel_test_steps() {
-    targets="$(bazel query "attr(tags, integration, tests(${1}/...)) except attr(tags, \"lint|manual\", tests(${1}/...))" 2>/dev/null)"
+    parallel="${PARALLELISM:-1}"
+    echo "  - group: \"Integration Tests :bazel:\""
+    echo "    key: integration-tests"
+    echo "    if: build.message !~ /\[doc\]/"
+    echo "    steps:"
+
+    targets="$(bazel query "attr(tags, integration, tests(//...)) except attr(tags, \"lint|manual\", tests(//...))" 2>/dev/null)"
     for test in $targets; do
-        name=${test#"$1/"}
-        skip=false
-        cache="--cache_test_results=\${BAZEL_CACHE_TEST_RESULTS:-auto}"
-        parallel="${PARALLELISM:-1}"
+        name=${test#//}
+        cache=""
         args=""
 
         if [[ "$test" =~ "go" ]]; then
           args="--test_arg=-test.v"
         fi
 
-        ret=$(bazel query "attr(tags, '\\bskip\\b', $test)" 2>/dev/null)
-        if [[ $ret != "" ]]; then
-          continue
-        fi
-
         if [ -n "${SINGLE_TEST}" ]; then
           if [[ ! "${name}" =~ "${SINGLE_TEST}" ]]; then
             continue
@@ -30,29 +29,28 @@ gen_bazel_test_steps() {
             cache="--nocache_test_results"
         fi
 
-        n=${name/test_/}
-        echo "  - label: \"AT: ${n/gateway/gw} :bazel:\""
-        echo "    parallelism: $parallel"
-        echo "    if: build.message !~ /\[doc\]/"
-        echo "    command:"
-        echo "      - bazel test $test $args $cache"
-        if [ "$skip" = true ]; then
-            echo "    skip: true"
+        # Massage the name into a prettier label
+        label=$(echo "$name" | sed -e '
+          s#:test##
+          s#:go_default_test##
+          s#^acceptance/#AT: #
+          s#^demo/\(.*\):.*#Demo: \1#
+          s#\(.*\):go_integration_test$#IT: \1#
+          s#tools/cryptoplayground:\(.*\)_test#AT: \1#
+        ')
+        echo "      - label: \"${label}\""
+        if [ "$parallel" != "1" ]; then
+        echo "        parallelism: $parallel"
         fi
-        echo "    key: \"${name}_acceptance\""
-        echo "    plugins:"
-        echo "      - scionproto/metahook#v0.3.0:"
-        echo "          post-command: |"
-        echo "            echo \"--- Test outputs:\""
-        echo "            cat bazel-testlogs/${1}/${name//://}/test.log"
-        echo "            echo \"--- unzip testlogs\""
-        echo "            unzip bazel-testlogs/${1}/${name//://}/test.outputs/outputs.zip -d outputs 2>//dev//null || true"
-        echo "    artifact_paths:"
-        echo "      - \"artifacts.out/**/*\""
-        echo "    timeout_in_minutes: 20"
-        echo "    retry:"
-        echo "      automatic:"
-        echo "        - exit_status: -1 # Agent was lost"
-        echo "        - exit_status: 255 # Forced agent shutdown"
+        echo "        command:"
+        echo "          - bazel test --test_output=streamed $test $args $cache"
+        echo "        key: \"${name////_}\""
+        echo "        artifact_paths:"
+        echo "          - \"artifacts.out/**/*\""
+        echo "        timeout_in_minutes: 20"
+        echo "        retry:"
+        echo "          automatic:"
+        echo "            - exit_status: -1 # Agent was lost"
+        echo "            - exit_status: 255 # Forced agent shutdown"
     done
 }
diff --git a/Makefile b/Makefile
@@ -26,8 +26,8 @@ bazel:
 test:
 	bazel test --config=unit_all
 
-test-acceptance:
-	bazel test --config=acceptance_all
+test-integration:
+	bazel test --config=integration_all
 
 go_deps.bzl: go.mod
 	bazel run //:gazelle -- update-repos -prune -from_file=go.mod -to_macro=go_deps.bzl%go_deps

diff --git a/acceptance/README.md b/acceptance/README.md
@@ -5,12 +5,13 @@ Each test is defined as a bazel test target, with tags `integration` and `exclus
 
 ## Basic Commands
 
-To run all acceptance tests, execute one of the following (equivalent) commands
+To run all integration tests which include the acceptance tests, execute one of
+the following (equivalent) commands
 
 ```bash
-make test-acceptance                # or,
-bazel test --config=acceptance_all  # or,
-bazel test --config=integration //acceptance/... //demo/...
+make test-integration                # or,
+bazel test --config=integration_all  # or,
+bazel test --config=integration //...
 ```
 
 Run a subset of the tests by specifying a different list of targets:

diff --git a/doc/build/setup.rst b/doc/build/setup.rst
@@ -60,11 +60,12 @@ Bazel & Co.
 
       make
 
-#. Finally, check that unit tests run correctly:
+#. Finally, check that tests run correctly:
 
    .. code-block:: bash
 
       make test
+      make test-integration
 
 #. (Optional) If you already have some code you wish to contribute upstream, you can also run the
    linters locally with: