Add smoke tests (#152)

* Add script to run tests

* Exclude more files from Docker build context

* Run smoke tests in CI

* Use tmp dir

* Add expected analysis files

.dockerignore

@@ -7,3 +7,6 @@ node_modules
 .gitattributes
 .dockerignore
 Dockerfile
+test/
+bin/
+!bin/run.sh

.github/workflows/ci.js.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Run exercism/javascript-analyzer ci precheck (lint code)
         run: yarn lint
 
-  ci:
+  unit_tests:
     runs-on: ubuntu-latest
 
     strategy:
@@ -46,3 +46,27 @@ jobs:
 
       - name: Run end-to-end tests
         run: yarn test:e2e
+
+  smoke_tests:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226
+        with:
+          install: true
+
+      - name: Build Docker image and store in cache
+        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09
+        with:
+          context: .
+          push: false
+          load: true
+          tags: exercism/javascript-analyzer
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Run Tests in Docker
+        run: bin/run-tests-in-docker.sh

.github/workflows/pr.ci.js.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Run exercism/javascript ci precheck (lint code)
         run: yarn lint
 
-  ci:
+  unit_tests:
     runs-on: ubuntu-latest
 
     strategy:
@@ -48,3 +48,27 @@ jobs:
 
       - name: Run end-to-end tests
         run: yarn test:e2e
+
+  smoke_tests:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226
+        with:
+          install: true
+
+      - name: Build Docker image and store in cache
+        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09
+        with:
+          context: .
+          push: false
+          load: true
+          tags: exercism/javascript-analyzer
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Run Tests in Docker
+        run: bin/run-tests-in-docker.sh

bin/run-in-docker.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env sh
+
+# Synopsis:
+# Run the analyzer on a solution using the analyzer Docker image.
+# The analyzer Docker image is built automatically.
+
+# Arguments:
+# $1: exercise slug
+# $2: path to solution folder
+# $3: path to output directory
+
+# Output:
+# Run the analyzer on a solution using the analyzer Docker image.
+# The analyzer Docker image is built automatically.
+
+# Example:
+# ./bin/run-in-docker.sh two-fer path/to/solution/folder/ path/to/output/directory/
+
+# Stop executing when a command returns a non-zero return code
+set -e
+
+# If any required arguments is missing, print the usage and exit
+if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
+    echo "usage: ./bin/run-in-docker.sh exercise-slug path/to/solution/folder/ path/to/output/directory/"
+    exit 1
+fi
+
+slug="$1"
+solution_dir=$(realpath "${2%/}")
+output_dir=$(realpath "${3%/}")
+
+# Create the output directory if it doesn't exist
+mkdir -p "${output_dir}"
+
+# Build the Docker image
+docker build --rm -t exercism/javascript-analyzer .
+
+# Run the Docker image using the settings mimicking the production environment
+docker run \
+    --rm \
+    --network none \
+    --read-only \
+    --mount type=bind,src="${solution_dir}",dst=/solution \
+    --mount type=bind,src="${output_dir}",dst=/output \
+    --mount type=tmpfs,dst=/tmp \
+    exercism/javascript-analyzer "${slug}" /solution /output 

bin/run-tests-in-docker.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env sh
+
+# Synopsis:
+# Test the analyzer Docker image by running it against a predefined set of 
+# solutions with an expected output.
+# The analyzer Docker image is built automatically.
+
+# Output:
+# Outputs the diff of the expected analysis against the actual analysis
+# generated by the analyzer Docker image.
+
+# Example:
+# ./bin/run-tests-in-docker.sh
+
+# Stop executing when a command returns a non-zero return code
+set -e
+
+# Build the Docker image
+docker build --rm -t exercism/javascript-analyzer .
+
+# Run the Docker image using the settings mimicking the production environment
+docker run \
+    --rm \
+    --network none \
+    --read-only \
+    --mount type=bind,src="${PWD}/test",dst=/opt/analyzer/test \
+    --mount type=tmpfs,dst=/tmp \
+    --volume "${PWD}/bin/run-tests.sh:/opt/analyzer/bin/run-tests.sh" \
+    --workdir /opt/analyzer \
+    --entrypoint /opt/analyzer/bin/run-tests.sh \
+    exercism/javascript-analyzer

bin/run-tests.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env sh
+
+# Synopsis:
+# Test the analyzer by running it against a predefined set of solutions 
+# with an expected output.
+
+# Output:
+# Outputs the diff of the expected analysis against the actual analysis
+# generated by the analyzer.
+
+# Example:
+# ./bin/run-tests.sh
+
+exit_code=0
+
+# We need to copy the fixtures to a temp directory as the user
+# running within the Docker container does not have permissions 
+# to run the sed command on the fixtures directory
+fixtures_dir="test/fixtures"
+tmp_fixtures_dir="/tmp/test/fixtures"
+rm -rf "${tmp_fixtures_dir}"
+mkdir -p "${tmp_fixtures_dir}"
+cp -R ${fixtures_dir}/* "${tmp_fixtures_dir}"
+
+# Iterate over all test directories
+for analysis_file in $(find "${tmp_fixtures_dir}" -name expected_analysis.json); do
+    slug=$(echo "${analysis_file:${#tmp_fixtures_dir}+1}" | cut -d / -f 1)
+    test_dir=$(dirname "${analysis_file}")
+    test_dir_name=$(basename "${analysis_file}")
+    test_dir_path=$(realpath "${test_dir}")
+    bin/run.sh "${slug}" "${test_dir_path}" "${test_dir_path}"
+
+    file="analysis.json"
+    expected_file="expected_${file}"
+    echo "${test_dir}: comparing ${file} to ${expected_file}"
+
+    if ! diff "${test_dir}/${file}" "${test_dir}/${expected_file}"; then
+        exit_code=1
+    fi
+done
+
+exit ${exit_code}

test/fixtures/amusement-park/exemplar/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]}

test/fixtures/annalyns-infiltration/exemplar/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]}

test/fixtures/annalyns-infiltration/optimal/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]}

test/fixtures/bird-watcher/exemplar/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]}

test/fixtures/coordinate-transformation/exemplar/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]}

test/fixtures/elyses-analytic-enchantments/exemplar/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]}

test/fixtures/elyses-destructured-enchantments/exemplar/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]}

test/fixtures/elyses-enchantments/exemplar/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]}

test/fixtures/elyses-looping-enchantments/exemplar/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]}

test/fixtures/elyses-transformative-enchantments/exemplar/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]}

test/fixtures/factory-sensors/exemplar/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]}

test/fixtures/freelancer-rates/exemplar/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]}

test/fixtures/high-score-board/exemplar/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]}

test/fixtures/lasagna-master/exemplar/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]}

test/fixtures/lucky-numbers/exemplar/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]}

test/fixtures/mixed-juices/exemplar/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]}

test/fixtures/nullability/exemplar/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]}

test/fixtures/ozans-playlist/exemplar/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]}

test/fixtures/pizza-order/exemplar/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]}

test/fixtures/poetry-club-door-policy/exemplar/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]}

test/fixtures/translation-service/exemplar/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]}

test/fixtures/vehicle-purchase/exemplar/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"javascript.general.exemplar","type":"celebratory"}]}

test/fixtures/vehicle-purchase/uses-if-else/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"javascript.vehicle-purchase.use_if_else","params":{"function":"chooseVehicle"},"type":"informative"},{"comment":"javascript.vehicle-purchase.use_if_else","params":{"function":"calculateResellPrice"},"type":"informative"}]}

test/fixtures/vehicle-purchase/uses-if/expected_analysis.json

@@ -0,0 +1 @@
+{"comments":[{"comment":"javascript.vehicle-purchase.unnecessary_if_statement","type":"actionable"}]}