refactor: cleanup and add more logging to waiter

src/jobs/e2e/waiter.yml

@@ -10,58 +10,79 @@ steps:
   - run:
       name: Check if all jobs have completed
       command: |
-        # Global variable to store the status code
-        check_jobs_status=0
         ## Function to check if the API request was successful and process the response
         check_jobs() {
+          date -u
           echo "Fetching jobs from CircleCI API..."
-          response=$(curl --silent --location --request GET "https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID/job" --header "Circle-Token: $CIRCLECI_API_TOKEN")
+          response=$(curl "https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID/job" \
+            --silent \
+            --location \
+            --request GET \
+            --header "Circle-Token: $CIRCLECI_API_TOKEN")
+
           if [[ $? -ne 0 ]]; then
             echo "Error: Failed to fetch jobs from CircleCI API"
-            check_jobs_status=1
-            return
+            return 1
           fi
-          echo "API response received:" >&2
-          echo "$response" >&2
-          if echo "$response" | jq -e . >/dev/null 2>&1; then
-            statuses=$(echo "$response" | jq -r '.items[] | select(.name != "vfcommon/waiter") | .status')
-            if [[ -z "$statuses" ]]; then
-              echo "No job statuses found or unexpected response format"
-              check_jobs_status=1
-              return
-            fi
-            echo "Job statuses extracted:"
-            echo "$statuses"
-          else
+
+          echo "API response received" >&2
+          echo "$response" >responses.log
+
+          if ! jq -e . >/dev/null 2>&1 \<<<"$response" ; then
             echo "Error: Malformed JSON response"
-            check_jobs_status=1
-            return
+            cat responses.log
+            return 1
+          fi
+
+          STATUS_FILTER=$(cat \<<EOF
+          .items
+            | sort_by(.status, .started_at, .name)[]
+            | select(.name != "vfcommon/waiter")
+            | { status, name, started_at }
+        EOF
+          )
+
+          statuses=$(jq -cr "$STATUS_FILTER" \<<<"$response" )
+          if [[ -z "$statuses" ]]; then
+            echo "No job statuses found or unexpected response format"
+            return 1
           fi
+
+          echo "Job statuses extracted:"
+          echo "$statuses"
+
           ## Check if any job is still running
-          if echo "$statuses" | grep -q "running"; then
+          ## We likely need to consider blocked too
+          if grep -q "running" \<<<"$statuses" ; then
             echo "There are still running jobs."
-            check_jobs_status=0  # jobs are still running
+            # jobs are still running
+            return 0
           else
             echo "No jobs are running."
-            check_jobs_status=2  # all jobs are completed successfully
+            # all jobs are completed successfully
+            return 2
           fi
         }
+
         ## The waiter job keeps looping through to check if all running jobs have been completed
         while true; do
           echo "Running check_jobs function..."
+
+          set +e
           check_jobs
-          status=$check_jobs_status
+          status=$?
+          set -e
+
           echo "Status returned from check_jobs: $status"
           if [[ $status -eq 1 ]]; then
             echo "Retrying in << parameters.wait-duration >> seconds due to error..."
-            sleep << parameters.wait-duration >>
-            continue
           elif [[ $status -eq 2 ]]; then
             echo "All jobs completed successfully."
-            exit 0  # success
+            exit 0 # success
           else
             echo "Jobs are still running, checking again in << parameters.wait-duration >> seconds..."
-            sleep << parameters.wait-duration >>
           fi
+          sleep << parameters.wait-duration >>
         done
+
   - run: echo "All required jobs have now completed"