Pin CI container images to v0.4 for 7.17 branch (#16497)

Recently CI images got updated to Java 21 which breaks CI for the 7.17
branch.

This commit pins the CI container images to the older version for fix failures like https://buildkite.com/elastic/logstash-exhaustive-tests-pipeline/builds/760

includes a cherry pick of #16499

.buildkite/pull_request_pipeline.yml

@@ -4,7 +4,7 @@ steps:
   - label: ":passport_control: License check"
     key: "license-check"
     agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci"
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci:0.4"
       cpu: "4"
       memory: "6Gi"
       ephemeralStorage: "100Gi"
@@ -22,7 +22,7 @@ steps:
   - label: ":rspec: Ruby unit tests"
     key: "ruby-unit-tests"
     agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci"
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci:0.4"
       cpu: "4"
       memory: "8Gi"
       ephemeralStorage: "100Gi"
@@ -38,7 +38,7 @@ steps:
   - label: ":java: Java unit tests"
     key: "java-unit-tests"
     agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci"
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci:0.4"
       cpu: "8"
       memory: "16Gi"
       ephemeralStorage: "100Gi"
@@ -62,7 +62,7 @@ steps:
   - label: ":lab_coat: Integration Tests / part 1"
     key: "integration-tests-part-1"
     agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root:0.4"
       cpu: "8"
       memory: "16Gi"
       ephemeralStorage: "100Gi"
@@ -80,7 +80,7 @@ steps:
   - label: ":lab_coat: Integration Tests / part 2"
     key: "integration-tests-part-2"
     agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root:0.4"
       cpu: "8"
       memory: "16Gi"
       ephemeralStorage: "100Gi"
@@ -98,7 +98,7 @@ steps:
   - label: ":lab_coat: IT Persistent Queues / part 1"
     key: "integration-tests-qa-part-1"
     agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root:0.4"
       cpu: "8"
       memory: "16Gi"
       ephemeralStorage: "100Gi"
@@ -117,7 +117,7 @@ steps:
   - label: ":lab_coat: IT Persistent Queues / part 2"
     key: "integration-tests-qa-part-2"
     agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root:0.4"
       cpu: "8"
       memory: "16Gi"
       ephemeralStorage: "100Gi"
@@ -136,7 +136,7 @@ steps:
   - label: ":lab_coat: x-pack unit tests"
     key: "x-pack-unit-tests"
     agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root:0.4"
       cpu: "8"
       memory: "16Gi"
       ephemeralStorage: "100Gi"
@@ -154,7 +154,7 @@ steps:
   - label: ":lab_coat: x-pack integration"
     key: "integration-tests-x-pack"
     agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root"
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci-no-root:0.4"
       cpu: "8"
       memory: "16Gi"
       ephemeralStorage: "100Gi"

.buildkite/scripts/exhaustive-tests/generate-steps.py

@@ -148,11 +148,6 @@ def acceptance_linux_steps() -> list[typing.Any]:
 set -eo pipefail
 source .buildkite/scripts/common/vm-agent-multi-jdk.sh
 source /etc/os-release
-if [[ "$$(echo $$ID_LIKE | tr '[:upper:]' '[:lower:]')" =~ (rhel|fedora) && "$${VERSION_ID%.*}" -le 7 ]]; then
-  # jruby-9.3.10.0 unavailable on centos-7 / oel-7, see https://github.com/jruby/jruby/issues/7579#issuecomment-1425885324 / https://github.com/jruby/jruby/issues/7695
-  # we only need a working jruby to run the acceptance test framework -- the packages have been prebuilt in a previous stage
-  rbenv local jruby-9.4.5.0
-fi
 ci/acceptance_tests.sh"""),
         }
         steps.append(step)