Merge branch 'awslabs:main' into main

awslabs · Aug 22, 2024 · b4e1a73 · b4e1a73
2 parents c278061 + 1f779eb
commit b4e1a73
Show file tree

Hide file tree

Showing 12 changed files with 102 additions and 146 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -27,15 +27,15 @@ jobs:
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       - name: Set up JDK ${{ matrix.java }}
-        uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9
+        uses: actions/setup-java@6a0805fcefea3d4657a47ac4c165951e33482018
         with:
           distribution: 'temurin'
           java-version: ${{ matrix.java }}
           cache: 'maven'
       - name: Build JAR Artifacts and verify coverage
         run: mvn --batch-mode --no-transfer-progress --errors --update-snapshots clean package -Drevision=$(git describe --tags --always)
       - name: Upload Artifacts
-        uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029
+        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a
         with:
           name: maven-build-java-${{ matrix.java }}
           path: target/

diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -16,65 +16,87 @@ concurrency:
   group: java-e2e-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+env:
+  AWS_ACCESS_KEY_ID: test
+  AWS_SECRET_ACCESS_KEY: test
+  AWS_DEFAULT_REGION: us-east-1
+
 jobs:
-  e2e:
+  e2e-apache:
     strategy:
       matrix:
         os: [ubuntu-22.04]
         java: ["11", "17"]
-        kafka: ["3.6.2","3.7.0"]
-        broker: ["kafka","redpanda"]
+        kafka: ["3.7.1","3.8.0"]
       fail-fast: false
     runs-on: ${{ matrix.os }}
     timeout-minutes: 15
     env:
-      AWS_ACCESS_KEY_ID: test
-      AWS_SECRET_ACCESS_KEY: test
-      AWS_DEFAULT_REGION: us-east-1
       KAFKA_VERSION: ${{ matrix.kafka }}
       COMPOSE_FILE: "e2e/docker_compose.yaml"
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       - name: Set up JDK ${{ matrix.java }}
-        uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9
+        uses: actions/setup-java@6a0805fcefea3d4657a47ac4c165951e33482018
         with:
           distribution: "temurin"
           java-version: ${{ matrix.java }}
           cache: "maven"
-      - name: Run Kafka Connect ${{ matrix.kafka }} E2E Tests with ${{ matrix.broker }}
+      - name: Run Kafka Connect ${{ matrix.kafka }} E2E Tests with Apache Kafka
         run: |
-          if [[ ${{ matrix.broker }} == "redpanda" ]]; then 
-            export COMPOSE_FILE=e2e/docker_compose_redpanda.yaml
-          fi
           echo "Running integration tests with Docker Compose file: ${COMPOSE_FILE}"
           docker compose -f ${COMPOSE_FILE} pull
           mvn --batch-mode --no-transfer-progress --errors --update-snapshots clean verify -Drevision=$(git describe --tags --always)
-  # special treatment due to different versioning scheme
+
   e2e-confluent:
     strategy:
       matrix:
         os: [ubuntu-22.04]
         java: ["11", "17"]
         kafka: ["7.5.4","7.6.1"]
-        broker: ["confluent"]
       fail-fast: false
     runs-on: ${{ matrix.os }}
     timeout-minutes: 15
     env:
-      AWS_ACCESS_KEY_ID: test
-      AWS_SECRET_ACCESS_KEY: test
-      AWS_DEFAULT_REGION: us-east-1
       COMPOSE_FILE: "e2e/docker_compose_confluent.yaml"
       KAFKA_VERSION: ${{ matrix.kafka }}
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       - name: Set up JDK ${{ matrix.java }}
-        uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9
+        uses: actions/setup-java@6a0805fcefea3d4657a47ac4c165951e33482018
+        with:
+          distribution: "temurin"
+          java-version: ${{ matrix.java }}
+          cache: "maven"
+      - name: Run Confluent Kafka Platform ${{ matrix.kafka }} E2E Tests
+        run: |
+          echo "Running integration tests with Docker Compose file: ${COMPOSE_FILE}"
+          docker compose -f ${COMPOSE_FILE} pull
+          mvn --batch-mode --no-transfer-progress --errors --update-snapshots clean verify -Drevision=$(git describe --tags --always)
+
+  e2e-redpanda:
+    strategy:
+      matrix:
+        os: [ubuntu-22.04]
+        java: [ "11", "17" ]
+        kafka: ["3.7.1","3.8.0"]
+        redpanda: ["v24.2.1"]
+      fail-fast: false
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 15
+    env:
+      KAFKA_VERSION: ${{ matrix.kafka }}
+      REDPANDA_VERSION: ${{ matrix.redpanda }}
+      COMPOSE_FILE: "e2e/docker_compose_redpanda.yaml"
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
+      - name: Set up JDK ${{ matrix.java }}
+        uses: actions/setup-java@6a0805fcefea3d4657a47ac4c165951e33482018
         with:
           distribution: "temurin"
           java-version: ${{ matrix.java }}
           cache: "maven"
-      - name: Run ${{ matrix.broker }} Platform ${{ matrix.kafka }} E2E Tests
+      - name: Run Kafka Connect ${{ matrix.kafka }} E2E Tests with Redpanda ${{ matrix.redpanda }}
         run: |
           echo "Running integration tests with Docker Compose file: ${COMPOSE_FILE}"
           docker compose -f ${COMPOSE_FILE} pull

diff --git a/.github/workflows/format.yaml b/.github/workflows/format.yaml
@@ -23,7 +23,7 @@ jobs:
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       - name: Set up JDK ${{ matrix.java }}
-        uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9
+        uses: actions/setup-java@6a0805fcefea3d4657a47ac4c165951e33482018
         with:
           distribution: 'temurin'
           java-version: ${{ matrix.java }}

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -28,7 +28,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
       - name: Set up JDK ${{ matrix.java }}
-        uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9
+        uses: actions/setup-java@6a0805fcefea3d4657a47ac4c165951e33482018
         with:
           distribution: 'temurin'
           java-version: ${{ matrix.java }}

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -148,11 +148,11 @@ The output should look like:
 
 ```bash
 HTTP/1.1 201 Created
-Date: Fri, 24 Nov 2023 14:11:22 GMT
-Location: http://0.0.0.0:32816/connectors/eventbridge-e2e
+Date: Thu, 29 Feb 2024 19:05:29 GMT
+Location: http://0.0.0.0:32771/connectors/eventbridge-e2e
 Content-Type: application/json
 Content-Length: 688
-Server: Jetty(9.4.52.v20230823)
+Server: Jetty(9.4.53.v20231009)
 
 {"name":"eventbridge-e2e","config":{"auto.offset.reset":"earliest","connector.class":"software.amazon.event.kafkaconnector.EventBridgeSinkConnector","topics":"eventbridge-e2e","aws.eventbridge.connector.id":"eventbridge-e2e-connector","aws.eventbridge.eventbus.arn":"arn:aws:events:us-east-1:000000000000:event-bus/eventbridge-e2e","aws.eventbridge.region":"us-east-1","aws.eventbridge.endpoint.uri":"http://localstack:4566","key.converter":"org.apache.kafka.connect.storage.StringConverter","value.converter":"org.apache.kafka.connect.json.JsonConverter","value.converter.schemas.enable":"false","name":"eventbridge-e2e"},"tasks":[{"connector":"eventbridge-e2e","task":0}],"type":"sink"}
 ```
@@ -169,7 +169,7 @@ Produce a Kafka record to invoke the EventBridge sink connector:
 
 ```bash
 # open a shell in the Kafka broker container
-docker-compose -f docker_compose.yaml exec -w /opt/bitnami/kafka/bin kafka /bin/bash
+docker-compose -f docker_compose.yaml exec -w /opt/kafka/bin kafka /bin/bash
 
 # produce an event
 # replace the topic with your connector settings if needed

diff --git a/e2e/connect-standalone.properties b/e2e/connect-standalone.properties
@@ -37,4 +37,4 @@ offset.flush.interval.ms=10000
 # Note: symlinks will be followed to discover dependencies or plugins.
 # Examples:
 # plugin.path=/usr/local/share/java,/usr/local/share/kafka/plugins,/opt/connectors,
-plugin.path=/usr/local/share/java/connect
+plugin.path=/opt/connectors
diff --git a/e2e/docker_compose.yaml b/e2e/docker_compose.yaml
@@ -2,27 +2,27 @@ version: "3"
 
 services:
   kafka:
-    image: docker.io/bitnami/kafka:${KAFKA_VERSION:-3.7.0}
+    image: docker.io/apache/kafka:${KAFKA_VERSION:-3.8.0}
     ports:
       - "9092:9092"
     environment:
-      ALLOW_PLAINTEXT_LISTENER: 'yes'
-      KAFKA_ENABLE_KRAFT: 'true'
-      KAFKA_CFG_NODE_ID: 1
-      KAFKA_CFG_BROKER_ID: 1
-      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
-      KAFKA_CFG_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092'
-      KAFKA_CFG_PROCESS_ROLES: 'broker,controller'
-      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: '1@kafka:29093'
-      KAFKA_CFG_LISTENERS: 'PLAINTEXT://kafka:29092,CONTROLLER://kafka:29093,PLAINTEXT_HOST://0.0.0.0:9092'
-      KAFKA_CFG_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
-      KAFKA_CFG_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
-      KAFKA_CFG_LOG_DIRS: '/tmp/kraft-combined-logs'
-    volumes:
-      - ${PWD}/log4j.properties:/opt/bitnami/kafka/config/log4j.properties
+      KAFKA_NODE_ID: 1
+      CLUSTER_ID: 1
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
+      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT_HOST://localhost:9092,PLAINTEXT://kafka:29092'
+      KAFKA_PROCESS_ROLES: 'broker,controller'
+      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka:29093'
+      KAFKA_LISTENERS: 'CONTROLLER://kafka:29093,PLAINTEXT_HOST://0.0.0.0:9092,PLAINTEXT://kafka:29092'
+      KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
+      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
 
   connect:
-    image: docker.io/bitnami/kafka:${KAFKA_VERSION:-3.7.0}
+    image: docker.io/apache/kafka:${KAFKA_VERSION:-3.8.0}
     depends_on:
       - kafka
     ports:
@@ -32,12 +32,12 @@ services:
       AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
       AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
       AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
+      KAFKA_LOG4J_OPTS: -Dlog4j.configuration=file:/mnt/connect-log4j.properties
       # KAFKA_OPTS: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005
     volumes:
-      - ${PWD}/connect-standalone.properties:/opt/bitnami/kafka/config/connect-standalone.properties
-      - ${PWD}/connect-log4j.properties:/opt/bitnami/kafka/config/connect-log4j.properties
-      - ${PWD}/../target:/usr/local/share/java/connect/kafka-eventbridge-sink
-    command: ["/bin/bash","-c","env && /opt/bitnami/kafka/bin/connect-standalone.sh /opt/bitnami/kafka/config/connect-standalone.properties"]
+      - ${PWD}:/mnt
+      - ${PWD}/../target:/opt/connectors/kafka-eventbridge-sink
+    command: ["/bin/bash","-c","env && /opt/kafka/bin/connect-standalone.sh /mnt/connect-standalone.properties"]
 
   localstack:
     image: docker.io/localstack/localstack:3.5.0

diff --git a/e2e/docker_compose_redpanda.yaml b/e2e/docker_compose_redpanda.yaml
@@ -3,6 +3,7 @@ version: "3"
 services:
   # is redpanda: reusing names to not break kafka integration tests
   kafka:
+    image: docker.redpanda.com/redpandadata/redpanda:${REDPANDA_VERSION:-v24.2.1}
     command:
       - redpanda
       - start
@@ -14,27 +15,26 @@ services:
       - --memory 1G
       - --mode dev-container
       # - --default-log-level=debug
-    image: docker.redpanda.com/redpandadata/redpanda:v24.1.8
     ports:
       - "9092:9092"
 
   connect:
-    image: docker.io/bitnami/kafka:${KAFKA_VERSION:-3.6.1}
+    image: docker.io/apache/kafka:${KAFKA_VERSION:-3.8.0}
     depends_on:
       - kafka
     ports:
       - "8083"
-      # - "5005:5005
+      # - "5005:5005"
     environment:
       AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
       AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
       AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
+      KAFKA_LOG4J_OPTS: -Dlog4j.configuration=file:/mnt/connect-log4j.properties
       # KAFKA_OPTS: -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005
     volumes:
-      - ${PWD}/connect-standalone.properties:/opt/bitnami/kafka/config/connect-standalone.properties
-      - ${PWD}/connect-log4j.properties:/opt/bitnami/kafka/config/connect-log4j.properties
-      - ${PWD}/../target:/usr/local/share/java/connect/kafka-eventbridge-sink
-    command: ["/bin/bash","-c","env && /opt/bitnami/kafka/bin/connect-standalone.sh /opt/bitnami/kafka/config/connect-standalone.properties"]
+      - ${PWD}:/mnt
+      - ${PWD}/../target:/opt/connectors/kafka-eventbridge-sink
+    command: ["/bin/bash","-c","env && /opt/kafka/bin/connect-standalone.sh /mnt/connect-standalone.properties"]
 
   localstack:
     image: docker.io/localstack/localstack:3.5.0

diff --git a/e2e/log4j.properties b/e2e/log4j.properties