Update Benchmark set up and persist benchmarks

.github/workflows/benchmarks.yml

@@ -0,0 +1,46 @@
+name: Benchmarks
+
+on:
+  workflow_call:
+    inputs:
+      benchmark_package_path:
+        type: string
+        description: "Path to the directory containing the benchmarking package. Defaults to ."
+        default: "."
+      swift_package_arguments:
+        type: string
+        description: "Arguments to the switch package command invocation e.g. `--disable-sandbox`"
+      linux_5_9_enabled:
+        type: boolean
+        description: "Boolean to enable the Linux 5.9 Swift version matrix job. Defaults to true."
+        default: true
+      linux_5_10_enabled:
+        type: boolean
+        description: "Boolean to enable the Linux 5.10 Swift version matrix job. Defaults to true."
+        default: true
+      linux_6_0_enabled:
+        type: boolean
+        description: "Boolean to enable the Linux 6.0 Swift version matrix job. Defaults to true."
+        default: true
+      linux_nightly_6_0_enabled:
+        type: boolean
+        description: "Boolean to enable the Linux nightly 6.0 Swift version matrix job. Defaults to true."
+        default: true
+      linux_nightly_main_enabled:
+        type: boolean
+        description: "Boolean to enable the Linux nightly main Swift version matrix job. Defaults to true."
+        default: true
+
+jobs:
+  benchmarks:
+    name: Benchmarks
+    # Workaround https://github.com/nektos/act/issues/1875
+    uses: ./.github/workflows/swift_matrix.yml
+    with:
+      name: "Benchmarks"
+      matrix_linux_command: "apt-get update -y -q && apt-get install -y -q libjemalloc-dev && apt-get -y install libsasl2-dev && swift package --package-path ${{ inputs.benchmark_package_path }} ${{ inputs.swift_package_arguments }} benchmark baseline check --check-absolute-path ${{ inputs.benchmark_package_path }}/Thresholds/${SWIFT_VERSION}/"
+      matrix_linux_5_9_enabled: ${{ inputs.linux_5_9_enabled }}
+      matrix_linux_5_10_enabled: ${{ inputs.linux_5_10_enabled }}
+      matrix_linux_6_0_enabled: ${{ inputs.linux_6_0_enabled }}
+      matrix_linux_nightly_6_0_enabled: ${{ inputs.linux_nightly_6_0_enabled }}
+      matrix_linux_nightly_main_enabled: ${{ inputs.linux_nightly_main_enabled }}

.github/workflows/cxx_interop.yml

@@ -0,0 +1,39 @@
+name: Cxx interop
+
+on:
+  workflow_call:
+    inputs:
+      linux_5_9_enabled:
+        type: boolean
+        description: "Boolean to enable the Linux 5.9 Swift version matrix job. Defaults to true."
+        default: true
+      linux_5_10_enabled:
+        type: boolean
+        description: "Boolean to enable the Linux 5.10 Swift version matrix job. Defaults to true."
+        default: true
+      linux_6_0_enabled:
+        type: boolean
+        description: "Boolean to enable the Linux 6.0 Swift version matrix job. Defaults to true."
+        default: true
+      linux_nightly_6_0_enabled:
+        type: boolean
+        description: "Boolean to enable the Linux nightly 6.0 Swift version matrix job. Defaults to true."
+        default: true
+      linux_nightly_main_enabled:
+        type: boolean
+        description: "Boolean to enable the Linux nightly main Swift version matrix job. Defaults to true."
+        default: true
+
+jobs:
+  cxx-interop:
+    name: Cxx interop
+    # Workaround https://github.com/nektos/act/issues/1875
+    uses: ./.github/workflows/swift_matrix.yml
+    with:
+      name: "Cxx interop"
+      matrix_linux_command: "apt-get update -y -q && apt-get install -y -q jq && apt-get -y install libsasl2-dev && curl -s  https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-cxx-interop-compatibility.sh | bash"
+      matrix_linux_5_9_enabled: ${{ inputs.linux_5_9_enabled }}
+      matrix_linux_5_10_enabled: ${{ inputs.linux_5_10_enabled }}
+      matrix_linux_6_0_enabled: ${{ inputs.linux_6_0_enabled }}
+      matrix_linux_nightly_6_0_enabled: ${{ inputs.linux_nightly_6_0_enabled }}
+      matrix_linux_nightly_main_enabled: ${{ inputs.linux_nightly_main_enabled }}

.github/workflows/main.yml

@@ -9,10 +9,23 @@ on:
 jobs:
   unit-tests:
     name: Unit tests
+    # Workaround https://github.com/nektos/act/issues/1875
     uses: ./.github/workflows/unit_tests.yml
     with:
       linux_5_9_arguments_override: "--explicit-target-dependency-import-check error -Xswiftc -strict-concurrency=complete"
       linux_5_10_arguments_override: "--explicit-target-dependency-import-check error -Xswiftc -strict-concurrency=complete"
       linux_6_0_arguments_override: "--explicit-target-dependency-import-check error"
       linux_nightly_6_0_arguments_override: "--explicit-target-dependency-import-check error"
       linux_nightly_main_arguments_override: "--explicit-target-dependency-import-check error"
+
+  benchmarks:
+    name: Benchmarks
+    # Workaround https://github.com/nektos/act/issues/1875
+    uses: ./.github/workflows/benchmarks.yml@main
+    with:
+      benchmark_package_path: "Benchmarks"
+
+  cxx-interop:
+    name: Cxx interop
+    # Workaround https://github.com/nektos/act/issues/1875
+    uses: ./.github/workflows/cxx_interop.yml

.github/workflows/pull_request.yml

@@ -23,17 +23,14 @@ jobs:
       linux_nightly_6_0_arguments_override: "--explicit-target-dependency-import-check error"
       linux_nightly_main_arguments_override: "--explicit-target-dependency-import-check error"
 
+  benchmarks:
+    name: Benchmarks
+    # Workaround https://github.com/nektos/act/issues/1875
+    uses: ./.github/workflows/benchmarks.yml
+    with:
+      benchmark_package_path: "Benchmarks"
+
   cxx-interop:
     name: Cxx interop
-    uses: apple/swift-nio/.github/workflows/swift_matrix.yml@main
-    with:
-      name: "Cxx interop"
-      matrix_linux_command: "apt-get update -y -q && apt-get install -y -q jq && apt-get -y install libsasl2-dev && curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-cxx-interop-compatibility.sh | bash"
-      matrix_linux_5_9_enabled: true
-      matrix_linux_5_10_enabled: true
-      matrix_linux_6_0_enabled: true
-      matrix_linux_nightly_6_0_enabled: true
-      matrix_linux_nightly_main_enabled: true
-      matrix_windows_6_0_enabled: false
-      matrix_windows_nightly_6_0_enabled: false
-      matrix_windows_nightly_main_enabled: false
+    # Workaround https://github.com/nektos/act/issues/1875
+    uses: ./.github/workflows/cxx_interop.yml

.github/workflows/swift_matrix.yml

@@ -0,0 +1,133 @@
+name: Matrix
+
+on:
+  workflow_call:
+    inputs:
+      name:
+        type: string
+        description: "The name of the workflow used for the concurrency group."
+        required: true
+      matrix_linux_command:
+        type: string
+        description: "The command of the current Swift version linux matrix job to execute."
+        required: true
+      matrix_linux_5_9_enabled:
+        type: boolean
+        description: "Boolean to enable the 5.9 Swift version matrix job. Defaults to true."
+        default: true
+      matrix_linux_5_9_container_image:
+        type: string
+        description: "Container image for the 5.9 Swift version matrix job. Defaults to matching Swift Ubuntu image."
+        default: "swift:5.9-jammy"
+      matrix_linux_5_9_command_override:
+        type: string
+        description: "The command of the 5.9 Swift version linux matrix job to execute."
+      matrix_linux_5_10_enabled:
+        type: boolean
+        description: "Boolean to enable the 5.10 Swift version matrix job. Defaults to true."
+        default: true
+      matrix_linux_5_10_container_image:
+        type: string
+        description: "Container image for the 5.10 Swift version matrix job. Defaults to matching Swift Ubuntu image."
+        default: "swift:5.10-jammy"
+      matrix_linux_5_10_command_override:
+        type: string
+        description: "The command of the 5.10 Swift version linux matrix job to execute."
+      matrix_linux_6_0_enabled:
+        type: boolean
+        description: "Boolean to enable the 6.0 Swift version matrix job. Defaults to true."
+        default: true
+      matrix_linux_6_0_container_image:
+        type: string
+        description: "Container image for the 6.0 Swift version matrix job. Defaults to matching Swift Ubuntu image."
+        default: "swift:6.0-jammy"
+      matrix_linux_6_0_command_override:
+        type: string
+        description: "The command of the 6.0 Swift version linux matrix job to execute."
+      matrix_linux_nightly_6_0_enabled:
+        type: boolean
+        description: "Boolean to enable the nightly 6.0 Swift version matrix job. Defaults to true."
+        default: true
+      matrix_linux_nightly_6_0_container_image:
+        type: string
+        description: "Container image for the nightly 6.0 Swift version matrix job. Defaults to matching Swift Ubuntu image."
+        default: "swiftlang/swift:nightly-6.0-jammy"
+      matrix_linux_nightly_6_0_command_override:
+        type: string
+        description: "The command of the nightly 6.0 Swift version linux matrix job to execute."
+      matrix_linux_nightly_main_enabled:
+        type: boolean
+        description: "Boolean to enable the nightly main Swift version matrix job. Defaults to true."
+        default: true
+      matrix_linux_nightly_main_container_image:
+        type: string
+        description: "Container image for the nightly main Swift version matrix job. Defaults to matching Swift Ubuntu image."
+        default: "swiftlang/swift:nightly-main-jammy"
+      matrix_linux_nightly_main_command_override:
+        type: string
+        description: "The command of the nightly main Swift version linux matrix job to execute."
+
+# We are cancelling previously triggered workflow runs
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.name }}
+  cancel-in-progress: true
+
+jobs:
+  linux:
+    name: Linux (${{ matrix.swift.swift_version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # We are specifying only the major and minor of the docker images to automatically pick up the latest patch release
+        swift:
+          - image: ${{ inputs.matrix_linux_5_9_container_image }}
+            swift_version: "5.9"
+            enabled: ${{ inputs.matrix_linux_5_9_enabled }}
+          - image: ${{ inputs.matrix_linux_5_10_container_image }}
+            swift_version: "5.10"
+            enabled: ${{ inputs.matrix_linux_5_10_enabled }}
+          - image: ${{ inputs.matrix_linux_6_0_container_image }}
+            swift_version: "6.0"
+            enabled: ${{ inputs.matrix_linux_6_0_enabled }}
+          - image: ${{ inputs.matrix_linux_nightly_6_0_container_image }}
+            swift_version: "nightly-6.0"
+            enabled: ${{ inputs.matrix_linux_nightly_6_0_enabled }}
+          - image: ${{ inputs.matrix_linux_nightly_main_container_image }}
+            swift_version: "nightly-main"
+            enabled: ${{ inputs.matrix_linux_nightly_main_enabled }}
+    container:
+      image: ${{ matrix.swift.image }}
+    steps:
+      - name: Checkout repository
+        if: ${{ matrix.swift.enabled }}
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          submodules: true
+      - name: Mark the workspace as safe
+        if: ${{ matrix.swift.enabled }}
+        # https://github.com/actions/checkout/issues/766
+        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
+      - name: Run matrix job
+        if: ${{ matrix.swift.enabled }}
+        env:
+          SWIFT_VERSION: ${{ matrix.swift.swift_version }}
+          COMMAND: ${{ inputs.matrix_linux_command }}
+          COMMAND_OVERRIDE_5_9: ${{ inputs.matrix_linux_5_9_command_override }}
+          COMMAND_OVERRIDE_5_10: ${{ inputs.matrix_linux_5_10_command_override }}
+          COMMAND_OVERRIDE_6_0: ${{ inputs.matrix_linux_6_0_command_override }}
+          COMMAND_OVERRIDE_NIGHTLY_6_0: ${{ inputs.matrix_linux_nightly_6_0_command_override }}
+          COMMAND_OVERRIDE_NIGHTLY_MAIN: ${{ inputs.matrix_linux_nightly_main_command_override }}
+        run: |
+          apt-get -qq update && apt-get -qq -y install curl && apt-get -y install libsasl2-dev
+          curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-matrix-job.sh | bash
+    services:
+      zookeeper:
+        image: ubuntu/zookeeper
+      kafka:
+        image: ubuntu/kafka
+        env:
+          ZOOKEEPER_HOST: zookeeper
+    env:
+      KAFKA_HOST: kafka

.github/workflows/unit_tests.yml

@@ -47,59 +47,18 @@ on:
 jobs:
   unit-tests:
     name: Unit tests
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        # We are specifying only the major and minor of the docker images to automatically pick up the latest patch release
-        swift:
-          - image: "swift:5.9-jammy"
-            swift_version: "5.9"
-            enabled: ${{ inputs.linux_5_9_enabled }}
-          - image: "swift:5.10-jammy"
-            swift_version: "5.10"
-            enabled: ${{ inputs.linux_5_10_enabled }}
-          - image: "swift:6.0-jammy"
-            swift_version: "6.0"
-            enabled: ${{ inputs.linux_6_0_enabled }}
-          - image: "swiftlang/swift:nightly-6.0-jammy"
-            swift_version: "nightly-6.0"
-            enabled: ${{ inputs.linux_nightly_6_0_enabled }}
-          - image: "swiftlang/swift:nightly-main-jammy"
-            swift_version: "nightly-main"
-            enabled: ${{ inputs.linux_nightly_main_enabled }}
-    steps:
-      - name: Checkout repository
-        if: ${{ matrix.swift.enabled }}
-        uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-          submodules: true
-      - name: Mark the workspace as safe
-        if: ${{ matrix.swift.enabled }}
-        # https://github.com/actions/checkout/issues/766
-        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
-      - name: Run matrix job
-        if: ${{ matrix.swift.enabled }}
-        env:
-          SWIFT_VERSION: ${{ matrix.swift.swift_version }}
-          COMMAND: "swift test"
-          COMMAND_OVERRIDE_5_9: "swift test ${{ inputs.linux_5_9_arguments_override }}"
-          COMMAND_OVERRIDE_5_10: "swift test ${{ inputs.linux_5_10_arguments_override }}"
-          COMMAND_OVERRIDE_6_0: "swift test ${{ inputs.linux_6_0_arguments_override }}"
-          COMMAND_OVERRIDE_NIGHTLY_6_0: "swift test ${{ inputs.linux_nightly_6_0_arguments_override }}"
-          COMMAND_OVERRIDE_NIGHTLY_MAIN: "swift test ${{ inputs.linux_nightly_main_arguments_override }}"
-        run: |
-          apt-get -qq update && apt-get -qq -y install curl && apt-get -y install libsasl2-dev
-          curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-matrix-job.sh | bash
-    container:
-      image: ${{ matrix.swift.image }}
-    services:
-      zookeeper:
-        image: ubuntu/zookeeper
-      kafka:
-        image: ubuntu/kafka
-        env:
-          ZOOKEEPER_HOST: zookeeper
-    env:
-      KAFKA_HOST: kafka
+    # Workaround https://github.com/nektos/act/issues/1875
+    uses: ./.github/workflows/swift_matrix.yml
+    with:
+      name: "Unit tests"
+      matrix_linux_command: "swift test"
+      matrix_linux_5_9_enabled: ${{ inputs.linux_5_9_enabled }}
+      matrix_linux_5_9_command_override: "swift test ${{ inputs.linux_5_9_arguments_override }}"
+      matrix_linux_5_10_enabled: ${{ inputs.linux_5_10_enabled }}
+      matrix_linux_5_10_command_override: "swift test ${{ inputs.linux_5_10_arguments_override }}"
+      matrix_linux_6_0_enabled: ${{ inputs.linux_6_0_enabled }}
+      matrix_linux_6_0_command_override: "swift test ${{ inputs.linux_6_0_arguments_override }}"
+      matrix_linux_nightly_6_0_enabled: ${{ inputs.linux_nightly_6_0_enabled }}
+      matrix_linux_nightly_6_0_command_override: "swift test ${{ inputs.linux_nightly_6_0_arguments_override }}"
+      matrix_linux_nightly_main_enabled: ${{ inputs.linux_nightly_main_enabled }}
+      matrix_linux_nightly_main_command_override: "swift test ${{ inputs.linux_nightly_main_arguments_override }}"

Benchmarks/Package.swift

@@ -1,4 +1,4 @@
-// swift-tools-version: 5.7
+// swift-tools-version: 5.9
 //===----------------------------------------------------------------------===//
 //
 // This source file is part of the swift-kafka-client open source project

Benchmarks/Thresholds/5.10/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_basic_consumer_messages_1000.p90.json

@@ -0,0 +1,10 @@
+{
+  "allocatedResidentMemory" : 77266944,
+  "cpuTotal" : 200000000,
+  "objectAllocCount" : 5549,
+  "releaseCount" : 15168,
+  "retainCount" : 7108,
+  "retainReleaseDelta" : 2511,
+  "throughput" : 2,
+  "wallClock" : 695307500
+}

Benchmarks/Thresholds/5.10/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_with_offset_commit_messages_1000.p90.json

@@ -0,0 +1,3 @@
+{
+
+}

Benchmarks/Thresholds/5.10/SwiftKafkaConsumerBenchmarks.librdkafka_basic_consumer_messages_1000.p90.json

@@ -0,0 +1,10 @@
+{
+  "allocatedResidentMemory" : 47382528,
+  "cpuTotal" : 10000000,
+  "objectAllocCount" : 16,
+  "releaseCount" : 48,
+  "retainCount" : 2,
+  "retainReleaseDelta" : 30,
+  "throughput" : 2,
+  "wallClock" : 640572501
+}

Benchmarks/Thresholds/5.10/SwiftKafkaConsumerBenchmarks.librdkafka_with_offset_commit_messages_1000.p90.json

@@ -0,0 +1,10 @@
+{
+  "allocatedResidentMemory" : 44204031,
+  "cpuTotal" : 80000000,
+  "objectAllocCount" : 16,
+  "releaseCount" : 48,
+  "retainCount" : 2,
+  "retainReleaseDelta" : 30,
+  "throughput" : 3,
+  "wallClock" : 320339967
+}