From add609360196a85e58c5efea6e67a19774449f61 Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Mon, 7 Oct 2024 17:48:29 +0300 Subject: [PATCH 01/32] add airbyte rock --- .licenserc.yaml | 1 + airbyte_rock/local-files/pod-sweeper.sh | 69 +++++++++ airbyte_rock/patches/db-jooq.patch | 31 ++++ airbyte_rock/rockcraft.yaml | 189 ++++++++++++++++++++++++ charmcraft.yaml | 7 - src/charm.py | 15 +- tests/integration/conftest.py | 17 ++- tests/unit/test_charm.py | 2 +- 8 files changed, 306 insertions(+), 25 deletions(-) create mode 100644 airbyte_rock/local-files/pod-sweeper.sh create mode 100644 airbyte_rock/patches/db-jooq.patch create mode 100644 airbyte_rock/rockcraft.yaml diff --git a/.licenserc.yaml b/.licenserc.yaml index 5aa32de..ca9d0ed 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -34,4 +34,5 @@ header: - 'LICENSE' - 'trivy.yaml' - 'lib/**' + - '**/*.patch' comment: on-failure diff --git a/airbyte_rock/local-files/pod-sweeper.sh b/airbyte_rock/local-files/pod-sweeper.sh new file mode 100644 index 0000000..d5e0d0b --- /dev/null +++ b/airbyte_rock/local-files/pod-sweeper.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +# https://github.com/airbytehq/airbyte-platform/blob/main/charts/airbyte-pod-sweeper/templates/configmap.yaml +# TODO(kelkawi-a): Move this to Airbyte ROCK + +get_job_pods() { + # echo "Running kubectl command to get job pods..." + kubectl -n "${JOB_KUBE_NAMESPACE}" -L airbyte -l airbyte=job-pod \ + get pods \ + -o=jsonpath='{range .items[*]} {.metadata.name} {.status.phase} {.status.conditions[0].lastTransitionTime} {.status.startTime}{"\n"}{end}' +} + +delete_pod() { + printf "From status '%s' since '%s', " "$2" "$3" + echo "$1" | grep -v "STATUS" | awk '{print $1}' | xargs --no-run-if-empty kubectl -n "${JOB_KUBE_NAMESPACE}" delete pod +} + +while : +do + echo "Starting pod sweeper cycle:" + + if [ -n "${RUNNING_TTL_MINUTES}" ]; then + # Time window for running pods + RUNNING_DATE_STR=$(date -d "now - ${RUNNING_TTL_MINUTES} minutes" --utc -Ins) + RUNNING_DATE=$(date -d "${RUNNING_DATE_STR}" +%s) + echo "Will sweep running pods from before ${RUNNING_DATE_STR}" + fi + + if [ -n "${SUCCEEDED_TTL_MINUTES}" ]; then + # Shorter time window for succeeded pods + SUCCESS_DATE_STR=$(date -d "now - ${SUCCEEDED_TTL_MINUTES} minutes" --utc -Ins) + SUCCESS_DATE=$(date -d "${SUCCESS_DATE_STR}" +%s) + echo "Will sweep succeeded pods from before ${SUCCESS_DATE_STR}" + fi + + if [ -n "${UNSUCCESSFUL_TTL_MINUTES}" ]; then + # Longer time window for unsuccessful pods (to debug) + NON_SUCCESS_DATE_STR=$(date -d "now - ${UNSUCCESSFUL_TTL_MINUTES} minutes" --utc -Ins) + NON_SUCCESS_DATE=$(date -d "${NON_SUCCESS_DATE_STR}" +%s) + echo "Will sweep unsuccessful pods from before ${NON_SUCCESS_DATE_STR}" + fi + + echo "Running kubectl command to get job pods..." + get_job_pods | while read -r POD; do + IFS=' ' read -r POD_NAME POD_STATUS POD_DATE_STR POD_START_DATE_STR <<< "$POD" + + POD_DATE=$(date -d "${POD_DATE_STR:-$POD_START_DATE_STR}" '+%s') + echo "Evaluating pod: $POD_NAME with status $POD_STATUS since $POD_DATE_STR" + + if [ -n "${RUNNING_TTL_MINUTES}" ] && [ "$POD_STATUS" = "Running" ]; then + if [ "$POD_DATE" -lt "$RUNNING_DATE" ]; then + delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" + fi + elif [ -n "${SUCCEEDED_TTL_MINUTES}" ] && { [[ "$POD_STATUS" = "Succeeded" ]] || [[ "$POD_STATUS" = "Completed" ]]; }; then + if [ "$POD_DATE" -lt "$SUCCESS_DATE" ]; then + delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" + fi + elif [ -n "${UNSUCCESSFUL_TTL_MINUTES}" ] && [ "$POD_STATUS" != "Running" ] && [ "$POD_STATUS" != "Succeeded" ]; then + if [ "$POD_DATE" -lt "$NON_SUCCESS_DATE" ]; then + delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" + fi + fi + done + + echo "Completed pod sweeper cycle. Sleeping for 60 seconds..." + sleep 60 +done diff --git a/airbyte_rock/patches/db-jooq.patch b/airbyte_rock/patches/db-jooq.patch new file mode 100644 index 0000000..2197c0b --- /dev/null +++ b/airbyte_rock/patches/db-jooq.patch @@ -0,0 +1,31 @@ +diff --git a/airbyte-db/jooq/build.gradle.kts b/airbyte-db/jooq/build.gradle.kts +index 6a4b2d6138..d031b1bb3e 100644 +--- a/airbyte-db/jooq/build.gradle.kts ++++ b/airbyte-db/jooq/build.gradle.kts +@@ -15,7 +15,7 @@ dependencies { + + // jOOQ code generation) + implementation(libs.jooq.codegen) +- implementation(libs.platform.testcontainers.postgresql) ++ // implementation(libs.platform.testcontainers.postgresql) + + // These are required because gradle might be using lower version of Jna from other + // library transitive dependency. Can be removed if we can figure out which library is the cause. +@@ -50,7 +50,7 @@ jooq { + + configurations { + create("configsDatabase") { +- generateSchemaSourceOnCompilation = true ++ generateSchemaSourceOnCompilation = false + jooqConfiguration.apply { + generator.apply { + name = "org.jooq.codegen.DefaultGenerator" +@@ -68,7 +68,7 @@ jooq { + } + + create("jobsDatabase") { +- generateSchemaSourceOnCompilation = true ++ generateSchemaSourceOnCompilation = false + jooqConfiguration.apply { + generator.apply { + name = "org.jooq.codegen.DefaultGenerator" diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml new file mode 100644 index 0000000..12859b7 --- /dev/null +++ b/airbyte_rock/rockcraft.yaml @@ -0,0 +1,189 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +name: airbyte +summary: Airbyte rock +description: Airbyte OCI image for the Airbyte charm +version: "1.0" +base: ubuntu@22.04 +build-base: ubuntu@22.04 +license: Apache-2.0 +platforms: + amd64: + +# Please refer to +# https://discourse.ubuntu.com/t/unifying-user-identity-across-snaps-and-rocks/36469 +# for more information about shared user. +# The UID 584792 corresponds to _daemon_ user. +run_user: _daemon_ + +services: + airbyte-server: + override: replace + summary: "airbyte-server service" + startup: enabled + command: "/bin/bash -c airbyte-server/bin/airbyte-server" + airbyte-workers: + override: replace + summary: "airbyte-workers service" + startup: enabled + command: "/bin/bash -c airbyte-workers/bin/airbyte-workers" + airbyte-api-server: + override: replace + summary: "airbyte-api-server service" + startup: enabled + command: "/bin/bash -c airbyte-api-server/bin/airbyte-api-server" + airbyte-bootloader: + override: replace + summary: "airbyte-bootloader service" + startup: enabled + command: "/bin/bash -c airbyte-bootloader/bin/airbyte-bootloader" + airbyte-connector-builder-server: + override: replace + summary: "airbyte-connector-builder-server service" + startup: enabled + command: "/bin/bash -c airbyte-connector-builder-server/bin/airbyte-connector-builder-server" + airbyte-cron: + override: replace + summary: "airbyte-cron service" + startup: enabled + command: "/bin/bash -c airbyte-cron/bin/airbyte-cron" + airbyte-pod-sweeper: + override: replace + summary: "airbyte-pod-sweeper service" + startup: enabled + command: "/bin/bash -c airbyte-pod-sweeper/bin/airbyte-pod-sweeper" + +environment: + JAVA_HOME: /usr/lib/jvm/java-21-openjdk-amd64 + + +parts: + # patches: + # plugin: dump + # source: ./patches + # organize: + # db-jooq.patch: patches/db-jooq.patch + # stage: + # - patches/db-jooq.patch + # prime: + # - "-*" + + # airbyte-platform: + # plugin: dump + # source: https://github.com/airbytehq/airbyte-platform.git # yamllint disable-line + # source-type: git + # source-tag: v0.60.0 + # stage: + # - '*' + + db-lib: + # after: [airbyte-platform] + plugin: dump + source: https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/io/airbyte/airbyte-db/db-lib/0.60.0/db-lib-0.60.0.jar + source-checksum: sha512/5417e491007a80276a51a81c49815da51ecf855eb243d5c48c4ddce75cb330c2930d6dd5100a24d6185e1d1067bf8c176d28d7aadc62d21db078b62327aa77e8 + source-type: file + # organize: + # db-lib-0.60.0.jar: airbyte-db/db-lib/build/libs/db-lib-0.60.0.jar + stage: + - '*' + + jooq: + after: [db-lib] + plugin: dump + source: https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/io/airbyte/airbyte-db/jooq/0.60.0/jooq-0.60.0.jar + source-checksum: sha512/da09c7f4e127af269e6733baa68e3b9ffccd85eadead6d32e441e9b5cb014056720e23039c76914e242ef9ee22d607d20983ffb71b12e1f2748e6630e9bbda86 + source-type: file + # organize: + # jooq-0.60.0.jar: airbyte-db/jooq/build/libs/jooq-0.60.0.jar + stage: + - '*' + + assemble: + after: [jooq] + # plugin: nil + plugin: dump + source: https://github.com/airbytehq/airbyte-platform.git # yamllint disable-line + source-type: git + source-tag: v0.60.0 + build-packages: + - jq + - curl + - coreutils + - bash + - gradle + - openjdk-21-jdk-headless + - npm + - docker.io + build-snaps: + - docker + stage-packages: + - openjdk-21-jdk-headless + # override-build: | + # mkdir -p + # mkdir -p ./airbyte-db/jooq/build/libs && cp ${CRAFT_STAGE}/jooq-0.60.0.jar ./airbyte-db/jooq/build/libs/io.airbyte.airbyte-db-jooq-dev.jar + # mkdir -p ./airbyte-db/db-lib/build/libs && cp ${CRAFT_STAGE}/db-lib-0.60.0.jar ./airbyte-db/db-lib/build/libs/io.airbyte.airbyte-db-db-lib-dev.jar + # # unzip ./airbyte-db/jooq/build/libs/io.airbyte.airbyte-db-jooq-dev.jar && unzip ./airbyte-db/db-lib/build/libs/io.airbyte.airbyte-db-db-lib-dev.jar + # cp -r . ${CRAFT_PART_INSTALL}/app + # stage: + # - app + override-build: | + # git apply ${CRAFT_STAGE}/patches/*.patch + # ./gradlew :airbyte-db:jooq:compileJava + mkdir -p ./airbyte-db/jooq/build/generated/jobsDatabase/src/main/java/ && cp ${CRAFT_STAGE}/jooq-0.60.0.jar ./airbyte-db/jooq/build/generated/jobsDatabase/src/main/java/io.airbyte.airbyte-db-jooq-dev.jar && unzip ./airbyte-db/jooq/build/generated/jobsDatabase/src/main/java/io.airbyte.airbyte-db-jooq-dev.jar -d ./airbyte-db/jooq/build/generated/jobsDatabase/src/main/java/ + ./gradlew generateJobsDatabaseJooq -x dockerBuildImage + ./gradlew generateConfigsDatabaseJooq -x dockerBuildImage + # mkdir -p ./airbyte-db/db-lib/build/libs && cp ${CRAFT_STAGE}/db-lib-0.60.0.jar ./airbyte-db/db-lib/build/libs/io.airbyte.airbyte-db-db-lib-dev.jar + # unzip ./airbyte-db/jooq/build/libs/io.airbyte.airbyte-db-jooq-dev.jar -d ./airbyte-db/jooq/build/libs/ && unzip ./airbyte-db/db-lib/build/libs/io.airbyte.airbyte-db-db-lib-dev.jar -d ./airbyte-db/db-lib/build/libs/ && ./gradlew assemble -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq --stacktrace + ./gradlew assemble -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq --stacktrace + + # ./gradlew :airbyte-server:assemble -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq + # ./gradlew :airbyte-api-server:assemble -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq + # ./gradlew :airbyte-workers:assemble -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq + # ./gradlew :airbyte-bootloader:assemble -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq + # ./gradlew :airbyte-cron:assemble -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq + # ./gradlew :airbyte-connector-builder-server:assemble -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq + + + tar -xvf ./airbyte-server/build/distributions/airbyte-app.tar + tar -xvf ./airbyte-api-server/build/distributions/airbyte-app.tar + tar -xvf ./airbyte-workers/build/distributions/airbyte-app.tar + tar -xvf ./airbyte-bootloader/build/distributions/airbyte-app.tar + tar -xvf ./airbyte-cron/build/distributions/airbyte-app.tar + tar -xvf ./airbyte-connector-builder-server/build/distributions/airbyte-app.tar + + cp -r ./airbyte-server/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-server + cp -r ./airbyte-api-server/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-api-server + cp -r ./airbyte-workers/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-workers + cp -r ./airbyte-bootloader/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-bootloader + cp -r ./airbyte-cron/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-cron + cp -r ./airbyte-connector-builder-server/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-connector-builder-server + stage: + - airbyte-server + - airbyte-api-server + - airbyte-workers + - airbyte-bootloader + - airbyte-cron + - airbyte-connector-builder-server + + local-files: + plugin: dump + source: ./local-files + organize: + pod-sweeper.sh: airbyte-pod-sweeper/bin/airbyte-pod-sweeper.sh + stage: + - airbyte-pod-sweeper/bin/airbyte-pod-sweeper.sh + permissions: + - path: airbyte-pod-sweeper/bin/airbyte-pod-sweeper.sh + owner: 584792 + group: 584792 + mode: "755" + + # ./gradlew distTar -x dockerBuildImage + # ./gradlew distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq + # ./gradlew :airbyte-server:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin + # ./gradlew :airbyte-api-server:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin + # ./gradlew :airbyte-workers:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin + # ./gradlew :airbyte-bootloader:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin + # ./gradlew :airbyte-cron:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin + # ./gradlew :airbyte-connector-builder-server:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin \ No newline at end of file diff --git a/charmcraft.yaml b/charmcraft.yaml index e20d97e..564c6bc 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -448,28 +448,21 @@ resources: airbyte-api-server: type: oci-image description: OCI image for Airbyte API server - upstream-source: airbyte/airbyte-api-server:0.60.0 airbyte-bootloader: type: oci-image description: OCI image for Airbyte Bootloader - upstream-source: airbyte/bootloader:0.60.0 airbyte-connector-builder-server: type: oci-image description: OCI image for Airbyte Connector Builder Server - upstream-source: airbyte/connector-builder-server:0.60.0 airbyte-cron: type: oci-image description: OCI image for Airbyte Cron - upstream-source: airbyte/cron:0.60.0 airbyte-pod-sweeper: type: oci-image description: OCI image for Airbyte Pod Sweeper - upstream-source: bitnami/kubectl:1.29.4 airbyte-server: type: oci-image description: OCI image for Airbyte Server - upstream-source: airbyte/server:0.60.0 airbyte-workers: type: oci-image description: OCI image for Airbyte Worker - upstream-source: airbyte/worker:0.60.0 diff --git a/src/charm.py b/src/charm.py index 80e2412..3f1d565 100755 --- a/src/charm.py +++ b/src/charm.py @@ -5,7 +5,6 @@ """Charm the application.""" import logging -from pathlib import Path import ops from botocore.exceptions import ClientError @@ -53,7 +52,7 @@ def get_pebble_layer(application_name, context): "services": { application_name: { "summary": application_name, - "command": f"/bin/bash -c airbyte-app/bin/{application_name}", + "command": f"/bin/bash -c {application_name}/bin/{application_name}", "startup": "enabled", "override": "replace", # Including config values here so that a change in the @@ -293,18 +292,6 @@ def _update(self, event): event.defer() return - if container_name == "airbyte-pod-sweeper": - script_path = Path(__file__).parent / "scripts/pod-sweeper.sh" - - with open(script_path, "r") as file_source: - logger.info("pushing pod-sweeper script...") - container.push( - f"/airbyte-app/bin/{container_name}", - file_source, - make_dirs=True, - permissions=0o755, - ) - env = create_env(self.model.name, self.app.name, container_name, self.config, self._state) env = {k: v for k, v in env.items() if v is not None} pebble_layer = get_pebble_layer(container_name, env) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index cf09a4d..0a4ca83 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -4,6 +4,7 @@ """Charm integration test config.""" import logging +from pathlib import Path import pytest_asyncio from helpers import ( @@ -16,17 +17,27 @@ perform_temporal_integrations, run_sample_workflow, ) +from pytest import FixtureRequest from pytest_operator.plugin import OpsTest logger = logging.getLogger(__name__) +@pytest_asyncio.fixture(scope="module", name="charm") +async def charm_fixture(request: FixtureRequest, ops_test: OpsTest) -> str | Path: + """Fetch the path to charm.""" + charms = request.config.getoption("--charm-file") + if not charms: + charm = await ops_test.build_charm(".") + assert charm, "Charm not built" + return charm + return charms[0] + + @pytest_asyncio.fixture(name="deploy", scope="module") -async def deploy(ops_test: OpsTest): +async def deploy(ops_test: OpsTest, charm: str): """Test the app is up and running.""" await ops_test.model.set_config({"update-status-hook-interval": "1m"}) - - charm = await ops_test.build_charm(".") resources = get_airbyte_charm_resources() await ops_test.model.deploy(charm, resources=resources, application_name=APP_NAME_AIRBYTE_SERVER, trust=True) diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 9968c3f..058e839 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -348,7 +348,7 @@ def create_plan(container_name, storage_type): "services": { container_name: { "summary": container_name, - "command": f"/bin/bash -c airbyte-app/bin/{container_name}", + "command": f"/bin/bash -c {container_name}/bin/{container_name}", "startup": "enabled", "override": "replace", "environment": { From 14b061df3ccd94aaa38f8c8953cb1de74d9bdf35 Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Wed, 9 Oct 2024 12:45:47 +0300 Subject: [PATCH 02/32] test updated operator workflow with security nesting --- .github/workflows/integration_test.yaml | 3 ++- airbyte_rock/rockcraft.yaml | 13 +++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 638bb97..02fcfa0 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -9,7 +9,7 @@ on: jobs: integration-tests: - uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main + uses: kelkawi-a/operator-workflows/.github/workflows/integration_test.yaml@fc96d4589130d5cfdb96a1116b7284c3780d5fa5 secrets: inherit with: channel: 1.28-strict/stable @@ -18,3 +18,4 @@ jobs: self-hosted-runner: true self-hosted-runner-label: "xlarge" microk8s-addons: "dns ingress rbac storage metallb:10.15.119.2-10.15.119.4 registry" + rockcraft-enable-security-nesting: true diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index 12859b7..8c0dfca 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -117,10 +117,10 @@ parts: - docker.io build-snaps: - docker + - lxd stage-packages: - openjdk-21-jdk-headless # override-build: | - # mkdir -p # mkdir -p ./airbyte-db/jooq/build/libs && cp ${CRAFT_STAGE}/jooq-0.60.0.jar ./airbyte-db/jooq/build/libs/io.airbyte.airbyte-db-jooq-dev.jar # mkdir -p ./airbyte-db/db-lib/build/libs && cp ${CRAFT_STAGE}/db-lib-0.60.0.jar ./airbyte-db/db-lib/build/libs/io.airbyte.airbyte-db-db-lib-dev.jar # # unzip ./airbyte-db/jooq/build/libs/io.airbyte.airbyte-db-jooq-dev.jar && unzip ./airbyte-db/db-lib/build/libs/io.airbyte.airbyte-db-db-lib-dev.jar @@ -130,12 +130,13 @@ parts: override-build: | # git apply ${CRAFT_STAGE}/patches/*.patch # ./gradlew :airbyte-db:jooq:compileJava - mkdir -p ./airbyte-db/jooq/build/generated/jobsDatabase/src/main/java/ && cp ${CRAFT_STAGE}/jooq-0.60.0.jar ./airbyte-db/jooq/build/generated/jobsDatabase/src/main/java/io.airbyte.airbyte-db-jooq-dev.jar && unzip ./airbyte-db/jooq/build/generated/jobsDatabase/src/main/java/io.airbyte.airbyte-db-jooq-dev.jar -d ./airbyte-db/jooq/build/generated/jobsDatabase/src/main/java/ - ./gradlew generateJobsDatabaseJooq -x dockerBuildImage - ./gradlew generateConfigsDatabaseJooq -x dockerBuildImage + # mkdir -p ./airbyte-db/jooq/build/generated/jobsDatabase/src/main/java/ && cp ${CRAFT_STAGE}/jooq-0.60.0.jar ./airbyte-db/jooq/build/generated/jobsDatabase/src/main/java/io.airbyte.airbyte-db-jooq-dev.jar && unzip ./airbyte-db/jooq/build/generated/jobsDatabase/src/main/java/io.airbyte.airbyte-db-jooq-dev.jar -d ./airbyte-db/jooq/build/generated/jobsDatabase/src/main/java/ + # ./gradlew generateJobsDatabaseJooq -x dockerBuildImage + # ./gradlew generateConfigsDatabaseJooq -x dockerBuildImage # mkdir -p ./airbyte-db/db-lib/build/libs && cp ${CRAFT_STAGE}/db-lib-0.60.0.jar ./airbyte-db/db-lib/build/libs/io.airbyte.airbyte-db-db-lib-dev.jar # unzip ./airbyte-db/jooq/build/libs/io.airbyte.airbyte-db-jooq-dev.jar -d ./airbyte-db/jooq/build/libs/ && unzip ./airbyte-db/db-lib/build/libs/io.airbyte.airbyte-db-db-lib-dev.jar -d ./airbyte-db/db-lib/build/libs/ && ./gradlew assemble -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq --stacktrace - ./gradlew assemble -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq --stacktrace + # ./gradlew assemble -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq --stacktrace + ./gradlew assemble -x dockerBuildImage --no-daemon --no-parallel --continue --stacktrace # ./gradlew :airbyte-server:assemble -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq # ./gradlew :airbyte-api-server:assemble -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq @@ -186,4 +187,4 @@ parts: # ./gradlew :airbyte-workers:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin # ./gradlew :airbyte-bootloader:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin # ./gradlew :airbyte-cron:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin - # ./gradlew :airbyte-connector-builder-server:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin \ No newline at end of file + # ./gradlew :airbyte-connector-builder-server:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin From db47c389dbbfe57faff2302cd4b5c1977a208aee Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Wed, 9 Oct 2024 17:32:28 +0300 Subject: [PATCH 03/32] attempt to fix rock build hanging --- airbyte_rock/rockcraft.yaml | 66 ++++++------------------------------- 1 file changed, 10 insertions(+), 56 deletions(-) diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index 8c0dfca..83546a2 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -69,38 +69,8 @@ parts: # prime: # - "-*" - # airbyte-platform: - # plugin: dump - # source: https://github.com/airbytehq/airbyte-platform.git # yamllint disable-line - # source-type: git - # source-tag: v0.60.0 - # stage: - # - '*' - - db-lib: - # after: [airbyte-platform] - plugin: dump - source: https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/io/airbyte/airbyte-db/db-lib/0.60.0/db-lib-0.60.0.jar - source-checksum: sha512/5417e491007a80276a51a81c49815da51ecf855eb243d5c48c4ddce75cb330c2930d6dd5100a24d6185e1d1067bf8c176d28d7aadc62d21db078b62327aa77e8 - source-type: file - # organize: - # db-lib-0.60.0.jar: airbyte-db/db-lib/build/libs/db-lib-0.60.0.jar - stage: - - '*' - - jooq: - after: [db-lib] - plugin: dump - source: https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/io/airbyte/airbyte-db/jooq/0.60.0/jooq-0.60.0.jar - source-checksum: sha512/da09c7f4e127af269e6733baa68e3b9ffccd85eadead6d32e441e9b5cb014056720e23039c76914e242ef9ee22d607d20983ffb71b12e1f2748e6630e9bbda86 - source-type: file - # organize: - # jooq-0.60.0.jar: airbyte-db/jooq/build/libs/jooq-0.60.0.jar - stage: - - '*' - assemble: - after: [jooq] + # after: [jooq] # plugin: nil plugin: dump source: https://github.com/airbytehq/airbyte-platform.git # yamllint disable-line @@ -117,7 +87,6 @@ parts: - docker.io build-snaps: - docker - - lxd stage-packages: - openjdk-21-jdk-headless # override-build: | @@ -128,30 +97,15 @@ parts: # stage: # - app override-build: | - # git apply ${CRAFT_STAGE}/patches/*.patch - # ./gradlew :airbyte-db:jooq:compileJava - # mkdir -p ./airbyte-db/jooq/build/generated/jobsDatabase/src/main/java/ && cp ${CRAFT_STAGE}/jooq-0.60.0.jar ./airbyte-db/jooq/build/generated/jobsDatabase/src/main/java/io.airbyte.airbyte-db-jooq-dev.jar && unzip ./airbyte-db/jooq/build/generated/jobsDatabase/src/main/java/io.airbyte.airbyte-db-jooq-dev.jar -d ./airbyte-db/jooq/build/generated/jobsDatabase/src/main/java/ - # ./gradlew generateJobsDatabaseJooq -x dockerBuildImage - # ./gradlew generateConfigsDatabaseJooq -x dockerBuildImage - # mkdir -p ./airbyte-db/db-lib/build/libs && cp ${CRAFT_STAGE}/db-lib-0.60.0.jar ./airbyte-db/db-lib/build/libs/io.airbyte.airbyte-db-db-lib-dev.jar - # unzip ./airbyte-db/jooq/build/libs/io.airbyte.airbyte-db-jooq-dev.jar -d ./airbyte-db/jooq/build/libs/ && unzip ./airbyte-db/db-lib/build/libs/io.airbyte.airbyte-db-db-lib-dev.jar -d ./airbyte-db/db-lib/build/libs/ && ./gradlew assemble -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq --stacktrace - # ./gradlew assemble -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq --stacktrace - ./gradlew assemble -x dockerBuildImage --no-daemon --no-parallel --continue --stacktrace - - # ./gradlew :airbyte-server:assemble -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq - # ./gradlew :airbyte-api-server:assemble -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq - # ./gradlew :airbyte-workers:assemble -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq - # ./gradlew :airbyte-bootloader:assemble -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq - # ./gradlew :airbyte-cron:assemble -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq - # ./gradlew :airbyte-connector-builder-server:assemble -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq - - - tar -xvf ./airbyte-server/build/distributions/airbyte-app.tar - tar -xvf ./airbyte-api-server/build/distributions/airbyte-app.tar - tar -xvf ./airbyte-workers/build/distributions/airbyte-app.tar - tar -xvf ./airbyte-bootloader/build/distributions/airbyte-app.tar - tar -xvf ./airbyte-cron/build/distributions/airbyte-app.tar - tar -xvf ./airbyte-connector-builder-server/build/distributions/airbyte-app.tar + ./gradlew assemble -x dockerBuildImage --no-daemon --no-parallel --continue --stacktrace --info + # ./gradlew :airbyte-server:assemble -x dockerBuildImage --no-daemon --no-parallel --continue --stacktrace --max-workers 1 + + tar -xvf ./airbyte-server/build/distributions/airbyte-app.tar -C ./airbyte-server/build/distributions/ + tar -xvf ./airbyte-api-server/build/distributions/airbyte-app.tar -C ./airbyte-api-server/build/distributions/ + tar -xvf ./airbyte-workers/build/distributions/airbyte-app.tar -C ./airbyte-workers/build/distributions/ + tar -xvf ./airbyte-bootloader/build/distributions/airbyte-app.tar -C ./airbyte-bootloader/build/distributions/ + tar -xvf ./airbyte-cron/build/distributions/airbyte-app.tar -C ./airbyte-cron/build/distributions/ + tar -xvf ./airbyte-connector-builder-server/build/distributions/airbyte-app.tar -C ./airbyte-connector-builder-server/build/distributions/ cp -r ./airbyte-server/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-server cp -r ./airbyte-api-server/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-api-server From bd0b0ea542bc290256b2a545144670e0809b4d5c Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Wed, 9 Oct 2024 17:52:41 +0300 Subject: [PATCH 04/32] test updated operator workflow with security nesting --- airbyte_rock/patches/db-jooq.patch | 46 +++++++--------------- airbyte_rock/rockcraft.yaml | 63 ++++++++++++++++-------------- 2 files changed, 48 insertions(+), 61 deletions(-) diff --git a/airbyte_rock/patches/db-jooq.patch b/airbyte_rock/patches/db-jooq.patch index 2197c0b..75dcb8d 100644 --- a/airbyte_rock/patches/db-jooq.patch +++ b/airbyte_rock/patches/db-jooq.patch @@ -1,31 +1,15 @@ -diff --git a/airbyte-db/jooq/build.gradle.kts b/airbyte-db/jooq/build.gradle.kts -index 6a4b2d6138..d031b1bb3e 100644 ---- a/airbyte-db/jooq/build.gradle.kts -+++ b/airbyte-db/jooq/build.gradle.kts -@@ -15,7 +15,7 @@ dependencies { - - // jOOQ code generation) - implementation(libs.jooq.codegen) -- implementation(libs.platform.testcontainers.postgresql) -+ // implementation(libs.platform.testcontainers.postgresql) - - // These are required because gradle might be using lower version of Jna from other - // library transitive dependency. Can be removed if we can figure out which library is the cause. -@@ -50,7 +50,7 @@ jooq { - - configurations { - create("configsDatabase") { -- generateSchemaSourceOnCompilation = true -+ generateSchemaSourceOnCompilation = false - jooqConfiguration.apply { - generator.apply { - name = "org.jooq.codegen.DefaultGenerator" -@@ -68,7 +68,7 @@ jooq { - } - - create("jobsDatabase") { -- generateSchemaSourceOnCompilation = true -+ generateSchemaSourceOnCompilation = false - jooqConfiguration.apply { - generator.apply { - name = "org.jooq.codegen.DefaultGenerator" +diff --git a/gradle.properties b/gradle.properties +index 038c2ac2c4..95c25d9dad 100644 +--- a/gradle.properties ++++ b/gradle.properties +@@ -3,8 +3,9 @@ + org.gradle.parallel=true + org.gradle.jvmargs=-Xmx8g -Xss4m + org.gradle.caching=true ++org.gradle.daemon=false + # Note, this might have issues on the normal Github runner. +-org.gradle.vfs.watch=true ++org.gradle.vfs.watch=false + + # Remove once we've migrated to Gradle 9 + # This opts in early to Gradle 9 behavior diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index 83546a2..6db6f03 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -59,18 +59,18 @@ environment: parts: - # patches: - # plugin: dump - # source: ./patches - # organize: - # db-jooq.patch: patches/db-jooq.patch - # stage: - # - patches/db-jooq.patch - # prime: - # - "-*" + patches: + plugin: dump + source: ./patches + organize: + db-jooq.patch: patches/db-jooq.patch + stage: + - patches/db-jooq.patch + prime: + - "-*" assemble: - # after: [jooq] + after: [patches] # plugin: nil plugin: dump source: https://github.com/airbytehq/airbyte-platform.git # yamllint disable-line @@ -97,29 +97,32 @@ parts: # stage: # - app override-build: | - ./gradlew assemble -x dockerBuildImage --no-daemon --no-parallel --continue --stacktrace --info + git apply ${CRAFT_STAGE}/patches/*.patch + ./gradlew :airbyte-api-server:assemble -x dockerBuildImage -Dkotlin.daemon.enabled=false -Dorg.gradle.daemon.idletimeout=60000 --max-workers 1 --info + # ./gradlew :airbyte-api-server:assemble -Dorg.gradle.daemon.idletimeout=60000 -Dorg.gradle.vfs.watch=false -x dockerBuildImage --max-workers 1 + ./gradlew --stop # ./gradlew :airbyte-server:assemble -x dockerBuildImage --no-daemon --no-parallel --continue --stacktrace --max-workers 1 - tar -xvf ./airbyte-server/build/distributions/airbyte-app.tar -C ./airbyte-server/build/distributions/ - tar -xvf ./airbyte-api-server/build/distributions/airbyte-app.tar -C ./airbyte-api-server/build/distributions/ - tar -xvf ./airbyte-workers/build/distributions/airbyte-app.tar -C ./airbyte-workers/build/distributions/ - tar -xvf ./airbyte-bootloader/build/distributions/airbyte-app.tar -C ./airbyte-bootloader/build/distributions/ - tar -xvf ./airbyte-cron/build/distributions/airbyte-app.tar -C ./airbyte-cron/build/distributions/ - tar -xvf ./airbyte-connector-builder-server/build/distributions/airbyte-app.tar -C ./airbyte-connector-builder-server/build/distributions/ + # tar -xvf ./airbyte-server/build/distributions/airbyte-app.tar -C ./airbyte-server/build/distributions/ + # tar -xvf ./airbyte-api-server/build/distributions/airbyte-app.tar -C ./airbyte-api-server/build/distributions/ + # tar -xvf ./airbyte-workers/build/distributions/airbyte-app.tar -C ./airbyte-workers/build/distributions/ + # tar -xvf ./airbyte-bootloader/build/distributions/airbyte-app.tar -C ./airbyte-bootloader/build/distributions/ + # tar -xvf ./airbyte-cron/build/distributions/airbyte-app.tar -C ./airbyte-cron/build/distributions/ + # tar -xvf ./airbyte-connector-builder-server/build/distributions/airbyte-app.tar -C ./airbyte-connector-builder-server/build/distributions/ - cp -r ./airbyte-server/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-server - cp -r ./airbyte-api-server/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-api-server - cp -r ./airbyte-workers/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-workers - cp -r ./airbyte-bootloader/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-bootloader - cp -r ./airbyte-cron/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-cron - cp -r ./airbyte-connector-builder-server/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-connector-builder-server - stage: - - airbyte-server - - airbyte-api-server - - airbyte-workers - - airbyte-bootloader - - airbyte-cron - - airbyte-connector-builder-server + # cp -r ./airbyte-server/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-server + # cp -r ./airbyte-api-server/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-api-server + # cp -r ./airbyte-workers/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-workers + # cp -r ./airbyte-bootloader/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-bootloader + # cp -r ./airbyte-cron/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-cron + # cp -r ./airbyte-connector-builder-server/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-connector-builder-server + # stage: + # - airbyte-server + # - airbyte-api-server + # - airbyte-workers + # - airbyte-bootloader + # - airbyte-cron + # - airbyte-connector-builder-server local-files: plugin: dump From c672e4fe2ffc1ea447fc4b5233cfbefe2d70840a Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Thu, 10 Oct 2024 00:39:08 +0300 Subject: [PATCH 05/32] split parts into assemble and organize --- .github/workflows/integration_test.yaml | 1 + airbyte_rock/rockcraft.yaml | 36 ++++++++++++++++++++----- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 02fcfa0..764b3e0 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -18,4 +18,5 @@ jobs: self-hosted-runner: true self-hosted-runner-label: "xlarge" microk8s-addons: "dns ingress rbac storage metallb:10.15.119.2-10.15.119.4 registry" + trivy-severity-config: CRITICAL rockcraft-enable-security-nesting: true diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index 6db6f03..b503eb3 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -96,13 +96,18 @@ parts: # cp -r . ${CRAFT_PART_INSTALL}/app # stage: # - app - override-build: | - git apply ${CRAFT_STAGE}/patches/*.patch - ./gradlew :airbyte-api-server:assemble -x dockerBuildImage -Dkotlin.daemon.enabled=false -Dorg.gradle.daemon.idletimeout=60000 --max-workers 1 --info - # ./gradlew :airbyte-api-server:assemble -Dorg.gradle.daemon.idletimeout=60000 -Dorg.gradle.vfs.watch=false -x dockerBuildImage --max-workers 1 - ./gradlew --stop - # ./gradlew :airbyte-server:assemble -x dockerBuildImage --no-daemon --no-parallel --continue --stacktrace --max-workers 1 - + # override-build: | + # ./gradlew assemble -x dockerBuildImage --info + # # git apply ${CRAFT_STAGE}/patches/*.patch + # # ./gradlew assemble -x dockerBuildImage -Dkotlin.daemon.enabled=false -Dorg.gradle.daemon.idletimeout=60000 --max-workers 1 --info + # # cp -r . ${CRAFT_PART_INSTALL}/airbyte-platform + # # ./gradlew :airbyte-api-server:assemble -Dorg.gradle.daemon.idletimeout=60000 -Dorg.gradle.vfs.watch=false -x dockerBuildImage --max-workers 1 + # # ./gradlew --stop + # # ./gradlew :airbyte-server:assemble -x dockerBuildImage --no-daemon --no-parallel --continue --stacktrace --max-workers 1 + organize: + "*": airbyte-platform/ + stage: + - airbyte-platform # tar -xvf ./airbyte-server/build/distributions/airbyte-app.tar -C ./airbyte-server/build/distributions/ # tar -xvf ./airbyte-api-server/build/distributions/airbyte-app.tar -C ./airbyte-api-server/build/distributions/ # tar -xvf ./airbyte-workers/build/distributions/airbyte-app.tar -C ./airbyte-workers/build/distributions/ @@ -124,6 +129,23 @@ parts: # - airbyte-cron # - airbyte-connector-builder-server + organize-tars: + plugin: nil + override-build: | + tar -xvf ${CRAFT_PART_INSTALL}/airbyte-platform/airbyte-server/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-server + tar -xvf ${CRAFT_PART_INSTALL}/airbyte-platform/airbyte-api-server/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-api-server + tar -xvf ${CRAFT_PART_INSTALL}/airbyte-platform/airbyte-workers/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-workers + tar -xvf ${CRAFT_PART_INSTALL}/airbyte-platform/airbyte-bootloader/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-bootloader + tar -xvf ${CRAFT_PART_INSTALL}/airbyte-platform/airbyte-cron/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-cron + tar -xvf ${CRAFT_PART_INSTALL}/airbyte-platform/airbyte-connector-builder-server/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-connector-builder-server + stage: + - airbyte-server + - airbyte-api-server + - airbyte-workers + - airbyte-bootloader + - airbyte-cron + - airbyte-connector-builder-server + local-files: plugin: dump source: ./local-files From e068ecc69d4e31b1aa22ab58d53a70bcad53f2fc Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Thu, 10 Oct 2024 01:47:00 +0300 Subject: [PATCH 06/32] split assemble and tar parts --- airbyte_rock/rockcraft.yaml | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index b503eb3..e93bf9b 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -96,14 +96,14 @@ parts: # cp -r . ${CRAFT_PART_INSTALL}/app # stage: # - app - # override-build: | - # ./gradlew assemble -x dockerBuildImage --info - # # git apply ${CRAFT_STAGE}/patches/*.patch - # # ./gradlew assemble -x dockerBuildImage -Dkotlin.daemon.enabled=false -Dorg.gradle.daemon.idletimeout=60000 --max-workers 1 --info - # # cp -r . ${CRAFT_PART_INSTALL}/airbyte-platform - # # ./gradlew :airbyte-api-server:assemble -Dorg.gradle.daemon.idletimeout=60000 -Dorg.gradle.vfs.watch=false -x dockerBuildImage --max-workers 1 - # # ./gradlew --stop - # # ./gradlew :airbyte-server:assemble -x dockerBuildImage --no-daemon --no-parallel --continue --stacktrace --max-workers 1 + override-build: | + ./gradlew assemble -x dockerBuildImage + # git apply ${CRAFT_STAGE}/patches/*.patch + # ./gradlew assemble -x dockerBuildImage -Dkotlin.daemon.enabled=false -Dorg.gradle.daemon.idletimeout=60000 --max-workers 1 --info + # cp -r . ${CRAFT_PART_INSTALL}/airbyte-platform + # ./gradlew :airbyte-api-server:assemble -Dorg.gradle.daemon.idletimeout=60000 -Dorg.gradle.vfs.watch=false -x dockerBuildImage --max-workers 1 + # ./gradlew --stop + # ./gradlew :airbyte-server:assemble -x dockerBuildImage --no-daemon --no-parallel --continue --stacktrace --max-workers 1 organize: "*": airbyte-platform/ stage: @@ -130,14 +130,17 @@ parts: # - airbyte-connector-builder-server organize-tars: + after: [assemble] plugin: nil override-build: | - tar -xvf ${CRAFT_PART_INSTALL}/airbyte-platform/airbyte-server/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-server - tar -xvf ${CRAFT_PART_INSTALL}/airbyte-platform/airbyte-api-server/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-api-server - tar -xvf ${CRAFT_PART_INSTALL}/airbyte-platform/airbyte-workers/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-workers - tar -xvf ${CRAFT_PART_INSTALL}/airbyte-platform/airbyte-bootloader/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-bootloader - tar -xvf ${CRAFT_PART_INSTALL}/airbyte-platform/airbyte-cron/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-cron - tar -xvf ${CRAFT_PART_INSTALL}/airbyte-platform/airbyte-connector-builder-server/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-connector-builder-server + ls ${CRAFT_STAGE}/airbyte-platform + ls ${CRAFT_STAGE}/airbyte-platform/airbyte-server + tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-server/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-server + tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-api-server/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-api-server + tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-workers/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-workers + tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-bootloader/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-bootloader + tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-cron/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-cron + tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-connector-builder-server/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-connector-builder-server stage: - airbyte-server - airbyte-api-server From 69775764145a0e6658102e487dc48bb8a77d6710 Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Thu, 10 Oct 2024 03:44:47 +0300 Subject: [PATCH 07/32] split assemble part --- airbyte_rock/rockcraft.yaml | 72 +++++++++++-------------------------- 1 file changed, 20 insertions(+), 52 deletions(-) diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index e93bf9b..deaf5e8 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -59,23 +59,19 @@ environment: parts: - patches: - plugin: dump - source: ./patches - organize: - db-jooq.patch: patches/db-jooq.patch - stage: - - patches/db-jooq.patch - prime: - - "-*" - - assemble: - after: [patches] - # plugin: nil + pull-repo: plugin: dump source: https://github.com/airbytehq/airbyte-platform.git # yamllint disable-line source-type: git source-tag: v0.60.0 + override-build: | + cp -r . ${CRAFT_PART_INSTALL}/airbyte-platform + stage: + - airbyte-platform + + assemble: + after: [pull-repo] + plugin: nil build-packages: - jq - curl @@ -89,52 +85,24 @@ parts: - docker stage-packages: - openjdk-21-jdk-headless - # override-build: | - # mkdir -p ./airbyte-db/jooq/build/libs && cp ${CRAFT_STAGE}/jooq-0.60.0.jar ./airbyte-db/jooq/build/libs/io.airbyte.airbyte-db-jooq-dev.jar - # mkdir -p ./airbyte-db/db-lib/build/libs && cp ${CRAFT_STAGE}/db-lib-0.60.0.jar ./airbyte-db/db-lib/build/libs/io.airbyte.airbyte-db-db-lib-dev.jar - # # unzip ./airbyte-db/jooq/build/libs/io.airbyte.airbyte-db-jooq-dev.jar && unzip ./airbyte-db/db-lib/build/libs/io.airbyte.airbyte-db-db-lib-dev.jar - # cp -r . ${CRAFT_PART_INSTALL}/app - # stage: - # - app override-build: | - ./gradlew assemble -x dockerBuildImage - # git apply ${CRAFT_STAGE}/patches/*.patch - # ./gradlew assemble -x dockerBuildImage -Dkotlin.daemon.enabled=false -Dorg.gradle.daemon.idletimeout=60000 --max-workers 1 --info - # cp -r . ${CRAFT_PART_INSTALL}/airbyte-platform - # ./gradlew :airbyte-api-server:assemble -Dorg.gradle.daemon.idletimeout=60000 -Dorg.gradle.vfs.watch=false -x dockerBuildImage --max-workers 1 - # ./gradlew --stop - # ./gradlew :airbyte-server:assemble -x dockerBuildImage --no-daemon --no-parallel --continue --stacktrace --max-workers 1 - organize: - "*": airbyte-platform/ - stage: - - airbyte-platform - # tar -xvf ./airbyte-server/build/distributions/airbyte-app.tar -C ./airbyte-server/build/distributions/ - # tar -xvf ./airbyte-api-server/build/distributions/airbyte-app.tar -C ./airbyte-api-server/build/distributions/ - # tar -xvf ./airbyte-workers/build/distributions/airbyte-app.tar -C ./airbyte-workers/build/distributions/ - # tar -xvf ./airbyte-bootloader/build/distributions/airbyte-app.tar -C ./airbyte-bootloader/build/distributions/ - # tar -xvf ./airbyte-cron/build/distributions/airbyte-app.tar -C ./airbyte-cron/build/distributions/ - # tar -xvf ./airbyte-connector-builder-server/build/distributions/airbyte-app.tar -C ./airbyte-connector-builder-server/build/distributions/ - - # cp -r ./airbyte-server/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-server - # cp -r ./airbyte-api-server/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-api-server - # cp -r ./airbyte-workers/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-workers - # cp -r ./airbyte-bootloader/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-bootloader - # cp -r ./airbyte-cron/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-cron - # cp -r ./airbyte-connector-builder-server/build/distributions/airbyte-app/* ${CRAFT_PART_INSTALL}/airbyte-connector-builder-server - # stage: - # - airbyte-server - # - airbyte-api-server - # - airbyte-workers - # - airbyte-bootloader - # - airbyte-cron - # - airbyte-connector-builder-server + cd ${CRAFT_STAGE}/airbyte-platform + ./gradlew assemble -x dockerBuildImage --continue --max-workers 1 + ./gradlew --stop organize-tars: after: [assemble] plugin: nil override-build: | ls ${CRAFT_STAGE}/airbyte-platform - ls ${CRAFT_STAGE}/airbyte-platform/airbyte-server + ls ${CRAFT_STAGE}/airbyte-platform/airbyte-bootloader + mkdir ${CRAFT_PART_INSTALL}/airbyte-server + mkdir ${CRAFT_PART_INSTALL}/airbyte-api-server + mkdir ${CRAFT_PART_INSTALL}/airbyte-workers + mkdir ${CRAFT_PART_INSTALL}/airbyte-bootloader + mkdir ${CRAFT_PART_INSTALL}/airbyte-cron + mkdir ${CRAFT_PART_INSTALL}/airbyte-connector-builder-server + tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-server/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-server tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-api-server/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-api-server tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-workers/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-workers From a85b6cffae1ad1b6954e8876563c3161ef36b10f Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Thu, 10 Oct 2024 10:59:44 +0300 Subject: [PATCH 08/32] test github runner --- .github/workflows/integration_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 764b3e0..6180147 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -15,8 +15,8 @@ jobs: channel: 1.28-strict/stable modules: '["test_charm.py"]' juju-channel: 3.4/stable - self-hosted-runner: true - self-hosted-runner-label: "xlarge" + # self-hosted-runner: true + # self-hosted-runner-label: "xlarge" microk8s-addons: "dns ingress rbac storage metallb:10.15.119.2-10.15.119.4 registry" trivy-severity-config: CRITICAL rockcraft-enable-security-nesting: true From c922b454690b7e02173a39a34d41ff7329b34c10 Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Thu, 10 Oct 2024 13:59:56 +0300 Subject: [PATCH 09/32] update charm code --- .github/workflows/integration_test.yaml | 2 +- airbyte_rock/rockcraft.yaml | 29 +++++-------- charmcraft.yaml | 57 +++++++++++++------------ src/charm.py | 2 +- tests/conftest.py | 2 + tests/integration/conftest.py | 15 +++++-- tests/unit/test_charm.py | 2 +- 7 files changed, 57 insertions(+), 52 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 6180147..dd74a66 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -9,7 +9,7 @@ on: jobs: integration-tests: - uses: kelkawi-a/operator-workflows/.github/workflows/integration_test.yaml@fc96d4589130d5cfdb96a1116b7284c3780d5fa5 + uses: kelkawi-a/operator-workflows/.github/workflows/integration_test.yaml@2119d8c5890abff0db8ba00ffcbaa9549f6fec95 secrets: inherit with: channel: 1.28-strict/stable diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index deaf5e8..d66a580 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -22,37 +22,37 @@ services: override: replace summary: "airbyte-server service" startup: enabled - command: "/bin/bash -c airbyte-server/bin/airbyte-server" + command: "/bin/bash -c airbyte-server/airbyte-app/bin/airbyte-server" airbyte-workers: override: replace summary: "airbyte-workers service" startup: enabled - command: "/bin/bash -c airbyte-workers/bin/airbyte-workers" + command: "/bin/bash -c airbyte-workers/airbyte-app/bin/airbyte-workers" airbyte-api-server: override: replace summary: "airbyte-api-server service" startup: enabled - command: "/bin/bash -c airbyte-api-server/bin/airbyte-api-server" + command: "/bin/bash -c airbyte-api-server/airbyte-app/bin/airbyte-api-server" airbyte-bootloader: override: replace summary: "airbyte-bootloader service" startup: enabled - command: "/bin/bash -c airbyte-bootloader/bin/airbyte-bootloader" + command: "/bin/bash -c airbyte-bootloader/airbyte-app/bin/airbyte-bootloader" airbyte-connector-builder-server: override: replace summary: "airbyte-connector-builder-server service" startup: enabled - command: "/bin/bash -c airbyte-connector-builder-server/bin/airbyte-connector-builder-server" + command: "/bin/bash -c airbyte-connector-builder-server/airbyte-app/bin/airbyte-connector-builder-server" airbyte-cron: override: replace summary: "airbyte-cron service" startup: enabled - command: "/bin/bash -c airbyte-cron/bin/airbyte-cron" + command: "/bin/bash -c airbyte-cron/airbyte-app/bin/airbyte-cron" airbyte-pod-sweeper: override: replace summary: "airbyte-pod-sweeper service" startup: enabled - command: "/bin/bash -c airbyte-pod-sweeper/bin/airbyte-pod-sweeper" + command: "/bin/bash -c airbyte-pod-sweeper/airbyte-app/bin/airbyte-pod-sweeper" environment: JAVA_HOME: /usr/lib/jvm/java-21-openjdk-amd64 @@ -121,20 +121,11 @@ parts: plugin: dump source: ./local-files organize: - pod-sweeper.sh: airbyte-pod-sweeper/bin/airbyte-pod-sweeper.sh + pod-sweeper.sh: airbyte-pod-sweeper/airbyte-app/bin/airbyte-pod-sweeper.sh stage: - - airbyte-pod-sweeper/bin/airbyte-pod-sweeper.sh + - airbyte-pod-sweeper/airbyte-app/bin/airbyte-pod-sweeper.sh permissions: - - path: airbyte-pod-sweeper/bin/airbyte-pod-sweeper.sh + - path: airbyte-pod-sweeper/airbyte-app/bin/airbyte-pod-sweeper.sh owner: 584792 group: 584792 mode: "755" - - # ./gradlew distTar -x dockerBuildImage - # ./gradlew distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq - # ./gradlew :airbyte-server:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin - # ./gradlew :airbyte-api-server:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin - # ./gradlew :airbyte-workers:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin - # ./gradlew :airbyte-bootloader:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin - # ./gradlew :airbyte-cron:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin - # ./gradlew :airbyte-connector-builder-server:distTar -x dockerBuildImage -x :airbyte-db:jooq:generateConfigsDatabaseJooq -x :airbyte-db:jooq:generateJobsDatabaseJooq -x :airbyte-data:kaptGenerateStubsKotlin diff --git a/charmcraft.yaml b/charmcraft.yaml index 564c6bc..5efe5d7 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -429,40 +429,43 @@ config: # Your workload’s containers. containers: airbyte-api-server: - resource: airbyte-api-server + resource: airbyte-image airbyte-bootloader: - resource: airbyte-bootloader + resource: airbyte-image airbyte-connector-builder-server: - resource: airbyte-connector-builder-server + resource: airbyte-image airbyte-cron: - resource: airbyte-cron + resource: airbyte-image airbyte-pod-sweeper: - resource: airbyte-pod-sweeper + resource: airbyte-image airbyte-server: - resource: airbyte-server + resource: airbyte-image airbyte-workers: - resource: airbyte-workers + resource: airbyte-image # This field populates the Resources tab on Charmhub. resources: - airbyte-api-server: - type: oci-image - description: OCI image for Airbyte API server - airbyte-bootloader: - type: oci-image - description: OCI image for Airbyte Bootloader - airbyte-connector-builder-server: - type: oci-image - description: OCI image for Airbyte Connector Builder Server - airbyte-cron: - type: oci-image - description: OCI image for Airbyte Cron - airbyte-pod-sweeper: - type: oci-image - description: OCI image for Airbyte Pod Sweeper - airbyte-server: - type: oci-image - description: OCI image for Airbyte Server - airbyte-workers: + airbyte-image: type: oci-image - description: OCI image for Airbyte Worker + description: OCI image for Airbyte + # airbyte-api-server: + # type: oci-image + # description: OCI image for Airbyte API server + # airbyte-bootloader: + # type: oci-image + # description: OCI image for Airbyte Bootloader + # airbyte-connector-builder-server: + # type: oci-image + # description: OCI image for Airbyte Connector Builder Server + # airbyte-cron: + # type: oci-image + # description: OCI image for Airbyte Cron + # airbyte-pod-sweeper: + # type: oci-image + # description: OCI image for Airbyte Pod Sweeper + # airbyte-server: + # type: oci-image + # description: OCI image for Airbyte Server + # airbyte-workers: + # type: oci-image + # description: OCI image for Airbyte Worker diff --git a/src/charm.py b/src/charm.py index 3f1d565..913525e 100755 --- a/src/charm.py +++ b/src/charm.py @@ -52,7 +52,7 @@ def get_pebble_layer(application_name, context): "services": { application_name: { "summary": application_name, - "command": f"/bin/bash -c {application_name}/bin/{application_name}", + "command": f"/bin/bash -c {application_name}/airbyte-app/bin/{application_name}", "startup": "enabled", "override": "replace", # Including config values here so that a change in the diff --git a/tests/conftest.py b/tests/conftest.py index 04df5c6..97e16d7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,3 +14,5 @@ def pytest_addoption(parser: pytest.Parser): """ # The prebuilt charm file. parser.addoption("--charm-file", action="append", default=[]) + # The charm image name:tag. + parser.addoption("--airbyte-image", action="store", default="") diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 0a4ca83..f36a6d9 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -6,13 +6,13 @@ import logging from pathlib import Path +import pytest import pytest_asyncio from helpers import ( APP_NAME_AIRBYTE_SERVER, APP_NAME_TEMPORAL_ADMIN, APP_NAME_TEMPORAL_SERVER, create_default_namespace, - get_airbyte_charm_resources, perform_airbyte_integrations, perform_temporal_integrations, run_sample_workflow, @@ -23,6 +23,14 @@ logger = logging.getLogger(__name__) +@pytest.fixture(scope="module", name="charm_image") +def charm_image_fixture(request: FixtureRequest) -> str: + """The OCI image for charm.""" + charm_image = request.config.getoption("--airbyte-image") + assert charm_image, "--airbyte-image argument is required which should contain the name of the OCI image." + return charm_image + + @pytest_asyncio.fixture(scope="module", name="charm") async def charm_fixture(request: FixtureRequest, ops_test: OpsTest) -> str | Path: """Fetch the path to charm.""" @@ -35,10 +43,11 @@ async def charm_fixture(request: FixtureRequest, ops_test: OpsTest) -> str | Pat @pytest_asyncio.fixture(name="deploy", scope="module") -async def deploy(ops_test: OpsTest, charm: str): +async def deploy(ops_test: OpsTest, charm: str, charm_image: str): """Test the app is up and running.""" await ops_test.model.set_config({"update-status-hook-interval": "1m"}) - resources = get_airbyte_charm_resources() + # resources = get_airbyte_charm_resources() + resources = {"airbyte-image": charm_image} await ops_test.model.deploy(charm, resources=resources, application_name=APP_NAME_AIRBYTE_SERVER, trust=True) await ops_test.model.deploy( diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 058e839..9c679c1 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -348,7 +348,7 @@ def create_plan(container_name, storage_type): "services": { container_name: { "summary": container_name, - "command": f"/bin/bash -c {container_name}/bin/{container_name}", + "command": f"/bin/bash -c {container_name}/airbyte-app/bin/{container_name}", "startup": "enabled", "override": "replace", "environment": { From 0e42e563b65934066898d77435613134aa41e510 Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Thu, 10 Oct 2024 18:31:07 +0300 Subject: [PATCH 10/32] modify pebble command --- airbyte_rock/rockcraft.yaml | 78 ++++++++++++++++++------------------- src/charm.py | 2 +- tests/unit/test_charm.py | 2 +- 3 files changed, 41 insertions(+), 41 deletions(-) diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index d66a580..d96aa21 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -17,42 +17,42 @@ platforms: # The UID 584792 corresponds to _daemon_ user. run_user: _daemon_ -services: - airbyte-server: - override: replace - summary: "airbyte-server service" - startup: enabled - command: "/bin/bash -c airbyte-server/airbyte-app/bin/airbyte-server" - airbyte-workers: - override: replace - summary: "airbyte-workers service" - startup: enabled - command: "/bin/bash -c airbyte-workers/airbyte-app/bin/airbyte-workers" - airbyte-api-server: - override: replace - summary: "airbyte-api-server service" - startup: enabled - command: "/bin/bash -c airbyte-api-server/airbyte-app/bin/airbyte-api-server" - airbyte-bootloader: - override: replace - summary: "airbyte-bootloader service" - startup: enabled - command: "/bin/bash -c airbyte-bootloader/airbyte-app/bin/airbyte-bootloader" - airbyte-connector-builder-server: - override: replace - summary: "airbyte-connector-builder-server service" - startup: enabled - command: "/bin/bash -c airbyte-connector-builder-server/airbyte-app/bin/airbyte-connector-builder-server" - airbyte-cron: - override: replace - summary: "airbyte-cron service" - startup: enabled - command: "/bin/bash -c airbyte-cron/airbyte-app/bin/airbyte-cron" - airbyte-pod-sweeper: - override: replace - summary: "airbyte-pod-sweeper service" - startup: enabled - command: "/bin/bash -c airbyte-pod-sweeper/airbyte-app/bin/airbyte-pod-sweeper" +# services: +# airbyte-server: +# override: replace +# summary: "airbyte-server service" +# startup: enabled +# command: "/bin/bash -c airbyte-server/airbyte-app/bin/airbyte-server" +# airbyte-workers: +# override: replace +# summary: "airbyte-workers service" +# startup: enabled +# command: "/bin/bash -c airbyte-workers/airbyte-app/bin/airbyte-workers" +# airbyte-api-server: +# override: replace +# summary: "airbyte-api-server service" +# startup: enabled +# command: "/bin/bash -c airbyte-api-server/airbyte-app/bin/airbyte-api-server" +# airbyte-bootloader: +# override: replace +# summary: "airbyte-bootloader service" +# startup: enabled +# command: "/bin/bash -c airbyte-bootloader/airbyte-app/bin/airbyte-bootloader" +# airbyte-connector-builder-server: +# override: replace +# summary: "airbyte-connector-builder-server service" +# startup: enabled +# command: "/bin/bash -c airbyte-connector-builder-server/airbyte-app/bin/airbyte-connector-builder-server" +# airbyte-cron: +# override: replace +# summary: "airbyte-cron service" +# startup: enabled +# command: "/bin/bash -c airbyte-cron/airbyte-app/bin/airbyte-cron" +# airbyte-pod-sweeper: +# override: replace +# summary: "airbyte-pod-sweeper service" +# startup: enabled +# command: "/bin/bash -c airbyte-pod-sweeper/airbyte-app/bin/airbyte-pod-sweeper" environment: JAVA_HOME: /usr/lib/jvm/java-21-openjdk-amd64 @@ -121,11 +121,11 @@ parts: plugin: dump source: ./local-files organize: - pod-sweeper.sh: airbyte-pod-sweeper/airbyte-app/bin/airbyte-pod-sweeper.sh + pod-sweeper.sh: airbyte-pod-sweeper/airbyte-app/bin/airbyte-pod-sweeper stage: - - airbyte-pod-sweeper/airbyte-app/bin/airbyte-pod-sweeper.sh + - airbyte-pod-sweeper/airbyte-app/bin/airbyte-pod-sweeper permissions: - - path: airbyte-pod-sweeper/airbyte-app/bin/airbyte-pod-sweeper.sh + - path: airbyte-pod-sweeper/airbyte-app/bin/airbyte-pod-sweeper owner: 584792 group: 584792 mode: "755" diff --git a/src/charm.py b/src/charm.py index 913525e..5fa79ec 100755 --- a/src/charm.py +++ b/src/charm.py @@ -52,7 +52,7 @@ def get_pebble_layer(application_name, context): "services": { application_name: { "summary": application_name, - "command": f"/bin/bash -c {application_name}/airbyte-app/bin/{application_name}", + "command": f"/bin/bash {application_name}/airbyte-app/bin/{application_name}", "startup": "enabled", "override": "replace", # Including config values here so that a change in the diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 9c679c1..22fecae 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -348,7 +348,7 @@ def create_plan(container_name, storage_type): "services": { container_name: { "summary": container_name, - "command": f"/bin/bash -c {container_name}/airbyte-app/bin/{container_name}", + "command": f"/bin/bash {container_name}/airbyte-app/bin/{container_name}", "startup": "enabled", "override": "replace", "environment": { From aedadae38c318ca4ec118889735256511df03206 Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Thu, 10 Oct 2024 20:13:15 +0300 Subject: [PATCH 11/32] enable self-hosted runners --- .github/workflows/integration_test.yaml | 6 +- airbyte_rock/rockcraft.yaml | 74 +++++++++++++------------ 2 files changed, 41 insertions(+), 39 deletions(-) diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index dd74a66..0981bdf 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -9,14 +9,14 @@ on: jobs: integration-tests: - uses: kelkawi-a/operator-workflows/.github/workflows/integration_test.yaml@2119d8c5890abff0db8ba00ffcbaa9549f6fec95 + uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main secrets: inherit with: channel: 1.28-strict/stable modules: '["test_charm.py"]' juju-channel: 3.4/stable - # self-hosted-runner: true - # self-hosted-runner-label: "xlarge" + self-hosted-runner: true + self-hosted-runner-label: "xlarge" microk8s-addons: "dns ingress rbac storage metallb:10.15.119.2-10.15.119.4 registry" trivy-severity-config: CRITICAL rockcraft-enable-security-nesting: true diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index d96aa21..a024a66 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -17,42 +17,42 @@ platforms: # The UID 584792 corresponds to _daemon_ user. run_user: _daemon_ -# services: -# airbyte-server: -# override: replace -# summary: "airbyte-server service" -# startup: enabled -# command: "/bin/bash -c airbyte-server/airbyte-app/bin/airbyte-server" -# airbyte-workers: -# override: replace -# summary: "airbyte-workers service" -# startup: enabled -# command: "/bin/bash -c airbyte-workers/airbyte-app/bin/airbyte-workers" -# airbyte-api-server: -# override: replace -# summary: "airbyte-api-server service" -# startup: enabled -# command: "/bin/bash -c airbyte-api-server/airbyte-app/bin/airbyte-api-server" -# airbyte-bootloader: -# override: replace -# summary: "airbyte-bootloader service" -# startup: enabled -# command: "/bin/bash -c airbyte-bootloader/airbyte-app/bin/airbyte-bootloader" -# airbyte-connector-builder-server: -# override: replace -# summary: "airbyte-connector-builder-server service" -# startup: enabled -# command: "/bin/bash -c airbyte-connector-builder-server/airbyte-app/bin/airbyte-connector-builder-server" -# airbyte-cron: -# override: replace -# summary: "airbyte-cron service" -# startup: enabled -# command: "/bin/bash -c airbyte-cron/airbyte-app/bin/airbyte-cron" -# airbyte-pod-sweeper: -# override: replace -# summary: "airbyte-pod-sweeper service" -# startup: enabled -# command: "/bin/bash -c airbyte-pod-sweeper/airbyte-app/bin/airbyte-pod-sweeper" +services: + airbyte-server: + override: replace + summary: "airbyte-server service" + startup: enabled + command: "/bin/bash airbyte-server/airbyte-app/bin/airbyte-server" + airbyte-workers: + override: replace + summary: "airbyte-workers service" + startup: enabled + command: "/bin/bash airbyte-workers/airbyte-app/bin/airbyte-workers" + airbyte-api-server: + override: replace + summary: "airbyte-api-server service" + startup: enabled + command: "/bin/bash airbyte-api-server/airbyte-app/bin/airbyte-api-server" + airbyte-bootloader: + override: replace + summary: "airbyte-bootloader service" + startup: enabled + command: "/bin/bash airbyte-bootloader/airbyte-app/bin/airbyte-bootloader" + airbyte-connector-builder-server: + override: replace + summary: "airbyte-connector-builder-server service" + startup: enabled + command: "/bin/bash airbyte-connector-builder-server/airbyte-app/bin/airbyte-connector-builder-server" + airbyte-cron: + override: replace + summary: "airbyte-cron service" + startup: enabled + command: "/bin/bash airbyte-cron/airbyte-app/bin/airbyte-cron" + airbyte-pod-sweeper: + override: replace + summary: "airbyte-pod-sweeper service" + startup: enabled + command: "/bin/bash airbyte-pod-sweeper/airbyte-app/bin/airbyte-pod-sweeper" environment: JAVA_HOME: /usr/lib/jvm/java-21-openjdk-amd64 @@ -83,6 +83,8 @@ parts: - docker.io build-snaps: - docker + stage-snaps: + - kubectl stage-packages: - openjdk-21-jdk-headless override-build: | From a33f43f7bb9ea04a9eb0b67dd6c13168f2c6e381 Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Thu, 10 Oct 2024 21:15:41 +0300 Subject: [PATCH 12/32] remove services from rock file --- airbyte_rock/rockcraft.yaml | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index a024a66..109289a 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -17,43 +17,6 @@ platforms: # The UID 584792 corresponds to _daemon_ user. run_user: _daemon_ -services: - airbyte-server: - override: replace - summary: "airbyte-server service" - startup: enabled - command: "/bin/bash airbyte-server/airbyte-app/bin/airbyte-server" - airbyte-workers: - override: replace - summary: "airbyte-workers service" - startup: enabled - command: "/bin/bash airbyte-workers/airbyte-app/bin/airbyte-workers" - airbyte-api-server: - override: replace - summary: "airbyte-api-server service" - startup: enabled - command: "/bin/bash airbyte-api-server/airbyte-app/bin/airbyte-api-server" - airbyte-bootloader: - override: replace - summary: "airbyte-bootloader service" - startup: enabled - command: "/bin/bash airbyte-bootloader/airbyte-app/bin/airbyte-bootloader" - airbyte-connector-builder-server: - override: replace - summary: "airbyte-connector-builder-server service" - startup: enabled - command: "/bin/bash airbyte-connector-builder-server/airbyte-app/bin/airbyte-connector-builder-server" - airbyte-cron: - override: replace - summary: "airbyte-cron service" - startup: enabled - command: "/bin/bash airbyte-cron/airbyte-app/bin/airbyte-cron" - airbyte-pod-sweeper: - override: replace - summary: "airbyte-pod-sweeper service" - startup: enabled - command: "/bin/bash airbyte-pod-sweeper/airbyte-app/bin/airbyte-pod-sweeper" - environment: JAVA_HOME: /usr/lib/jvm/java-21-openjdk-amd64 From 8e2c4a8a97c81236b73fec0436b296d99db7fe2f Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Thu, 10 Oct 2024 22:25:44 +0300 Subject: [PATCH 13/32] install airbyte cdk library --- airbyte_rock/rockcraft.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index 109289a..f8aff7e 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -19,6 +19,9 @@ run_user: _daemon_ environment: JAVA_HOME: /usr/lib/jvm/java-21-openjdk-amd64 + CDK_PYTHON: /usr/bin/python3.10 + CDK_ENTRYPOINT: /usr/lib/python3.10/dist-packages/airbyte_cdk/connector_builder/main.py + CDK_VERSION: "5.12.0" parts: @@ -27,10 +30,22 @@ parts: source: https://github.com/airbytehq/airbyte-platform.git # yamllint disable-line source-type: git source-tag: v0.60.0 + stage-packages: + - python3.10-venv override-build: | cp -r . ${CRAFT_PART_INSTALL}/airbyte-platform + pip install --upgrade setuptools pip airbyte-cdk==5.13.0 \ + --target=/${CRAFT_PART_INSTALL}/dist + organize: + dist: usr/local/lib/python3.10/dist-packages stage: - airbyte-platform + - usr/local/lib/python3.10/dist-packages + permissions: + - path: usr/local/lib/python3.10/dist-packages + owner: 584792 + group: 584792 + mode: "755" assemble: after: [pull-repo] From 96d9187488f2e5b859f2b3c708fbfa1809d25bdb Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Thu, 10 Oct 2024 23:25:16 +0300 Subject: [PATCH 14/32] add kubectl part --- airbyte_rock/rockcraft.yaml | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index f8aff7e..a4f9aeb 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -25,6 +25,23 @@ environment: parts: + install-kubectl: + plugin: nil + override-build: | + # Install kubectl + curl -LO https://dl.k8s.io/release/v1.31.0/bin/linux/amd64/kubectl + curl -LO "https://dl.k8s.io/release/v1.31.0/bin/linux/amd64/kubectl.sha256" + echo "$(cat kubectl.sha256) kubectl" | sha256sum --check + mkdir -p ${CRAFT_PART_INSTALL}/usr/local/bin/kubectl + install -o root -g root -m 0755 kubectl ${CRAFT_PART_INSTALL}/usr/local/bin/kubectl + stage: + - usr/local/bin/kubectl + permissions: + - path: usr/local/bin/kubectl + owner: 584792 + group: 584792 + mode: "755" + pull-repo: plugin: dump source: https://github.com/airbytehq/airbyte-platform.git # yamllint disable-line @@ -32,9 +49,9 @@ parts: source-tag: v0.60.0 stage-packages: - python3.10-venv - override-build: | + override-build: | cp -r . ${CRAFT_PART_INSTALL}/airbyte-platform - pip install --upgrade setuptools pip airbyte-cdk==5.13.0 \ + pip install --upgrade setuptools pip airbyte-cdk==5.12.0 \ --target=/${CRAFT_PART_INSTALL}/dist organize: dist: usr/local/lib/python3.10/dist-packages @@ -61,8 +78,6 @@ parts: - docker.io build-snaps: - docker - stage-snaps: - - kubectl stage-packages: - openjdk-21-jdk-headless override-build: | From 70bc4e807e994eca185a2e47f53db1127bbdc870 Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Fri, 11 Oct 2024 00:29:40 +0300 Subject: [PATCH 15/32] install kubectl with apt --- airbyte_rock/rockcraft.yaml | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index a4f9aeb..190841e 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -27,13 +27,22 @@ environment: parts: install-kubectl: plugin: nil - override-build: | - # Install kubectl - curl -LO https://dl.k8s.io/release/v1.31.0/bin/linux/amd64/kubectl - curl -LO "https://dl.k8s.io/release/v1.31.0/bin/linux/amd64/kubectl.sha256" - echo "$(cat kubectl.sha256) kubectl" | sha256sum --check + stage-packages: + - apt-transport-https + - ca-certificates + - curl + - gnupg + override-build: | + echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.31/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list + chmod 644 /etc/apt/sources.list.d/kubernetes.list # helps tools such as command-not-found to work correctly + + curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.31/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + chmod 644 /etc/apt/keyrings/kubernetes-apt-keyring.gpg # allow unprivileged APT programs to read this keyring + + apt-get update + apt-get install -y kubectl mkdir -p ${CRAFT_PART_INSTALL}/usr/local/bin/kubectl - install -o root -g root -m 0755 kubectl ${CRAFT_PART_INSTALL}/usr/local/bin/kubectl + cp -r $(which kubectl) ${CRAFT_PART_INSTALL}/usr/local/bin/kubectl stage: - usr/local/bin/kubectl permissions: @@ -43,6 +52,7 @@ parts: mode: "755" pull-repo: + after: [install-kubectl] plugin: dump source: https://github.com/airbytehq/airbyte-platform.git # yamllint disable-line source-type: git @@ -113,6 +123,7 @@ parts: - airbyte-connector-builder-server local-files: + after: [organize-tars] plugin: dump source: ./local-files organize: From ccab18717b731d1de835939ff371bc75465703cf Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Fri, 11 Oct 2024 02:31:08 +0300 Subject: [PATCH 16/32] install kubectl with snap --- airbyte_rock/rockcraft.yaml | 45 +++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index 190841e..199218d 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -27,29 +27,40 @@ environment: parts: install-kubectl: plugin: nil - stage-packages: - - apt-transport-https - - ca-certificates - - curl - - gnupg override-build: | - echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.31/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list - chmod 644 /etc/apt/sources.list.d/kubernetes.list # helps tools such as command-not-found to work correctly - - curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.31/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg - chmod 644 /etc/apt/keyrings/kubernetes-apt-keyring.gpg # allow unprivileged APT programs to read this keyring - - apt-get update - apt-get install -y kubectl - mkdir -p ${CRAFT_PART_INSTALL}/usr/local/bin/kubectl - cp -r $(which kubectl) ${CRAFT_PART_INSTALL}/usr/local/bin/kubectl + snap install kubectl --classic + mkdir -p ${CRAFT_PART_INSTALL}/snap/bin/kubectl + cp -r $(which kubectl) ${CRAFT_PART_INSTALL}/snap/bin/kubectl stage: - - usr/local/bin/kubectl + - snap/bin/kubectl permissions: - - path: usr/local/bin/kubectl + - path: snap/bin/kubectl owner: 584792 group: 584792 mode: "755" + # stage-packages: + # - apt-transport-https + # - ca-certificates + # - curl + # - gnupg + # override-build: | + # echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.31/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list + # chmod 644 /etc/apt/sources.list.d/kubernetes.list # helps tools such as command-not-found to work correctly + + # curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.31/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + # chmod 644 /etc/apt/keyrings/kubernetes-apt-keyring.gpg # allow unprivileged APT programs to read this keyring + + # apt-get update + # apt-get install -y kubectl + # mkdir -p ${CRAFT_PART_INSTALL}/usr/local/bin/kubectl + # cp -r $(which kubectl) ${CRAFT_PART_INSTALL}/usr/local/bin/kubectl + # stage: + # - usr/local/bin/kubectl + # permissions: + # - path: usr/local/bin/kubectl + # owner: 584792 + # group: 584792 + # mode: "755" pull-repo: after: [install-kubectl] From ecbe83938b883ace733de547cc462e3a3db8f7e0 Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Fri, 11 Oct 2024 03:35:38 +0300 Subject: [PATCH 17/32] stage snap bin in kubectl --- .trivyignore | 4 ++++ airbyte_rock/rockcraft.yaml | 48 ++----------------------------------- 2 files changed, 6 insertions(+), 46 deletions(-) create mode 100644 .trivyignore diff --git a/.trivyignore b/.trivyignore new file mode 100644 index 0000000..8436f1c --- /dev/null +++ b/.trivyignore @@ -0,0 +1,4 @@ +# stdlib CVEs +CVE-2024-24790 +CVE-2023-24538 +CVE-2023-24540 diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index 199218d..4b3b569 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -11,12 +11,6 @@ license: Apache-2.0 platforms: amd64: -# Please refer to -# https://discourse.ubuntu.com/t/unifying-user-identity-across-snaps-and-rocks/36469 -# for more information about shared user. -# The UID 584792 corresponds to _daemon_ user. -run_user: _daemon_ - environment: JAVA_HOME: /usr/lib/jvm/java-21-openjdk-amd64 CDK_PYTHON: /usr/bin/python3.10 @@ -25,7 +19,7 @@ environment: parts: - install-kubectl: + install-dependencies: plugin: nil override-build: | snap install kubectl --classic @@ -33,37 +27,9 @@ parts: cp -r $(which kubectl) ${CRAFT_PART_INSTALL}/snap/bin/kubectl stage: - snap/bin/kubectl - permissions: - - path: snap/bin/kubectl - owner: 584792 - group: 584792 - mode: "755" - # stage-packages: - # - apt-transport-https - # - ca-certificates - # - curl - # - gnupg - # override-build: | - # echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.31/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list - # chmod 644 /etc/apt/sources.list.d/kubernetes.list # helps tools such as command-not-found to work correctly - - # curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.31/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg - # chmod 644 /etc/apt/keyrings/kubernetes-apt-keyring.gpg # allow unprivileged APT programs to read this keyring - - # apt-get update - # apt-get install -y kubectl - # mkdir -p ${CRAFT_PART_INSTALL}/usr/local/bin/kubectl - # cp -r $(which kubectl) ${CRAFT_PART_INSTALL}/usr/local/bin/kubectl - # stage: - # - usr/local/bin/kubectl - # permissions: - # - path: usr/local/bin/kubectl - # owner: 584792 - # group: 584792 - # mode: "755" pull-repo: - after: [install-kubectl] + after: [install-dependencies] plugin: dump source: https://github.com/airbytehq/airbyte-platform.git # yamllint disable-line source-type: git @@ -79,11 +45,6 @@ parts: stage: - airbyte-platform - usr/local/lib/python3.10/dist-packages - permissions: - - path: usr/local/lib/python3.10/dist-packages - owner: 584792 - group: 584792 - mode: "755" assemble: after: [pull-repo] @@ -141,8 +102,3 @@ parts: pod-sweeper.sh: airbyte-pod-sweeper/airbyte-app/bin/airbyte-pod-sweeper stage: - airbyte-pod-sweeper/airbyte-app/bin/airbyte-pod-sweeper - permissions: - - path: airbyte-pod-sweeper/airbyte-app/bin/airbyte-pod-sweeper - owner: 584792 - group: 584792 - mode: "755" From a6ebfe11625e8701517e7b50c6bbcff3f95290ea Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Fri, 11 Oct 2024 13:19:46 +0300 Subject: [PATCH 18/32] install kubectl using apt --- airbyte_rock/rockcraft.yaml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index 4b3b569..6a5f5c7 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -21,12 +21,23 @@ environment: parts: install-dependencies: plugin: nil + stage-packages: + - apt-transport-https + - ca-certificates + - curl + - gnupg override-build: | - snap install kubectl --classic - mkdir -p ${CRAFT_PART_INSTALL}/snap/bin/kubectl - cp -r $(which kubectl) ${CRAFT_PART_INSTALL}/snap/bin/kubectl + echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.31/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list + chmod 644 /etc/apt/sources.list.d/kubernetes.list # helps tools such as command-not-found to work correctly + curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.31/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + chmod 644 /etc/apt/keyrings/kubernetes-apt-keyring.gpg # allow unprivileged APT programs to read this keyring + apt-get update + apt-get install -y kubectl + mkdir -p ${CRAFT_PART_INSTALL}/usr/local/bin/kubectl + install -o root -g root -m 0755 kubectl ${CRAFT_PART_INSTALL}/usr/local/bin/kubectl + cp -r $(which kubectl) ${CRAFT_PART_INSTALL}/usr/local/bin/kubectl stage: - - snap/bin/kubectl + - usr/local/bin/kubectl pull-repo: after: [install-dependencies] From 6755bb24245612b3fe04479a9ca20f22612ec479 Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Fri, 11 Oct 2024 13:32:33 +0300 Subject: [PATCH 19/32] attempt to fix kubectl staging --- airbyte_rock/rockcraft.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index 6a5f5c7..98cea73 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -33,9 +33,10 @@ parts: chmod 644 /etc/apt/keyrings/kubernetes-apt-keyring.gpg # allow unprivileged APT programs to read this keyring apt-get update apt-get install -y kubectl - mkdir -p ${CRAFT_PART_INSTALL}/usr/local/bin/kubectl - install -o root -g root -m 0755 kubectl ${CRAFT_PART_INSTALL}/usr/local/bin/kubectl - cp -r $(which kubectl) ${CRAFT_PART_INSTALL}/usr/local/bin/kubectl + mkdir -p ${CRAFT_PART_INSTALL}/usr/local/bin + cp -r $(which kubectl) ${CRAFT_PART_INSTALL}/kubectl + organize: + kubectl: usr/local/bin/kubectl stage: - usr/local/bin/kubectl From 7f871d6f5c6f828768ccd196d457b80aeab65fbc Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Fri, 11 Oct 2024 16:32:16 +0300 Subject: [PATCH 20/32] update airbyte version to address compatibility issues --- airbyte_rock/rockcraft.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index 98cea73..7c6c3ee 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -45,7 +45,7 @@ parts: plugin: dump source: https://github.com/airbytehq/airbyte-platform.git # yamllint disable-line source-type: git - source-tag: v0.60.0 + source-tag: v0.63.7 stage-packages: - python3.10-venv override-build: | From 330b0300abf955fd3506fb24ace272b1e45571d0 Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Fri, 11 Oct 2024 17:15:39 +0300 Subject: [PATCH 21/32] update airbyte to v0.64.7 --- airbyte_rock/rockcraft.yaml | 2 +- src/charm_helpers.py | 2 +- src/literals.py | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index 7c6c3ee..6c1a543 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -45,7 +45,7 @@ parts: plugin: dump source: https://github.com/airbytehq/airbyte-platform.git # yamllint disable-line source-type: git - source-tag: v0.63.7 + source-tag: v0.64.7 stage-packages: - python3.10-venv override-build: | diff --git a/src/charm_helpers.py b/src/charm_helpers.py index ec66b5f..6398ae6 100644 --- a/src/charm_helpers.py +++ b/src/charm_helpers.py @@ -137,7 +137,7 @@ def create_env(model_name, app_name, container_name, config, state): "CONNECTOR_BUILDER_SERVER_API_HOST": f"{app_name}:{CONNECTOR_BUILDER_SERVER_API_PORT}", "CONNECTOR_BUILDER_API_HOST": f"{app_name}:{CONNECTOR_BUILDER_SERVER_API_PORT}", "AIRBYTE_API_HOST": f"{app_name}:{AIRBYTE_API_PORT}/api/public", - "AIRBYTE_URL": config["webapp-url"], + # "AIRBYTE_URL": config["webapp-url"], } # https://github.com/airbytehq/airbyte/issues/29506#issuecomment-1775148609 diff --git a/src/literals.py b/src/literals.py index c6fedf3..8184ed6 100644 --- a/src/literals.py +++ b/src/literals.py @@ -7,6 +7,9 @@ INTERNAL_API_PORT = 8001 AIRBYTE_API_PORT = 8006 WORKLOAD_API_PORT = 8007 +AIRBYTE_VERSION = "0.64.7" +DB_NAME = "airbyte-k8s_db" + CONTAINER_HEALTH_CHECK_MAP = { "airbyte-api-server": { @@ -27,9 +30,7 @@ "airbyte-workers": {"port": 9000, "health_endpoint": "/"}, } -DB_NAME = "airbyte-k8s_db" REQUIRED_S3_PARAMETERS = ["region", "endpoint", "access-key", "secret-key"] -AIRBYTE_VERSION = "0.60.0" BUCKET_CONFIGS = [ "storage-bucket-logs", "storage-bucket-state", From 9d97a8df65578018f109fc1fce19ef928b4e476b Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Fri, 11 Oct 2024 17:45:49 +0300 Subject: [PATCH 22/32] update airbyte to v1.1.0 --- airbyte_rock/rockcraft.yaml | 8 ++++---- charmcraft.yaml | 24 +----------------------- src/charm_helpers.py | 2 +- src/literals.py | 8 ++++---- tests/integration/helpers.py | 19 ------------------- tests/unit/test_charm.py | 3 +-- 6 files changed, 11 insertions(+), 53 deletions(-) diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index 6c1a543..d9ff402 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -45,7 +45,7 @@ parts: plugin: dump source: https://github.com/airbytehq/airbyte-platform.git # yamllint disable-line source-type: git - source-tag: v0.64.7 + source-tag: v1.1.0 stage-packages: - python3.10-venv override-build: | @@ -86,21 +86,21 @@ parts: ls ${CRAFT_STAGE}/airbyte-platform ls ${CRAFT_STAGE}/airbyte-platform/airbyte-bootloader mkdir ${CRAFT_PART_INSTALL}/airbyte-server - mkdir ${CRAFT_PART_INSTALL}/airbyte-api-server + # mkdir ${CRAFT_PART_INSTALL}/airbyte-api-server mkdir ${CRAFT_PART_INSTALL}/airbyte-workers mkdir ${CRAFT_PART_INSTALL}/airbyte-bootloader mkdir ${CRAFT_PART_INSTALL}/airbyte-cron mkdir ${CRAFT_PART_INSTALL}/airbyte-connector-builder-server tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-server/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-server - tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-api-server/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-api-server + # tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-api-server/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-api-server tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-workers/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-workers tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-bootloader/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-bootloader tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-cron/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-cron tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-connector-builder-server/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-connector-builder-server stage: - airbyte-server - - airbyte-api-server + # - airbyte-api-server - airbyte-workers - airbyte-bootloader - airbyte-cron diff --git a/charmcraft.yaml b/charmcraft.yaml index 5efe5d7..5b6f230 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -428,8 +428,6 @@ config: # Your workload’s containers. containers: - airbyte-api-server: - resource: airbyte-image airbyte-bootloader: resource: airbyte-image airbyte-connector-builder-server: @@ -448,24 +446,4 @@ resources: airbyte-image: type: oci-image description: OCI image for Airbyte - # airbyte-api-server: - # type: oci-image - # description: OCI image for Airbyte API server - # airbyte-bootloader: - # type: oci-image - # description: OCI image for Airbyte Bootloader - # airbyte-connector-builder-server: - # type: oci-image - # description: OCI image for Airbyte Connector Builder Server - # airbyte-cron: - # type: oci-image - # description: OCI image for Airbyte Cron - # airbyte-pod-sweeper: - # type: oci-image - # description: OCI image for Airbyte Pod Sweeper - # airbyte-server: - # type: oci-image - # description: OCI image for Airbyte Server - # airbyte-workers: - # type: oci-image - # description: OCI image for Airbyte Worker + diff --git a/src/charm_helpers.py b/src/charm_helpers.py index 6398ae6..a0f9982 100644 --- a/src/charm_helpers.py +++ b/src/charm_helpers.py @@ -131,7 +131,7 @@ def create_env(model_name, app_name, container_name, config, state): "RUNNING_TTL_MINUTES": config["pod-running-ttl-minutes"], "SUCCEEDED_TTL_MINUTES": config["pod-successful-ttl-minutes"], "UNSUCCESSFUL_TTL_MINUTES": config["pod-unsuccessful-ttl-minutes"], - "INTERNAL_API_HOST": f"{app_name}:{INTERNAL_API_PORT}", + "INTERNAL_API_HOST": f"http://{app_name}:{INTERNAL_API_PORT}", "AIRBYTE_SERVER_HOST": f"{app_name}:{INTERNAL_API_PORT}", "CONFIG_API_HOST": f"{app_name}:{INTERNAL_API_PORT}", "CONNECTOR_BUILDER_SERVER_API_HOST": f"{app_name}:{CONNECTOR_BUILDER_SERVER_API_PORT}", diff --git a/src/literals.py b/src/literals.py index 8184ed6..dc2d1e6 100644 --- a/src/literals.py +++ b/src/literals.py @@ -12,10 +12,10 @@ CONTAINER_HEALTH_CHECK_MAP = { - "airbyte-api-server": { - "port": AIRBYTE_API_PORT, - "health_endpoint": "/health", - }, + # "airbyte-api-server": { + # "port": AIRBYTE_API_PORT, + # "health_endpoint": "/health", + # }, "airbyte-bootloader": None, "airbyte-connector-builder-server": None, "airbyte-cron": { diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 5e39cf2..98349b3 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -28,25 +28,6 @@ POST_HEADERS = {"accept": "application/json", "content-type": "application/json"} -def get_airbyte_charm_resources(): - """Fetch charm resources from charmcraft.yaml. - - Returns: - Charm resources. - """ - return { - "airbyte-api-server": METADATA["resources"]["airbyte-api-server"]["upstream-source"], - "airbyte-bootloader": METADATA["resources"]["airbyte-bootloader"]["upstream-source"], - "airbyte-connector-builder-server": METADATA["resources"]["airbyte-connector-builder-server"][ - "upstream-source" - ], - "airbyte-cron": METADATA["resources"]["airbyte-cron"]["upstream-source"], - "airbyte-pod-sweeper": METADATA["resources"]["airbyte-pod-sweeper"]["upstream-source"], - "airbyte-server": METADATA["resources"]["airbyte-server"]["upstream-source"], - "airbyte-workers": METADATA["resources"]["airbyte-workers"]["upstream-source"], - } - - async def run_sample_workflow(ops_test: OpsTest): """Connect a client and runs a basic Temporal workflow. diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 22fecae..8803bf3 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -355,7 +355,6 @@ def create_plan(container_name, storage_type): **BASE_ENV, "AIRBYTE_API_HOST": "airbyte-k8s:8006/api/public", "AIRBYTE_SERVER_HOST": "airbyte-k8s:8001", - "AIRBYTE_URL": "http://airbyte-ui-k8s:8080", "AWS_ACCESS_KEY_ID": "access", "AWS_SECRET_ACCESS_KEY": "secret", "CONFIG_API_HOST": "airbyte-k8s:8001", @@ -368,7 +367,7 @@ def create_plan(container_name, storage_type): "DATABASE_PORT": "5432", "DATABASE_URL": "jdbc:postgresql://myhost:5432/airbyte-k8s_db", "DATABASE_USER": "jean-luc@db", - "INTERNAL_API_HOST": "airbyte-k8s:8001", + "INTERNAL_API_HOST": "http://airbyte-k8s:8001", "JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION": "0.29.15.001", "JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY": "IfNotPresent", "JOB_KUBE_NAMESPACE": "airbyte-model", From 1632f92afed7b170f8fac26fa28d5f6e8d3f4a0e Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Fri, 11 Oct 2024 18:33:27 +0300 Subject: [PATCH 23/32] test airbyte v0.63.8 --- airbyte_rock/rockcraft.yaml | 8 ++++---- charmcraft.yaml | 2 ++ src/literals.py | 8 ++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index d9ff402..1cb96b1 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -45,7 +45,7 @@ parts: plugin: dump source: https://github.com/airbytehq/airbyte-platform.git # yamllint disable-line source-type: git - source-tag: v1.1.0 + source-tag: v0.63.8 stage-packages: - python3.10-venv override-build: | @@ -86,21 +86,21 @@ parts: ls ${CRAFT_STAGE}/airbyte-platform ls ${CRAFT_STAGE}/airbyte-platform/airbyte-bootloader mkdir ${CRAFT_PART_INSTALL}/airbyte-server - # mkdir ${CRAFT_PART_INSTALL}/airbyte-api-server + mkdir ${CRAFT_PART_INSTALL}/airbyte-api-server mkdir ${CRAFT_PART_INSTALL}/airbyte-workers mkdir ${CRAFT_PART_INSTALL}/airbyte-bootloader mkdir ${CRAFT_PART_INSTALL}/airbyte-cron mkdir ${CRAFT_PART_INSTALL}/airbyte-connector-builder-server tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-server/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-server - # tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-api-server/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-api-server + tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-api-server/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-api-server tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-workers/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-workers tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-bootloader/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-bootloader tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-cron/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-cron tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-connector-builder-server/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-connector-builder-server stage: - airbyte-server - # - airbyte-api-server + - airbyte-api-server - airbyte-workers - airbyte-bootloader - airbyte-cron diff --git a/charmcraft.yaml b/charmcraft.yaml index 5b6f230..d9dbb42 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -428,6 +428,8 @@ config: # Your workload’s containers. containers: + airbyte-api-server: + resource: airbyte-image airbyte-bootloader: resource: airbyte-image airbyte-connector-builder-server: diff --git a/src/literals.py b/src/literals.py index dc2d1e6..8184ed6 100644 --- a/src/literals.py +++ b/src/literals.py @@ -12,10 +12,10 @@ CONTAINER_HEALTH_CHECK_MAP = { - # "airbyte-api-server": { - # "port": AIRBYTE_API_PORT, - # "health_endpoint": "/health", - # }, + "airbyte-api-server": { + "port": AIRBYTE_API_PORT, + "health_endpoint": "/health", + }, "airbyte-bootloader": None, "airbyte-connector-builder-server": None, "airbyte-cron": { From 88d5886d048116dfe4babe6511a65080c1f94caa Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Fri, 11 Oct 2024 20:51:27 +0300 Subject: [PATCH 24/32] add logs to integration tests --- tests/integration/helpers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 98349b3..9d9a838 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -170,6 +170,7 @@ def create_airbyte_source(api_url, workspace_id): logger.info("creating Airbyte source") response = requests.post(url, json=payload, headers=POST_HEADERS, timeout=300) + logger.info(response.json()) assert response.status_code == 200 return response.json().get("sourceId") @@ -206,6 +207,7 @@ def create_airbyte_destination(api_url, model_name, workspace_id, db_password): logger.info("creating Airbyte destination") response = requests.post(url, json=payload, headers=POST_HEADERS, timeout=300) + logger.info(response.json()) assert response.status_code == 200 return response.json().get("destinationId") @@ -235,6 +237,7 @@ def create_airbyte_connection(api_url, source_id, destination_id): logger.info("creating Airbyte connection") response = requests.post(url, json=payload, headers=POST_HEADERS, timeout=300) + logger.info(response.json()) assert response.status_code == 200 return response.json().get("connectionId") @@ -254,6 +257,7 @@ def trigger_airbyte_connection(api_url, connection_id): payload = {"jobType": "sync", "connectionId": connection_id} logger.info("triggering Airbyte connection") response = requests.post(url, json=payload, headers=POST_HEADERS, timeout=300) + logger.info(response.json()) assert response.status_code == 200 return response.json().get("jobId") From 53a7a327a8b880b2d57eac3eebd8f0aeddaa92fb Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Fri, 11 Oct 2024 21:30:02 +0300 Subject: [PATCH 25/32] set orchestrator launcher to false --- src/literals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/literals.py b/src/literals.py index 8184ed6..2d4d28a 100644 --- a/src/literals.py +++ b/src/literals.py @@ -59,7 +59,7 @@ "SHOULD_RUN_NOTIFY_WORKFLOWS": "true", "CONNECTOR_BUILDER_API_URL": "/connector-builder-api", "TEMPORAL_WORKER_PORTS": "9001,9002,9003,9004,9005,9006,9007,9008,9009,9010,9011,9012,9013,9014,9015,9016,9017,9018,9019,9020,9021,9022,9023,9024,9025,9026,9027,9028,9029,9030", - "CONTAINER_ORCHESTRATOR_ENABLED": "true", + "CONTAINER_ORCHESTRATOR_ENABLED": "false", "CONTAINER_ORCHESTRATOR_IMAGE": "airbyte/container-orchestrator:0.60.0", "LOG4J_CONFIGURATION_FILE": "log4j2-minio.xml", } From a6469adb3dda72adfa5b8780837212d0a525a1e7 Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Fri, 11 Oct 2024 22:25:56 +0300 Subject: [PATCH 26/32] update orchestrator container version --- airbyte_rock/rockcraft.yaml | 6 +++--- src/literals.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index 1cb96b1..a519bb1 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -40,7 +40,7 @@ parts: stage: - usr/local/bin/kubectl - pull-repo: + pull-airbyte-repo: after: [install-dependencies] plugin: dump source: https://github.com/airbytehq/airbyte-platform.git # yamllint disable-line @@ -48,7 +48,7 @@ parts: source-tag: v0.63.8 stage-packages: - python3.10-venv - override-build: | + override-build: | cp -r . ${CRAFT_PART_INSTALL}/airbyte-platform pip install --upgrade setuptools pip airbyte-cdk==5.12.0 \ --target=/${CRAFT_PART_INSTALL}/dist @@ -59,7 +59,7 @@ parts: - usr/local/lib/python3.10/dist-packages assemble: - after: [pull-repo] + after: [pull-airbyte-repo] plugin: nil build-packages: - jq diff --git a/src/literals.py b/src/literals.py index 2d4d28a..91ce1b5 100644 --- a/src/literals.py +++ b/src/literals.py @@ -7,7 +7,7 @@ INTERNAL_API_PORT = 8001 AIRBYTE_API_PORT = 8006 WORKLOAD_API_PORT = 8007 -AIRBYTE_VERSION = "0.64.7" +AIRBYTE_VERSION = "0.63.8" DB_NAME = "airbyte-k8s_db" @@ -59,7 +59,7 @@ "SHOULD_RUN_NOTIFY_WORKFLOWS": "true", "CONNECTOR_BUILDER_API_URL": "/connector-builder-api", "TEMPORAL_WORKER_PORTS": "9001,9002,9003,9004,9005,9006,9007,9008,9009,9010,9011,9012,9013,9014,9015,9016,9017,9018,9019,9020,9021,9022,9023,9024,9025,9026,9027,9028,9029,9030", - "CONTAINER_ORCHESTRATOR_ENABLED": "false", - "CONTAINER_ORCHESTRATOR_IMAGE": "airbyte/container-orchestrator:0.60.0", + "CONTAINER_ORCHESTRATOR_ENABLED": "true", + "CONTAINER_ORCHESTRATOR_IMAGE": f"airbyte/container-orchestrator:{AIRBYTE_VERSION}", "LOG4J_CONFIGURATION_FILE": "log4j2-minio.xml", } From 6fb03bfaccc2ef391e782daecf065d49a51bb14a Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Fri, 11 Oct 2024 23:33:29 +0300 Subject: [PATCH 27/32] update api urls in itests --- tests/integration/helpers.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 9d9a838..8fe12e6 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -143,7 +143,7 @@ def get_airbyte_workspace_id(api_url): Returns: Airbyte workspace ID. """ - url = f"{api_url}/v1/workspaces?includeDeleted=false&limit=20&offset=0" + url = f"{api_url}/api/public/v1/workspaces?includeDeleted=false&limit=20&offset=0" logger.info("fetching Airbyte workspace ID") response = requests.get(url, headers=GET_HEADERS, timeout=300) @@ -161,7 +161,7 @@ def create_airbyte_source(api_url, workspace_id): Returns: Created source ID. """ - url = f"{api_url}/v1/sources" + url = f"{api_url}/api/public/v1/sources" payload = { "configuration": {"sourceType": "pokeapi", "pokemon_name": "pikachu"}, "name": "API Test", @@ -188,7 +188,7 @@ def create_airbyte_destination(api_url, model_name, workspace_id, db_password): Returns: Created destination ID. """ - url = f"{api_url}/v1/destinations" + url = f"{api_url}/api/public/v1/destinations" payload = { "configuration": { "destinationType": "postgres", @@ -224,7 +224,7 @@ def create_airbyte_connection(api_url, source_id, destination_id): Returns: Created connection ID. """ - url = f"{api_url}/v1/connections" + url = f"{api_url}/api/public/v1/connections" payload = { "schedule": {"scheduleType": "manual"}, "dataResidency": "auto", @@ -253,7 +253,7 @@ def trigger_airbyte_connection(api_url, connection_id): Returns: Created job ID. """ - url = f"{api_url}/v1/jobs" + url = f"{api_url}/api/public/v1/jobs" payload = {"jobType": "sync", "connectionId": connection_id} logger.info("triggering Airbyte connection") response = requests.post(url, json=payload, headers=POST_HEADERS, timeout=300) @@ -273,7 +273,7 @@ def check_airbyte_job_status(api_url, job_id): Returns: Job status. """ - url = f"{api_url}/v1/jobs/{job_id}" + url = f"{api_url}/api/public/v1/jobs/{job_id}" logger.info("fetching Airbyte job status") response = requests.get(url, headers=GET_HEADERS, timeout=120) logger.info(response.json()) @@ -291,7 +291,7 @@ def cancel_airbyte_job(api_url, job_id): Returns: Job status. """ - url = f"{api_url}/v1/jobs/{job_id}" + url = f"{api_url}/api/public/v1/jobs/{job_id}" logger.info("cancelling Airbyte job") response = requests.delete(url, headers=GET_HEADERS, timeout=120) logger.info(response.json()) @@ -347,7 +347,7 @@ async def run_test_sync_job(ops_test): # Wait until job is successful job_successful = False - for j in range(15): + for j in range(7): logger.info(f"job {i + 1} attempt {j + 1}: getting job status") status = check_airbyte_job_status(api_url, job_id) @@ -359,8 +359,8 @@ async def run_test_sync_job(ops_test): job_successful = True break - logger.info(f"job {i + 1} attempt {j + 1}: job still running, retrying in 20 seconds") - time.sleep(20) + logger.info(f"job {i + 1} attempt {j + 1}: job still running, retrying in 10 seconds") + time.sleep(10) if job_successful: break From 87e1012e1a6fb6e94deb19ba5c8efb331de1fd83 Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Sat, 12 Oct 2024 00:13:18 +0300 Subject: [PATCH 28/32] update api port in itest --- tests/integration/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 8fe12e6..c7289d0 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -325,7 +325,7 @@ async def run_test_sync_job(ops_test): ops_test: PyTest object. """ # Create connection - api_url = await get_unit_url(ops_test, application=APP_NAME_AIRBYTE_SERVER, unit=0, port=8006) + api_url = await get_unit_url(ops_test, application=APP_NAME_AIRBYTE_SERVER, unit=0, port=8001) logger.info("curling app address: %s", api_url) workspace_id = get_airbyte_workspace_id(api_url) db_password = await get_db_password(ops_test) From aadd131fd481e273b505918d7c0d427e7e14be48 Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Sat, 12 Oct 2024 01:08:33 +0300 Subject: [PATCH 29/32] refactor rock part and remove redundant artifacts --- airbyte_rock/patches/db-jooq.patch | 15 --------------- airbyte_rock/rockcraft.yaml | 15 ++++++++------- src/charm_helpers.py | 1 - 3 files changed, 8 insertions(+), 23 deletions(-) delete mode 100644 airbyte_rock/patches/db-jooq.patch diff --git a/airbyte_rock/patches/db-jooq.patch b/airbyte_rock/patches/db-jooq.patch deleted file mode 100644 index 75dcb8d..0000000 --- a/airbyte_rock/patches/db-jooq.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff --git a/gradle.properties b/gradle.properties -index 038c2ac2c4..95c25d9dad 100644 ---- a/gradle.properties -+++ b/gradle.properties -@@ -3,8 +3,9 @@ - org.gradle.parallel=true - org.gradle.jvmargs=-Xmx8g -Xss4m - org.gradle.caching=true -+org.gradle.daemon=false - # Note, this might have issues on the normal Github runner. --org.gradle.vfs.watch=true -+org.gradle.vfs.watch=false - - # Remove once we've migrated to Gradle 9 - # This opts in early to Gradle 9 behavior diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index a519bb1..7cb3306 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -26,7 +26,9 @@ parts: - ca-certificates - curl - gnupg + - python3.10-venv override-build: | + # Install kubectl echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.31/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list chmod 644 /etc/apt/sources.list.d/kubernetes.list # helps tools such as command-not-found to work correctly curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.31/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg @@ -35,10 +37,16 @@ parts: apt-get install -y kubectl mkdir -p ${CRAFT_PART_INSTALL}/usr/local/bin cp -r $(which kubectl) ${CRAFT_PART_INSTALL}/kubectl + + pip install --upgrade setuptools pip airbyte-cdk==5.12.0 \ + --target=/${CRAFT_PART_INSTALL}/dist + organize: kubectl: usr/local/bin/kubectl + dist: usr/local/lib/python3.10/dist-packages stage: - usr/local/bin/kubectl + - usr/local/lib/python3.10/dist-packages pull-airbyte-repo: after: [install-dependencies] @@ -46,17 +54,10 @@ parts: source: https://github.com/airbytehq/airbyte-platform.git # yamllint disable-line source-type: git source-tag: v0.63.8 - stage-packages: - - python3.10-venv override-build: | cp -r . ${CRAFT_PART_INSTALL}/airbyte-platform - pip install --upgrade setuptools pip airbyte-cdk==5.12.0 \ - --target=/${CRAFT_PART_INSTALL}/dist - organize: - dist: usr/local/lib/python3.10/dist-packages stage: - airbyte-platform - - usr/local/lib/python3.10/dist-packages assemble: after: [pull-airbyte-repo] diff --git a/src/charm_helpers.py b/src/charm_helpers.py index a0f9982..aa2c76f 100644 --- a/src/charm_helpers.py +++ b/src/charm_helpers.py @@ -137,7 +137,6 @@ def create_env(model_name, app_name, container_name, config, state): "CONNECTOR_BUILDER_SERVER_API_HOST": f"{app_name}:{CONNECTOR_BUILDER_SERVER_API_PORT}", "CONNECTOR_BUILDER_API_HOST": f"{app_name}:{CONNECTOR_BUILDER_SERVER_API_PORT}", "AIRBYTE_API_HOST": f"{app_name}:{AIRBYTE_API_PORT}/api/public", - # "AIRBYTE_URL": config["webapp-url"], } # https://github.com/airbytehq/airbyte/issues/29506#issuecomment-1775148609 From 6fe7361414accd4494b134ea64a55cef50d4c98e Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Mon, 14 Oct 2024 15:52:38 +0300 Subject: [PATCH 30/32] address review comments --- airbyte_rock/rockcraft.yaml | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index 7cb3306..b317b38 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -6,7 +6,6 @@ summary: Airbyte rock description: Airbyte OCI image for the Airbyte charm version: "1.0" base: ubuntu@22.04 -build-base: ubuntu@22.04 license: Apache-2.0 platforms: amd64: @@ -36,14 +35,11 @@ parts: apt-get update apt-get install -y kubectl mkdir -p ${CRAFT_PART_INSTALL}/usr/local/bin - cp -r $(which kubectl) ${CRAFT_PART_INSTALL}/kubectl + mkdir -p ${CRAFT_PART_INSTALL}/usr/local/lib/python3.10/dist-packages + cp -r $(which kubectl) ${CRAFT_PART_INSTALL}/usr/local/bin/kubectl pip install --upgrade setuptools pip airbyte-cdk==5.12.0 \ - --target=/${CRAFT_PART_INSTALL}/dist - - organize: - kubectl: usr/local/bin/kubectl - dist: usr/local/lib/python3.10/dist-packages + --target=/${CRAFT_PART_INSTALL}/usr/local/lib/python3.10/dist-packages stage: - usr/local/bin/kubectl - usr/local/lib/python3.10/dist-packages @@ -54,10 +50,6 @@ parts: source: https://github.com/airbytehq/airbyte-platform.git # yamllint disable-line source-type: git source-tag: v0.63.8 - override-build: | - cp -r . ${CRAFT_PART_INSTALL}/airbyte-platform - stage: - - airbyte-platform assemble: after: [pull-airbyte-repo] @@ -70,7 +62,6 @@ parts: - gradle - openjdk-21-jdk-headless - npm - - docker.io build-snaps: - docker stage-packages: @@ -84,8 +75,6 @@ parts: after: [assemble] plugin: nil override-build: | - ls ${CRAFT_STAGE}/airbyte-platform - ls ${CRAFT_STAGE}/airbyte-platform/airbyte-bootloader mkdir ${CRAFT_PART_INSTALL}/airbyte-server mkdir ${CRAFT_PART_INSTALL}/airbyte-api-server mkdir ${CRAFT_PART_INSTALL}/airbyte-workers From ad21197b801aef56c0dd32ff83ad01b0cce55d3a Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Tue, 15 Oct 2024 01:12:50 +0300 Subject: [PATCH 31/32] revert change of pull-repo step --- airbyte_rock/rockcraft.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/airbyte_rock/rockcraft.yaml b/airbyte_rock/rockcraft.yaml index b317b38..5dc6b4b 100644 --- a/airbyte_rock/rockcraft.yaml +++ b/airbyte_rock/rockcraft.yaml @@ -50,6 +50,10 @@ parts: source: https://github.com/airbytehq/airbyte-platform.git # yamllint disable-line source-type: git source-tag: v0.63.8 + override-build: | + cp -r . ${CRAFT_PART_INSTALL}/airbyte-platform + stage: + - airbyte-platform assemble: after: [pull-airbyte-repo] From f686cd9625aee7dd605919c69d39ba275114a03d Mon Sep 17 00:00:00 2001 From: Ali Kelkawi Date: Tue, 15 Oct 2024 10:47:54 +0300 Subject: [PATCH 32/32] remove redundant pod sweeper script --- src/scripts/pod-sweeper.sh | 69 -------------------------------------- 1 file changed, 69 deletions(-) delete mode 100755 src/scripts/pod-sweeper.sh diff --git a/src/scripts/pod-sweeper.sh b/src/scripts/pod-sweeper.sh deleted file mode 100755 index d5e0d0b..0000000 --- a/src/scripts/pod-sweeper.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -# Copyright 2024 Canonical Ltd. -# See LICENSE file for licensing details. - -# https://github.com/airbytehq/airbyte-platform/blob/main/charts/airbyte-pod-sweeper/templates/configmap.yaml -# TODO(kelkawi-a): Move this to Airbyte ROCK - -get_job_pods() { - # echo "Running kubectl command to get job pods..." - kubectl -n "${JOB_KUBE_NAMESPACE}" -L airbyte -l airbyte=job-pod \ - get pods \ - -o=jsonpath='{range .items[*]} {.metadata.name} {.status.phase} {.status.conditions[0].lastTransitionTime} {.status.startTime}{"\n"}{end}' -} - -delete_pod() { - printf "From status '%s' since '%s', " "$2" "$3" - echo "$1" | grep -v "STATUS" | awk '{print $1}' | xargs --no-run-if-empty kubectl -n "${JOB_KUBE_NAMESPACE}" delete pod -} - -while : -do - echo "Starting pod sweeper cycle:" - - if [ -n "${RUNNING_TTL_MINUTES}" ]; then - # Time window for running pods - RUNNING_DATE_STR=$(date -d "now - ${RUNNING_TTL_MINUTES} minutes" --utc -Ins) - RUNNING_DATE=$(date -d "${RUNNING_DATE_STR}" +%s) - echo "Will sweep running pods from before ${RUNNING_DATE_STR}" - fi - - if [ -n "${SUCCEEDED_TTL_MINUTES}" ]; then - # Shorter time window for succeeded pods - SUCCESS_DATE_STR=$(date -d "now - ${SUCCEEDED_TTL_MINUTES} minutes" --utc -Ins) - SUCCESS_DATE=$(date -d "${SUCCESS_DATE_STR}" +%s) - echo "Will sweep succeeded pods from before ${SUCCESS_DATE_STR}" - fi - - if [ -n "${UNSUCCESSFUL_TTL_MINUTES}" ]; then - # Longer time window for unsuccessful pods (to debug) - NON_SUCCESS_DATE_STR=$(date -d "now - ${UNSUCCESSFUL_TTL_MINUTES} minutes" --utc -Ins) - NON_SUCCESS_DATE=$(date -d "${NON_SUCCESS_DATE_STR}" +%s) - echo "Will sweep unsuccessful pods from before ${NON_SUCCESS_DATE_STR}" - fi - - echo "Running kubectl command to get job pods..." - get_job_pods | while read -r POD; do - IFS=' ' read -r POD_NAME POD_STATUS POD_DATE_STR POD_START_DATE_STR <<< "$POD" - - POD_DATE=$(date -d "${POD_DATE_STR:-$POD_START_DATE_STR}" '+%s') - echo "Evaluating pod: $POD_NAME with status $POD_STATUS since $POD_DATE_STR" - - if [ -n "${RUNNING_TTL_MINUTES}" ] && [ "$POD_STATUS" = "Running" ]; then - if [ "$POD_DATE" -lt "$RUNNING_DATE" ]; then - delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" - fi - elif [ -n "${SUCCEEDED_TTL_MINUTES}" ] && { [[ "$POD_STATUS" = "Succeeded" ]] || [[ "$POD_STATUS" = "Completed" ]]; }; then - if [ "$POD_DATE" -lt "$SUCCESS_DATE" ]; then - delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" - fi - elif [ -n "${UNSUCCESSFUL_TTL_MINUTES}" ] && [ "$POD_STATUS" != "Running" ] && [ "$POD_STATUS" != "Succeeded" ]; then - if [ "$POD_DATE" -lt "$NON_SUCCESS_DATE" ]; then - delete_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" - fi - fi - done - - echo "Completed pod sweeper cycle. Sleeping for 60 seconds..." - sleep 60 -done