Add Airbyte rock (#18)

* add airbyte rock

.github/workflows/integration_test.yaml

@@ -18,3 +18,5 @@ jobs:
       self-hosted-runner: true
       self-hosted-runner-label: "xlarge"
       microk8s-addons: "dns ingress rbac storage metallb:10.15.119.2-10.15.119.4 registry"
+      trivy-severity-config: CRITICAL
+      rockcraft-enable-security-nesting: true

.licenserc.yaml

@@ -34,4 +34,5 @@ header:
     - 'LICENSE'
     - 'trivy.yaml'
     - 'lib/**'
+    - '**/*.patch'
   comment: on-failure

.trivyignore

@@ -0,0 +1,4 @@
+# stdlib CVEs
+CVE-2024-24790
+CVE-2023-24538
+CVE-2023-24540

airbyte_rock/rockcraft.yaml

@@ -0,0 +1,110 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+name: airbyte
+summary: Airbyte rock
+description: Airbyte OCI image for the Airbyte charm
+version: "1.0"
+base: [email protected]
+license: Apache-2.0
+platforms:
+  amd64:
+
+environment:
+  JAVA_HOME: /usr/lib/jvm/java-21-openjdk-amd64
+  CDK_PYTHON: /usr/bin/python3.10
+  CDK_ENTRYPOINT: /usr/lib/python3.10/dist-packages/airbyte_cdk/connector_builder/main.py
+  CDK_VERSION: "5.12.0"
+
+
+parts:
+  install-dependencies:
+    plugin: nil
+    stage-packages:
+      - apt-transport-https
+      - ca-certificates
+      - curl 
+      - gnupg
+      - python3.10-venv  
+    override-build: |
+      # Install kubectl
+      echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.31/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list
+      chmod 644 /etc/apt/sources.list.d/kubernetes.list   # helps tools such as command-not-found to work correctly
+      curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.31/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
+      chmod 644 /etc/apt/keyrings/kubernetes-apt-keyring.gpg # allow unprivileged APT programs to read this keyring
+      apt-get update
+      apt-get install -y kubectl
+      mkdir -p ${CRAFT_PART_INSTALL}/usr/local/bin
+      mkdir -p ${CRAFT_PART_INSTALL}/usr/local/lib/python3.10/dist-packages
+      cp -r $(which kubectl) ${CRAFT_PART_INSTALL}/usr/local/bin/kubectl
+
+      pip install --upgrade setuptools pip airbyte-cdk==5.12.0 \
+          --target=/${CRAFT_PART_INSTALL}/usr/local/lib/python3.10/dist-packages
+    stage:
+      - usr/local/bin/kubectl
+      - usr/local/lib/python3.10/dist-packages
+
+  pull-airbyte-repo:
+    after: [install-dependencies]
+    plugin: dump
+    source: https://github.com/airbytehq/airbyte-platform.git # yamllint disable-line
+    source-type: git
+    source-tag: v0.63.8
+    override-build: |
+      cp -r . ${CRAFT_PART_INSTALL}/airbyte-platform
+    stage:
+      - airbyte-platform
+
+  assemble:
+    after: [pull-airbyte-repo]
+    plugin: nil
+    build-packages:
+      - jq
+      - curl
+      - coreutils
+      - bash
+      - gradle
+      - openjdk-21-jdk-headless
+      - npm
+    build-snaps:
+      - docker
+    stage-packages:
+      - openjdk-21-jdk-headless
+    override-build: |
+      cd ${CRAFT_STAGE}/airbyte-platform
+      ./gradlew assemble -x dockerBuildImage --continue --max-workers 1
+      ./gradlew --stop
+
+  organize-tars:
+    after: [assemble]
+    plugin: nil
+    override-build: |
+      mkdir ${CRAFT_PART_INSTALL}/airbyte-server
+      mkdir ${CRAFT_PART_INSTALL}/airbyte-api-server
+      mkdir ${CRAFT_PART_INSTALL}/airbyte-workers
+      mkdir ${CRAFT_PART_INSTALL}/airbyte-bootloader
+      mkdir ${CRAFT_PART_INSTALL}/airbyte-cron
+      mkdir ${CRAFT_PART_INSTALL}/airbyte-connector-builder-server
+
+      tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-server/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-server
+      tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-api-server/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-api-server
+      tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-workers/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-workers
+      tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-bootloader/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-bootloader
+      tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-cron/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-cron
+      tar -xvf ${CRAFT_STAGE}/airbyte-platform/airbyte-connector-builder-server/build/distributions/airbyte-app.tar -C ${CRAFT_PART_INSTALL}/airbyte-connector-builder-server
+    stage:
+      - airbyte-server
+      - airbyte-api-server
+      - airbyte-workers
+      - airbyte-bootloader
+      - airbyte-cron
+      - airbyte-connector-builder-server
+
+  local-files:
+    after: [organize-tars]
+    plugin: dump
+    source: ./local-files
+    organize:
+      pod-sweeper.sh: airbyte-pod-sweeper/airbyte-app/bin/airbyte-pod-sweeper
+    stage:
+      - airbyte-pod-sweeper/airbyte-app/bin/airbyte-pod-sweeper

charmcraft.yaml

@@ -429,47 +429,23 @@ config:
 # Your workload’s containers.
 containers:
   airbyte-api-server:
-    resource: airbyte-api-server
+    resource: airbyte-image
   airbyte-bootloader:
-    resource: airbyte-bootloader
+    resource: airbyte-image
   airbyte-connector-builder-server:
-    resource: airbyte-connector-builder-server
+    resource: airbyte-image
   airbyte-cron:
-    resource: airbyte-cron
+    resource: airbyte-image
   airbyte-pod-sweeper:
-    resource: airbyte-pod-sweeper
+    resource: airbyte-image
   airbyte-server:
-    resource: airbyte-server
+    resource: airbyte-image
   airbyte-workers:
-    resource: airbyte-workers
+    resource: airbyte-image
 
 # This field populates the Resources tab on Charmhub.
 resources:
-  airbyte-api-server:
-    type: oci-image
-    description: OCI image for Airbyte API server
-    upstream-source: airbyte/airbyte-api-server:0.60.0
-  airbyte-bootloader:
-    type: oci-image
-    description: OCI image for Airbyte Bootloader
-    upstream-source: airbyte/bootloader:0.60.0
-  airbyte-connector-builder-server:
-    type: oci-image
-    description: OCI image for Airbyte Connector Builder Server
-    upstream-source: airbyte/connector-builder-server:0.60.0
-  airbyte-cron:
+  airbyte-image:
     type: oci-image
-    description: OCI image for Airbyte Cron
-    upstream-source: airbyte/cron:0.60.0
-  airbyte-pod-sweeper:
-    type: oci-image
-    description: OCI image for Airbyte Pod Sweeper
-    upstream-source: bitnami/kubectl:1.29.4
-  airbyte-server:
-    type: oci-image
-    description: OCI image for Airbyte Server
-    upstream-source: airbyte/server:0.60.0
-  airbyte-workers:
-    type: oci-image
-    description: OCI image for Airbyte Worker
-    upstream-source: airbyte/worker:0.60.0
+    description: OCI image for Airbyte
+

src/charm.py

@@ -5,7 +5,6 @@
 """Charm the application."""
 
 import logging
-from pathlib import Path
 
 import ops
 from botocore.exceptions import ClientError
@@ -53,7 +52,7 @@ def get_pebble_layer(application_name, context):
         "services": {
             application_name: {
                 "summary": application_name,
-                "command": f"/bin/bash -c airbyte-app/bin/{application_name}",
+                "command": f"/bin/bash {application_name}/airbyte-app/bin/{application_name}",
                 "startup": "enabled",
                 "override": "replace",
                 # Including config values here so that a change in the
@@ -293,18 +292,6 @@ def _update(self, event):
                 event.defer()
                 return
 
-            if container_name == "airbyte-pod-sweeper":
-                script_path = Path(__file__).parent / "scripts/pod-sweeper.sh"
-
-                with open(script_path, "r") as file_source:
-                    logger.info("pushing pod-sweeper script...")
-                    container.push(
-                        f"/airbyte-app/bin/{container_name}",
-                        file_source,
-                        make_dirs=True,
-                        permissions=0o755,
-                    )
-
             env = create_env(self.model.name, self.app.name, container_name, self.config, self._state)
             env = {k: v for k, v in env.items() if v is not None}
             pebble_layer = get_pebble_layer(container_name, env)

src/charm_helpers.py

@@ -131,13 +131,12 @@ def create_env(model_name, app_name, container_name, config, state):
         "RUNNING_TTL_MINUTES": config["pod-running-ttl-minutes"],
         "SUCCEEDED_TTL_MINUTES": config["pod-successful-ttl-minutes"],
         "UNSUCCESSFUL_TTL_MINUTES": config["pod-unsuccessful-ttl-minutes"],
-        "INTERNAL_API_HOST": f"{app_name}:{INTERNAL_API_PORT}",
+        "INTERNAL_API_HOST": f"http://{app_name}:{INTERNAL_API_PORT}",
         "AIRBYTE_SERVER_HOST": f"{app_name}:{INTERNAL_API_PORT}",
         "CONFIG_API_HOST": f"{app_name}:{INTERNAL_API_PORT}",
         "CONNECTOR_BUILDER_SERVER_API_HOST": f"{app_name}:{CONNECTOR_BUILDER_SERVER_API_PORT}",
         "CONNECTOR_BUILDER_API_HOST": f"{app_name}:{CONNECTOR_BUILDER_SERVER_API_PORT}",
         "AIRBYTE_API_HOST": f"{app_name}:{AIRBYTE_API_PORT}/api/public",
-        "AIRBYTE_URL": config["webapp-url"],
     }
 
     # https://github.com/airbytehq/airbyte/issues/29506#issuecomment-1775148609

src/literals.py

@@ -7,6 +7,9 @@
 INTERNAL_API_PORT = 8001
 AIRBYTE_API_PORT = 8006
 WORKLOAD_API_PORT = 8007
+AIRBYTE_VERSION = "0.63.8"
+DB_NAME = "airbyte-k8s_db"
+
 
 CONTAINER_HEALTH_CHECK_MAP = {
     "airbyte-api-server": {
@@ -27,9 +30,7 @@
     "airbyte-workers": {"port": 9000, "health_endpoint": "/"},
 }
 
-DB_NAME = "airbyte-k8s_db"
 REQUIRED_S3_PARAMETERS = ["region", "endpoint", "access-key", "secret-key"]
-AIRBYTE_VERSION = "0.60.0"
 BUCKET_CONFIGS = [
     "storage-bucket-logs",
     "storage-bucket-state",
@@ -59,6 +60,6 @@
     "CONNECTOR_BUILDER_API_URL": "/connector-builder-api",
     "TEMPORAL_WORKER_PORTS": "9001,9002,9003,9004,9005,9006,9007,9008,9009,9010,9011,9012,9013,9014,9015,9016,9017,9018,9019,9020,9021,9022,9023,9024,9025,9026,9027,9028,9029,9030",
     "CONTAINER_ORCHESTRATOR_ENABLED": "true",
-    "CONTAINER_ORCHESTRATOR_IMAGE": "airbyte/container-orchestrator:0.60.0",
+    "CONTAINER_ORCHESTRATOR_IMAGE": f"airbyte/container-orchestrator:{AIRBYTE_VERSION}",
     "LOG4J_CONFIGURATION_FILE": "log4j2-minio.xml",
 }

tests/conftest.py

@@ -14,3 +14,5 @@ def pytest_addoption(parser: pytest.Parser):
     """
     # The prebuilt charm file.
     parser.addoption("--charm-file", action="append", default=[])
+    # The charm image name:tag.
+    parser.addoption("--airbyte-image", action="store", default="")

tests/integration/conftest.py

@@ -4,30 +4,50 @@
 """Charm integration test config."""
 
 import logging
+from pathlib import Path
 
+import pytest
 import pytest_asyncio
 from helpers import (
     APP_NAME_AIRBYTE_SERVER,
     APP_NAME_TEMPORAL_ADMIN,
     APP_NAME_TEMPORAL_SERVER,
     create_default_namespace,
-    get_airbyte_charm_resources,
     perform_airbyte_integrations,
     perform_temporal_integrations,
     run_sample_workflow,
 )
+from pytest import FixtureRequest
 from pytest_operator.plugin import OpsTest
 
 logger = logging.getLogger(__name__)
 
 
+@pytest.fixture(scope="module", name="charm_image")
+def charm_image_fixture(request: FixtureRequest) -> str:
+    """The OCI image for charm."""
+    charm_image = request.config.getoption("--airbyte-image")
+    assert charm_image, "--airbyte-image argument is required which should contain the name of the OCI image."
+    return charm_image
+
+
+@pytest_asyncio.fixture(scope="module", name="charm")
+async def charm_fixture(request: FixtureRequest, ops_test: OpsTest) -> str | Path:
+    """Fetch the path to charm."""
+    charms = request.config.getoption("--charm-file")
+    if not charms:
+        charm = await ops_test.build_charm(".")
+        assert charm, "Charm not built"
+        return charm
+    return charms[0]
+
+
 @pytest_asyncio.fixture(name="deploy", scope="module")
-async def deploy(ops_test: OpsTest):
+async def deploy(ops_test: OpsTest, charm: str, charm_image: str):
     """Test the app is up and running."""
     await ops_test.model.set_config({"update-status-hook-interval": "1m"})
-
-    charm = await ops_test.build_charm(".")
-    resources = get_airbyte_charm_resources()
+    # resources = get_airbyte_charm_resources()
+    resources = {"airbyte-image": charm_image}
 
     await ops_test.model.deploy(charm, resources=resources, application_name=APP_NAME_AIRBYTE_SERVER, trust=True)
     await ops_test.model.deploy(