ci(Makefile): optimise spin-up/tear-down flow (#356)

Because - `make down` won't go smoothly from a `make all` instance due to missing `release` compose images - the current 3rd-party `uuidgen` image doesn't support arm64 This commit - replace the Ubuntu image with Apline to save ~300MB for the compose image - add related `release` compose images in `make down` - add `make down` step in `make-all/latest` GA workflow to validate the complete flow - move `uuidgen` into compose image so we don't need to use an external image - make `~/.config/instill` folder logic and env variable `SYSTEM_CONFIG_PATH` in only the base project
instill-ai · Sep 19, 2023 · 6e5a915 · 6e5a915
1 parent f23ad0e
commit 6e5a915
Show file tree

Hide file tree

Showing 6 changed files with 121 additions and 216 deletions.
diff --git a/.env b/.env
@@ -1,7 +1,7 @@
 # docker compose project name
 COMPOSE_PROJECT_NAME=instill-vdp
 
-# system-wise config path
+# system-wise config path (all base, vdp, and model projects must use the same path)
 SYSTEM_CONFIG_PATH=~/.config/instill
 
 # configuration directory path for docker build
@@ -24,7 +24,6 @@ DOCKER_BUILDKIT=1
 COMPOSE_DOCKER_CLI_BUILD=1
 
 # version
-UBUNTU_VERSION=20.04
 ALPINE_VERSION=3.16
 
 GOLANG_VERSION=1.19.3
@@ -33,7 +32,7 @@ ARTIVC_VERSION=0.10.0
 K6_VERSION=0.44.0
 
 # Instill Base
-BASE_VERSION=0.3.0-alpha
+INSTILL_BASE_VERSION=0.3.0-alpha
 
 # api-gateway
 API_GATEWAY_HOST=api-gateway

diff --git a/.github/workflows/make-all.yml b/.github/workflows/make-all.yml
@@ -1,7 +1,11 @@
-name: Make all
+name: Make All
 
 on:
   workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - main
 
 jobs:
   make-all:
@@ -35,24 +39,6 @@ jobs:
           rm --recursive --force "$AGENT_TOOLSDIRECTORY"
           df --human-readable
 
-      - name: Checkout repo (base)
-        uses: actions/checkout@v3
-        with:
-          repository: instill-ai/base
-
-      - name: Load .env file (base)
-        uses: cardinalby/export-env-action@v2
-        with:
-          envFile: .env
-
-      - name: Launch Instill Base (release)
-        run: |
-          EDITION=local-ce \
-          DEFAULT_USER_UID=${DEFAULT_USER_UID} \
-          docker compose -f docker-compose.yml up -d --quiet-pull
-          EDITION=local-ce \
-          docker compose -f docker-compose.yml rm -f
-
       - name: Checkout repo (vdp)
         uses: actions/checkout@v3
         with:
@@ -65,24 +51,19 @@ jobs:
 
       - name: Launch Instill VDP (release)
         run: |
-          EDITION=local-ce \
-          docker compose -f docker-compose.yml up -d --quiet-pull
-          EDITION=local-ce \
-          docker compose -f docker-compose.yml rm -f
+          make all EDITION=local-ce:test
 
       - name: List all docker containers
         run: |
           docker ps -a
           sleep 60
 
-      - name: Curl to base (influxdb, temporal and console) healthcheck endpoint
-        run: |
-          curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:8086/health
-          curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:8088/health
-          curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:3000
-
-      - name: Curl to vdp services healthcheck endpoint
+      - name: Probe to vdp services healthcheck endpoint
         run: |
           curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:8080/base/v1alpha/health/mgmt
           curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:8080/vdp/v1alpha/health/pipeline
           curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:8080/vdp/v1alpha/health/connector
+
+      - name: Tear down Instill VDP (release)
+        run: |
+          make down
diff --git a/.github/workflows/make-latest.yml b/.github/workflows/make-latest.yml
@@ -1,7 +1,11 @@
-name: Make latest
+name: Make Latest
 
 on:
   workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - main
 
 jobs:
   make-latest:
@@ -35,27 +39,6 @@ jobs:
           rm --recursive --force "$AGENT_TOOLSDIRECTORY"
           df --human-readable
 
-      - name: Checkout repo (base)
-        uses: actions/checkout@v3
-        with:
-          repository: instill-ai/base
-
-      - name: Load .env file (base)
-        uses: cardinalby/export-env-action@v2
-        with:
-          envFile: .env
-
-      - name: Launch Instill Base (latest)
-        run: |
-          echo ${DEFAULT_USER_UID}
-          COMPOSE_PROFILES=all \
-          DEFAULT_USER_UID=${DEFAULT_USER_UID} \
-          EDITION=local-ce:latest \
-          docker compose -f docker-compose.yml -f docker-compose.latest.yml up -d --quiet-pull
-          COMPOSE_PROFILES=all \
-          EDITION=local-ce:latest \
-          docker compose -f docker-compose.yml -f docker-compose.latest.yml rm -f
-
       - name: Checkout repo (vdp)
         uses: actions/checkout@v3
         with:
@@ -68,28 +51,20 @@ jobs:
 
       - name: Launch Instill VDP (latest)
         run: |
-          COMPOSE_PROFILES=all \
-          EDITION=local-ce:latest \
-          docker compose -f docker-compose.yml -f docker-compose.latest.yml up -d --quiet-pull
-          COMPOSE_PROFILES=all \
-          EDITION=local-ce:latest \
-          docker compose -f docker-compose.yml -f docker-compose.latest.yml rm -f
+          make latest PROFILE=all EDITION=local-ce:test
 
       - name: List all docker containers
         run: |
           docker ps -a
           sleep 30
 
-      - name: Curl to base (influxdb, temporal, console and etcd) healthcheck endpoint
-        run: |
-          curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:8086/health
-          curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:8088/health
-          curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:3000
-          curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:3379/health
-
       - name: Curl to vdp services healthcheck endpoint
         run: |
           curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:8080/base/v1alpha/health/mgmt
           curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:8080/vdp/v1alpha/health/pipeline
           curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:8080/vdp/v1alpha/health/connector
           curl -s -o /dev/null -w ''%{http_code}'\n' http://localhost:3085/v1alpha/health/controller
+
+      - name: Tear down Instill VDP (latest)
+        run: |
+          make down
diff --git a/Dockerfile b/Dockerfile
@@ -1,29 +1,7 @@
-ARG UBUNTU_VERSION
-FROM ubuntu:${UBUNTU_VERSION} AS base
-
-ARG DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get -y install \
-    ca-certificates \
-    curl \
-    gnupg \
-    lsb-release
-
-RUN mkdir -p /etc/apt/keyrings
-RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
-RUN echo \
-    "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
-    $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
-
-RUN apt-get update && apt-get install -y \
-    docker-ce \
-    docker-ce-cli \
-    containerd.io \
-    docker-compose-plugin \
-    git \
-    bash \
-    make \
-    wget \
-    vim && rm -rf /var/lib/apt/lists/*
+ARG ALPINE_VERSION
+FROM alpine:${ALPINE_VERSION} AS base
+
+RUN apk add --update docker docker-compose docker-cli-compose docker-cli-buildx openrc containerd git bash make wget vim curl openssl
 
 # Install k6
 ARG TARGETARCH K6_VERSION
@@ -38,10 +16,11 @@ RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s
 RUN chmod +x ./kubectl
 RUN mv ./kubectl /usr/local/bin
 
-FROM ubuntu:${UBUNTU_VERSION} AS latest
+FROM alpine:${ALPINE_VERSION} AS latest
 
 COPY --from=base /etc /etc
 COPY --from=base /usr /usr
+COPY --from=base /lib /lib
 COPY --from=docker:dind /usr/local/bin /usr/local/bin
 
 ARG CACHE_DATE
@@ -57,19 +36,20 @@ RUN git clone https://github.com/instill-ai/pipeline-backend.git
 RUN git clone https://github.com/instill-ai/connector-backend.git
 RUN git clone https://github.com/instill-ai/controller-vdp.git
 
-FROM ubuntu:${UBUNTU_VERSION} AS release
+FROM alpine:${ALPINE_VERSION} AS release
 
 COPY --from=base /etc /etc
 COPY --from=base /usr /usr
+COPY --from=base /lib /lib
 COPY --from=docker:dind /usr/local/bin /usr/local/bin
 
 ARG CACHE_DATE
 RUN echo "VDP release codebase cloned on ${CACHE_DATE}"
 
 WORKDIR /instill-ai
 
-ARG BASE_VERSION
-RUN git clone -b v${BASE_VERSION} -c advice.detachedHead=false https://github.com/instill-ai/base.git
+ARG INSTILL_BASE_VERSION
+RUN git clone -b v${INSTILL_BASE_VERSION} -c advice.detachedHead=false https://github.com/instill-ai/base.git
 
 WORKDIR /instill-ai/vdp