From 3d9d9ef2e9bbf41143b217ff02d96c99d8259fe4 Mon Sep 17 00:00:00 2001 From: Sorin Sbarnea Date: Wed, 17 Jan 2024 17:39:11 +0000 Subject: [PATCH] Refactor taskfile - avoid parametrized recursive call of build step --- .github/workflows/task.yml | 6 ++--- .gitignore | 2 ++ Containerfile | 5 ++-- Taskfile.yml | 47 +++++++++++++++++++++++++++----------- _build/test-setup.sh | 5 +++- _build/validate | 16 ++++++------- base/Containerfile | 2 ++ 7 files changed, 56 insertions(+), 27 deletions(-) diff --git a/.github/workflows/task.yml b/.github/workflows/task.yml index 8e306d6..90170dc 100644 --- a/.github/workflows/task.yml +++ b/.github/workflows/task.yml @@ -47,12 +47,12 @@ jobs: cache: "pip" - name: Set cache for galaxy - uses: actions/cache@v3 - if: "contains(matrix.command, 'build')" + uses: actions/cache@v4 with: path: | collections key: galaxy-${{ hashFiles('_build/requirements.yml') }} + save-always: true - uses: dorny/paths-filter@v2 id: changes @@ -62,7 +62,7 @@ jobs: - 'base/**' - name: "Run: task base" - if: steps.changes.outputs.base == 'true' + if: steps.changes.outputs.base == 'true' && matrix.command == 'build' run: task base - name: "Run: task ${{ matrix.command }}" diff --git a/.gitignore b/.gitignore index d59d127..f77c010 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ .task out collections +.creator-base.done +.creator-ee.done diff --git a/Containerfile b/Containerfile index 7dca960..51d1cca 100644 --- a/Containerfile +++ b/Containerfile @@ -1,8 +1,9 @@ # Overly simplified single stage build process: we take all binary dependencies # using dnf and use pip to install the rest. -# this arg must be declared before FROM -ARG EE_BASE_IMAGE=quay.io/ansible/creator-base:latest +# this arg must be declared before FROM, also do not include registry part as +# it seems to confuse podman and make it avoid using locally build base image. +ARG EE_BASE_IMAGE=creator-base:latest FROM $EE_BASE_IMAGE # this arg must be declared after FROM ARG CONTAINER_NAME diff --git a/Taskfile.yml b/Taskfile.yml index 67e433f..392eb08 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -29,9 +29,11 @@ tasks: - echo Done {{.VERSION}}! base: desc: Build base image + deps: + - setup sources: - base/* - - .task/.{{.CNT_NAME}}.done + - .{{.CNT_NAME}}.done vars: &base-vars VERSION: "latest" CNT_NAME: creator-base @@ -40,24 +42,44 @@ tasks: CNT_ROOT: base/ RELEASE_PRECHECK: "true" cmds: - - task: build - vars: *base-vars + - podman container prune -f + - podman manifest exists {{.CNT_NAME_TAG}} && podman manifest rm {{.CNT_NAME_TAG}} || true + - podman image exists {{.CNT_NAME_TAG}} && podman image rm -f {{.CNT_NAME_TAG}} || true + - podman buildx build ${EXTRA_OPTS:---platform=linux/amd64,linux/arm64} --load {{.CNT_ROOT}} --manifest localhost/{{.CNT_NAME_TAG}} + # running manifest exists is mandatory as this fails if no manifest is + # created locally. If this is skipped the inspect might pull the last + # published manifest instead of using the local one. + - podman manifest exists {{.CNT_NAME_TAG}} + # Fail if the manifest does not contain exactly two images + # Validate image + - ./_build/validate {{.CNT_NAME_TAG}} + # Without next step, podman will pull the base image from quay.io instead + # of using the locally built image when we chain: task base && task build + - podman tag localhost/{{.CNT_NAME_TAG}} quay.io/ansible/{{.CNT_NAME}}:latest + - touch .{{.CNT_NAME}}.done + - echo "=== {{.CNT_NAME_TAG}} is ready! ===" interactive: true clean: desc: Clean up all files that not tracked by git cmds: - git clean -dxf + # will clean cached images to ensure podman will rebuild containers instead of using a cached result + - podman image prune -f setup: desc: Install dependencies env: # used inside test-setup.sh OS: "{{OS}}" ARCH: "{{ARCH}}" + # needed to avoid warning about installing from galaxy + ANSIBLE_COLLECTIONS_PATH: collections cmds: - bash ./_build/test-setup.sh + - ansible-galaxy collection install -r _build/requirements.yml -p collections sources: - - _build/test-setup.sh - Taskfile.yml + - _build/requirements.yml + - _build/test-setup.sh generates: - out/log/manifest.yml run: once @@ -66,14 +88,13 @@ tasks: desc: Build the project deps: - setup - env: - # needed to avoid warning about installing from galaxy - ANSIBLE_COLLECTIONS_PATH: collections + - base cmds: - - ansible-galaxy collection install -r _build/requirements.yml -p collections + - podman container prune -f - podman manifest exists {{.CNT_NAME_TAG}} && podman manifest rm {{.CNT_NAME_TAG}} || true - - podman image exists {{.CNT_NAME_TAG}} && podman image rm {{.CNT_NAME_TAG}} || true - - podman buildx build ${EXTRA_OPTS:---platform=linux/amd64,linux/arm64} --build-arg=CONTAINER_NAME={{.CNT_NAME_TAG}} --load {{.CNT_ROOT}} --manifest {{.CNT_NAME_TAG}} + - podman image exists {{.CNT_NAME_TAG}} && podman image rm -f {{.CNT_NAME_TAG}} || true + # --pull=missing is needed or podman will fail to use the locally built base image and pull the one from the registry instead + - podman buildx build --pull=missing ${EXTRA_OPTS:---platform=linux/amd64,linux/arm64} --build-arg=CONTAINER_NAME={{.CNT_NAME_TAG}} --load {{.CNT_ROOT}} --manifest {{.CNT_NAME_TAG}} # running manifest exists is mandatory as this fails if no manifest is # created locally. If this is skipped the inspect might pull the last # published manifest instead of using the local one. @@ -84,7 +105,7 @@ tasks: # Without next step, podman will pull the base image from quay.io instead # of using the locally built image when we chain: task base && task build - podman tag localhost/{{.CNT_NAME_TAG}} quay.io/ansible/{{.CNT_NAME}}:latest - - touch .task/.{{.CNT_NAME}}.done + - touch .{{.CNT_NAME}}.done - echo "=== {{.CNT_NAME_TAG}} is ready! ===" sources: - Taskfile.yml @@ -92,14 +113,14 @@ tasks: - _build/* - Containerfile generates: - - .task/.{{.CNT_NAME}}.done + - .{{.CNT_NAME}}.done interactive: true exec: desc: Open a container shell deps: - build cmds: - - podman run --rm -it {{.CNT_NAME_TAG}} /bin/bash + - podman run --arch=$(arch) --rm -it {{.CNT_NAME_TAG}} /bin/bash sources: - Taskfile.yml interactive: true diff --git a/_build/test-setup.sh b/_build/test-setup.sh index 463f010..a86545a 100755 --- a/_build/test-setup.sh +++ b/_build/test-setup.sh @@ -91,7 +91,7 @@ if [[ "${OS:-}" == "darwin" && "${SKIP_PODMAN:-}" != '1' ]]; then time podman machine init time podman machine start podman info - podman run hello-world + podman run --arch="$(arch)" --rm hello-world } fi @@ -148,6 +148,9 @@ if [[ "${PODMAN_VERSION}" != 'null' ]] && [[ "${SKIP_PODMAN:-}" != '1' ]]; then echo . fi fi +# verify podman ability to execute multi-arch commands: +podman run --arch=arm64 -qit bash arch +podman run --arch=amd64 -qit bash arch # Create a build manifest so we can compare between builds and machines, this # also has the role of ensuring that the required executables are present. diff --git a/_build/validate b/_build/validate index a6d4916..af78c58 100755 --- a/_build/validate +++ b/_build/validate @@ -7,24 +7,24 @@ CNT=${1:-quay.io/ansible/creator-ee:latest} # creator-base container is expected to have python3 if [[ "$CNT" == *"creator-base"* ]]; then - podman run -i "$CNT" python3 --version + podman run --arch="$(arch)" --rm -i "$CNT" python3 --version exit 0 fi # Do not use `-t` (interactive) with podman, especially inside scripts/automation as it is # likely to cause problems. -podman run -i --user="$(id -u)" -e EP_DEBUG=1 "$CNT" uname -a -podman run -i --user="$(id -u)" "$CNT" uname -a -podman run -i --user="$(id -u)" "$CNT" bash -c 'set | grep PATH' -podman run -i --user="$(id -u)" "$CNT" command -v ansible-lint +podman run --arch="$(arch)" --rm -i --user="$(id -u)" -e EP_DEBUG=1 "$CNT" uname -a +podman run --arch="$(arch)" --rm -i --user="$(id -u)" "$CNT" uname -a +podman run --arch="$(arch)" --rm -i --user="$(id -u)" "$CNT" bash -c 'set | grep PATH' +podman run --arch="$(arch)" --rm -i --user="$(id -u)" "$CNT" command -v ansible-lint # ensure that git is configured to consider any directory safe: -podman run -i --user="$(id -u)" "$CNT" bash -c '[[ $(git config safe.directory) == "/" ]]' +podman run --arch="$(arch)" --rm -i --user="$(id -u)" "$CNT" bash -c '[[ $(git config safe.directory) == "/" ]]' # Ensure some packages required are present # navigator requires the rpm db be intact as it runs rpm -qa -podman run -i --user="$(id -u)" "$CNT" bash -c 'rpm -qa | grep python' +podman run --arch="$(arch)" --rm -i --user="$(id -u)" "$CNT" bash -c 'rpm -qa | grep python' # ansible-runner is needed in all execution environment images -podman run -i --user="$(id -u)" "$CNT" ansible-runner --version +podman run --arch="$(arch)" --rm -i --user="$(id -u)" "$CNT" ansible-runner --version # ensure that the image is build for both supported architectures and the manifest is correct podman manifest inspect "$CNT" | jq '.manifests | length' | grep -q "${EXPECTED_IMAGES:-2}" diff --git a/base/Containerfile b/base/Containerfile index a77957f..cabfe61 100644 --- a/base/Containerfile +++ b/base/Containerfile @@ -25,6 +25,8 @@ podman \ python3 \ python3-bcrypt \ python3-cffi \ +# ansible-pylibssh needs: +python3-devel \ python3-markupsafe \ # pypi has binaries with better security: # python3-cryptography \