add docker_kit

.gitattributes

@@ -0,0 +1,25 @@
+# Ignore all differences in line endings
+*               text=auto eol=lf
+*.md            text eol=lf
+*.py            text eol=lf
+*.sh            text eol=lf
+*.yml           text eol=lf
+*.yaml          text eol=lf
+*.json          text eol=lf
+*.properties    text eol=lf
+*.conf          text eol=lf
+*.ipynb         text eol=lf
+Dockerfile*     text eol=lf
+.gitattributes  text eol=lf
+.gitignore      text eol=lf
+.dockerignore   text eol=lf
+
+# Files using LFS to track
+*.tgz   filter=lfs diff=lfs merge=lfs -text
+*.h5    filter=lfs diff=lfs merge=lfs -text
+*.jsonl filter=lfs diff=lfs merge=lfs -text
+*.xlsx  filter=lfs diff=lfs merge=lfs -text
+*.bin   filter=lfs diff=lfs merge=lfs -text
+*.png   filter=lfs diff=lfs merge=lfs -text
+*.jpg   filter=lfs diff=lfs merge=lfs -text
+*.jpeg  filter=lfs diff=lfs merge=lfs -text

.github/workflows/build-docker.yml

@@ -0,0 +1,47 @@
+name: build-docker-images
+
+on:
+  push:
+    branches: [ main ]
+    paths-ignore:
+      - "*.md"
+
+  pull_request:
+    branches: [ main ]
+    paths-ignore:
+      - "*.md"
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+env:
+  REGISTRY_URL: "docker.io"   # docker.io or other registry URL, DOCKER_REGISTRY_USER/DOCKER_REGISTRY_PASSWORD to be set in CI env.
+  BUILDKIT_PROGRESS: "plain"  # Full logs for CI build.
+
+  # DOCKER_REGISTRY_USER and DOCKER_REGISTRY_PASSWORD is required for docker image push, they should be set in CI secrets.
+  DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}
+  DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }}
+
+  # used to sync image to mirror registry
+  DOCKER_MIRROR_REGISTRY_USERNAME: ${{ secrets.DOCKER_MIRROR_REGISTRY_USERNAME }}
+  DOCKER_MIRROR_REGISTRY_PASSWORD: ${{ secrets.DOCKER_MIRROR_REGISTRY_PASSWORD }}
+
+jobs:
+  docker_kit:
+    name: 'docker-kit'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: |
+          source ./tool.sh && build_image docker-kit latest docker_docker_kit/Dockerfile && push_image
+
+  sync_images:
+    needs: ['docker_kit']
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: |
+          source ./tool.sh
+          setup_docker_syncer
+          echo "Syncing image to mirror registry..."
+          python docker_docker_kit/work/image-syncer/run_jobs.py

docker_docker_kit/Dockerfile

@@ -0,0 +1,17 @@
+# Distributed under the terms of the Modified BSD License.
+
+ARG BASE_NAMESPACE
+ARG BASE_IMG="base"
+FROM ${BASE_NAMESPACE:+$BASE_NAMESPACE/}${BASE_IMG}
+
+LABEL maintainer="[email protected]"
+
+COPY work /opt/utils/
+
+RUN source /opt/utils/script-setup.sh \
+ && setup_docker_compose && setup_docker_syncer \
+ && pip install -U PyYaml \
+ && ln -sf /usr/bin/image-syncer /opt/utils/image-syncer/
+
+# Clean up and display components version information...
+RUN source /opt/utils/script-utils.sh && install__clean && list_installed_packages

docker_docker_kit/work/image-syncer/run_jobs.py

@@ -0,0 +1,50 @@
+import os
+import json
+import sys
+import yaml
+
+import run_sync
+
+
+def get_job_names_from_yaml(file_path):
+    """Get all job names from GitHub Actions config file"""
+    with open(file_path, 'r') as file:
+        try:
+            yaml_content = yaml.safe_load(file)
+            # GitHub Actions YAML file structure has a 'jobs' key at its root
+            jobs = yaml_content.get('jobs', {})
+            for _, v in jobs.items():
+                name = v.get('name')
+                if name is not None:
+                    yield name
+        except yaml.YAMLError as exc:
+            print(f"Error parsing YAML file: {exc}")
+            return []
+
+
+def main():
+    namespace = os.environ.get('IMG_NAMESPACE')
+    if namespace is None:
+        print('Using default IMG_NAMESPACE=library !')
+        namespace = 'library'
+
+    images = []
+    job_names = get_job_names_from_yaml('.github/workflows/build-docker.yml')
+    for name in job_names:
+        images.extend(name.split(','))
+
+    for image in images:
+        img = '/'.join([namespace, image])
+        print("Docker image sync job name found:", img)
+        configs = run_sync.generate(image=img, tags=None)
+        for _, c in enumerate(configs):
+            s_config = json.dumps(c, ensure_ascii=False, sort_keys=True)
+            print('Config item:', json.dumps(c, ensure_ascii=False, sort_keys=True))
+            ret = run_sync.sync_image(cfg=c)
+            if ret != 0:
+                return ret
+    return ret
+
+
+if __name__ == '__main__':
+    sys.exit(main())

docker_docker_kit/work/image-syncer/run_sync.py

@@ -0,0 +1,64 @@
+import argparse
+import json
+import os
+import subprocess
+import sys
+import tempfile
+
+
+def generate(image: str, target_registries: list = None, tags: list = None, target_image: str = None):
+    """Generate a config item which will be used by `image-syncer`."""
+    uname = os.environ.get('DOCKER_MIRROR_REGISTRY_USERNAME', None)
+    passwd = os.environ.get('DOCKER_MIRROR_REGISTRY_PASSWORD', None)
+
+    if uname is None or passwd is None:
+        print('ENV variable required: DOCKER_MIRROR_REGISTRY_USERNAME and DOCKER_MIRROR_REGISTRY_PASSWORD !')
+        sys.exit(-2)
+
+    if target_registries is None:
+        # , 'cn-shanghai', 'cn-shenzhen', 'cn-chengdu', 'cn-hongkong', 'us-west-1', eu-central-1
+        destinations = ['cn-beijing', 'cn-hangzhou']
+        target_registries = ['registry.%s.aliyuncs.com' % i for i in destinations]
+
+    for dest in target_registries:
+        src = "%s:%s" % (image, tags) if tags is not None else image
+        yield {
+            'auth': {
+                dest: {"username": uname, "password": passwd}
+            },
+            'images': {
+                src: "%s/%s" % (dest, target_image or image)
+            }
+        }
+
+
+def sync_image(cfg: dict):
+    """Run the sync task using `image-syncer` with given config item."""
+    with tempfile.NamedTemporaryFile(mode='wt', encoding='UTF-8', suffix='.json') as fp:
+        json.dump(cfg, fp, ensure_ascii=False, indent=2, sort_keys=True)
+        fp.flush()
+        ret = 0
+        try:
+            subprocess.run(['image-syncer', '--proc=16', '--retries=2', '--config=' + fp.name], check=True)
+        except subprocess.CalledProcessError as e:
+            ret = e.returncode
+            print(e)
+    return ret
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('img', type=str, help='Source image, with or without tag')
+    parser.add_argument('--tags', type=str, action='extend', nargs='*', help='Tags to sync, optional.')
+    parser.add_argument('--dest-image', type=str, help='Target image name, with our without tag')
+    parser.add_argument('--dest-registry', type=str, action='extend', nargs='*', help='tTarget registry URL')
+    args = parser.parse_args()
+
+    configs = generate(image=args.img, tags=args.tags, target_registries=args.dest_registry, target_image=args.dest_image)
+    for _, c in enumerate(configs):
+        ret = sync_image(cfg=c)
+    return ret
+
+
+if __name__ == '__main__':
+    sys.exit(main())

tool.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+set -xu
+
+CI_PROJECT_NAME=${GITHUB_REPOSITORY:-"QPod/dev-lab"}
+CI_PROJECT_BRANCH=${GITHUB_HEAD_REF:-"main"}
+CI_PROJECT_SPACE=$(echo "${CI_PROJECT_BRANCH}" | cut -f1 -d'/')
+
+if [ "${CI_PROJECT_BRANCH}" = "main" ] ; then
+    # If on the main branch, docker images namespace will be same as CI_PROJECT_NAME's name space
+    export CI_PROJECT_NAMESPACE="$(dirname ${CI_PROJECT_NAME})" ;
+else
+    # not main branch, docker namespace = {CI_PROJECT_NAME's name space} + "0" + {1st substr before / in CI_PROJECT_SPACE}
+    export CI_PROJECT_NAMESPACE="$(dirname ${CI_PROJECT_NAME})0${CI_PROJECT_SPACE}" ;
+fi
+
+export IMG_NAMESPACE=$(echo "${CI_PROJECT_NAMESPACE}" | awk '{print tolower($0)}')
+export IMG_PREFIX=$(echo "${REGISTRY_URL:-"docker.io"}/${IMG_NAMESPACE}" | awk '{print tolower($0)}')
+
+echo "--------> CI_PROJECT_NAMESPACE=${CI_PROJECT_NAMESPACE}"
+echo "--------> DOCKER_IMG_NAMESPACE=${IMG_NAMESPACE}"
+echo "--------> DOCKER_IMG_PREFIX=${IMG_PREFIX}"
+
+
+if [ -f /etc/docker/daemon.json ]; then
+       jq '.experimental=true | ."data-root"="/mnt/docker"' /etc/docker/daemon.json > /tmp/daemon.json && sudo mv /tmp/daemon.json /etc/docker/ \
+    && ( sudo service docker restart || true )
+fi
+cat /etc/docker/daemon.json
+docker info
+
+build_image() {
+    echo "$@" ;
+    IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M); WORKDIR="$(dirname $FILE)";
+    docker build --compress --force-rm=true -t "${IMG_PREFIX}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${IMG_PREFIX}" "$@" "${WORKDIR}" ;
+    docker tag "${IMG_PREFIX}/${IMG}:${TAG}" "${IMG_PREFIX}/${IMG}:${VER}" ;
+}
+
+build_image_no_tag() {
+    echo "$@" ;
+    IMG=$1; TAG=$2; FILE=$3; shift 3; WORKDIR="$(dirname $FILE)";
+    docker build --compress --force-rm=true -t "${IMG_PREFIX}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${IMG_PREFIX}" "$@" "${WORKDIR}" ;
+}
+
+build_image_common() {
+    echo "$@" ;
+    IMG=$1; TAG=$2; FILE=$3; shift 3; VER=$(date +%Y.%m%d.%H%M); WORKDIR="$(dirname $FILE)";
+    docker build --compress --force-rm=true -t "${IMG_PREFIX}/${IMG}:${TAG}" -f "$FILE" --build-arg "BASE_NAMESPACE=${IMG_PREFIX}" "$@" "${WORKDIR}" ;
+    docker tag "${IMG_PREFIX}/${IMG}:${TAG}" "${IMG_PREFIX}/${IMG}:${VER}" ;
+}
+
+alias_image() {
+    IMG_1=$1; TAG_1=$2; IMG_2=$3; TAG_2=$4; shift 4; VER=$(date +%Y.%m%d.%H%M);
+    docker tag "${IMG_PREFIX}/${IMG_1}:${TAG_1}" "${IMG_PREFIX}/${IMG_2}:${TAG_2}" ;
+    docker tag "${IMG_PREFIX}/${IMG_2}:${TAG_2}" "${IMG_PREFIX}/${IMG_2}:${VER}" ;
+}
+
+push_image() {
+    KEYWORD="${1:-second}";
+    docker image prune --force && docker images | sort;
+    IMAGES=$(docker images | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ;
+    echo "$DOCKER_REGISTRY_PASSWORD" | docker login "${REGISTRY_URL}" -u "$DOCKER_REGISTRY_USER" --password-stdin ;
+    for IMG in $(echo "${IMAGES}" | tr " " "\n") ;
+    do
+      docker push "${IMG}";
+      status=$?;
+      echo "[${status}] Image pushed > ${IMG}";
+    done
+}
+
+clear_images() {
+    KEYWORD=${1:-'days ago\|weeks ago\|months ago\|years ago'}; # if no keyword is provided, clear all images build days ago
+    IMGS_1=$(docker images | grep "${KEYWORD}" | awk '{print $1 ":" $2}') ;
+    IMGS_2=$(docker images | grep "${KEYWORD}" | awk '{print $3}') ;
+
+    for IMG in $(echo "$IMGS_1 $IMGS_2" | tr " " "\n") ; do
+      docker rmi "${IMG}" || true; status=$?; echo "[${status}] image removed > ${IMG}";
+    done
+    docker image prune --force && docker images ;
+}
+
+setup_docker_syncer() {
+     ARCH="amd64" \
+  && SYNCER_VERSION="$(curl -sL https://github.com/AliyunContainerService/image-syncer/releases.atom | grep 'releases/tag' | head -1 | grep -Po '\d[.\d]+')" \
+  && SYNCER_URL="https://github.com/AliyunContainerService/image-syncer/releases/download/v${SYNCER_VERSION}/image-syncer-v${SYNCER_VERSION}-linux-${ARCH}.tar.gz" \
+  && echo "Downloading image-syncer from: ${SYNCER_URL}" \
+  && curl -o /tmp/image_syncer.tgz -sL ${SYNCER_URL} \
+  && mkdir -pv /tmp/image_syncer && tar -zxvf /tmp/image_syncer.tgz -C /tmp/image_syncer \
+  && sudo chmod +x /tmp/image_syncer/image-syncer \
+  && sudo mv /tmp/image_syncer/image-syncer /usr/bin/ \
+  && rm -rf /tmp/image_syncer* \
+  && echo "@ image-syncer installed to: $(which image-syncer)"
+}
+
+
+remove_folder() {
+    sudo du -h -d1 "$1" || true ;
+    sudo rm -rf "$1" || true ;
+}
+
+free_diskspace() {
+    remove_folder /usr/share/dotnet
+    remove_folder /usr/local/lib/android
+    # remove_folder /var/lib/docker
+    df -h
+}