Skip to content

Commit

Permalink
Merge pull request #3818 from cluibm/agent-install
Browse files Browse the repository at this point in the history
update cluster support for s390x
  • Loading branch information
LiilyZhang authored Jul 21, 2023
2 parents d5eee60 + a2a4ca7 commit ea438bd
Show file tree
Hide file tree
Showing 4 changed files with 83 additions and 9 deletions.
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -432,14 +432,14 @@ anax-k8s-image: anax-k8s-clean
cp $(CLI_EXECUTABLE) $(ANAX_K8S_CONTAINER_DIR)
cp -f $(LICENSE_FILE) $(ANAX_K8S_CONTAINER_DIR)
@echo "Producing ANAX K8S docker image $(ANAX_K8S_IMAGE_STG)"
if [[ $(arch) == "amd64" || $(arch) == "ppc64el" || $(arch) == "arm64" ]]; then \
if [[ $(arch) == "amd64" || $(arch) == "ppc64el" || $(arch) == "arm64" || $(arch) == "s390x" ]]; then \
cd $(ANAX_K8S_CONTAINER_DIR) && docker $(DOCKER_BUILD_CMD) $(DOCKER_MAYBE_CACHE) $(ANAX_K8S_IMAGE_LABELS) -t $(ANAX_K8S_IMAGE_STG) -f Dockerfile.ubi.$(arch) .; \
fi
docker tag $(ANAX_K8S_IMAGE_STG) $(ANAX_K8S_IMAGE_BASE):$(ANAX_K8S_IMAGE_VERSION)

auto-upgrade-cronjob-k8s-image: auto-upgrade-cronjob-k8s-clean
@echo "Producing Agent Auto Upgrade CronJob K8S docker image $(CRONJOB_AUTO_UPGRADE_K8S_IMAGE_STG)"
if [[ $(arch) == "amd64" || $(arch) == "ppc64el" || $(arch) == "arm64" ]]; then \
if [[ $(arch) == "amd64" || $(arch) == "ppc64el" || $(arch) == "arm64" || $(arch) == "s390x" ]]; then \
cd $(ANAX_K8S_CONTAINER_DIR) && docker build $(DOCKER_MAYBE_CACHE) $(CRONJOB_AUTO_UPGRADE_K8S_IMAGE_LABELS) -t $(CRONJOB_AUTO_UPGRADE_K8S_IMAGE_STG) -f Dockerfile.ubi.auto-upgrade-cron.$(arch) .; \
fi
docker tag $(CRONJOB_AUTO_UPGRADE_K8S_IMAGE_STG) $(CRONJOB_AUTO_UPGRADE_K8S_IMAGE_BASE):$(CRONJOB_AUTO_UPGRADE_K8S_IMAGE_VERSION)
Expand Down
17 changes: 10 additions & 7 deletions agent-install/agent-install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ SUPPORTED_REDHAT_VARIANTS=(rhel redhatenterprise centos fedora $SUPPORTED_REDHAT
SUPPORTED_REDHAT_VERSION=(7.6 7.9 8.1 8.2 8.3 8.4 8.5 8.6 8.7 8.8 9.0 9.1 9.2 8 9 32 35 36 37 38 $SUPPORTED_REDHAT_VERSION_APPEND) # compared to what our detect_distro() sets DISTRO_VERSION_NUM to. For fedora versions see https://fedoraproject.org/wiki/Releases,
SUPPORTED_REDHAT_ARCH=(x86_64 aarch64 ppc64le s390x riscv64 $SUPPORTED_REDHAT_ARCH_APPEND) # compared to uname -m

SUPPORTED_EDGE_CLUSTER_ARCH=(amd64)
SUPPORTED_EDGE_CLUSTER_ARCH=(amd64 s390x)
SUPPORTED_ANAX_IN_CONTAINER_ARCH=(amd64 arm64 s390x)

SUPPORTED_OS=(macos linux) # compared to what our get_os() returns
Expand All @@ -48,7 +48,7 @@ CURL_RETRY_PARMS="--retry 5 --retry-connrefused --retry-max-time 120"

SEMVER_REGEX='^[0-9]+\.[0-9]+(\.[0-9]+)+' # matches a version like 1.2.3 (must be at least 3 fields). Also allows a bld num on the end like: 1.2.3-RC1

# The following variable will need to have the $ARCH prepended to it before it can be used - currently only amd64 and arm64 are built
# The following variable will need to have the $ARCH prepended to it before it can be used
DEFAULT_AGENT_IMAGE_TAR_FILE='_anax.tar.gz'

INSTALLED_AGENT_CFG_FILE="/etc/default/horizon"
Expand All @@ -69,9 +69,10 @@ GET_RESOURCE_MAX_TRY=5
POD_ID=""
HZN_ENV_FILE="/tmp/agent-install-horizon-env"
DEFAULT_OCP_INTERNAL_URL_FOR_EDGE_CLUSTER_REGISTRY="image-registry.openshift-image-registry.svc:5000"
DEFAULT_AGENT_K8S_IMAGE_TAR_FILE='amd64_anax_k8s.tar.gz'
DEFAULT_CRONJOB_AUTO_UPGRADE_K8S_TAR_FILE='amd64_auto-upgrade-cronjob_k8s.tar.gz'
EDGE_CLUSTER_TAR_FILE_NAME='horizon-agent-edge-cluster-files.tar.gz'
# The following variables will need to have the $ARCH prepended before they can be used
DEFAULT_AGENT_K8S_IMAGE_TAR_FILE='_anax_k8s.tar.gz'
DEFAULT_CRONJOB_AUTO_UPGRADE_K8S_TAR_FILE='_auto-upgrade-cronjob_k8s.tar.gz'

# agent upgrade types. To update the certificate only, just do "-G cert" or set AGENT_UPGRADE_TYPES="cert"
UPGRADE_TYPE_SW="software"
Expand Down Expand Up @@ -134,7 +135,7 @@ Additional Variables (in environment or config file):
Additional Edge Device Variables (in environment or config file):
NODE_ID_MAPPING_FILE: File to map hostname or IP to node id, for bulk install. Default: node-id-mapping.csv
AGENT_IMAGE_TAR_FILE: the file name of the device agent docker image in tar.gz format. Default: $DEFAULT_AGENT_IMAGE_TAR_FILE
AGENT_IMAGE_TAR_FILE: the file name of the device agent docker image in tar.gz format. Default: \${ARCH}$DEFAULT_AGENT_IMAGE_TAR_FILE
AGENT_WAIT_MAX_SECONDS: Maximum seconds to wait for the Horizon agent to start or stop. Default: 30
Optional Edge Device Environment Variables For Testing New Distros - Not For Production Use
Expand All @@ -155,8 +156,8 @@ Additional Edge Cluster Variables (in environment or config file):
AGENT_NAMESPACE: The namespace the agent should run in. Default: openhorizon-agent
AGENT_WAIT_MAX_SECONDS: Maximum seconds to wait for the Horizon agent to start or stop. Default: 30
AGENT_DEPLOYMENT_STATUS_TIMEOUT_SECONDS: Maximum seconds to wait for the agent deployment rollout status to be successful. Default: 75
AGENT_K8S_IMAGE_TAR_FILE: the file name of the edge cluster agent docker image in tar.gz format. Default: $DEFAULT_AGENT_K8S_IMAGE_TAR_FILE
CRONJOB_AUTO_UPGRADE_K8S_TAR_FILE: the file name of the edge cluster auto-upgrade-cronjob cronjob docker image in tar.gz format. Default: $DEFAULT_CRONJOB_AUTO_UPGRADE_K8S_TAR_FILE
AGENT_K8S_IMAGE_TAR_FILE: the file name of the edge cluster agent docker image in tar.gz format. Default: \${ARCH}$DEFAULT_AGENT_K8S_IMAGE_TAR_FILE
CRONJOB_AUTO_UPGRADE_K8S_TAR_FILE: the file name of the edge cluster auto-upgrade-cronjob cronjob docker image in tar.gz format. Default: \${ARCH}$DEFAULT_CRONJOB_AUTO_UPGRADE_K8S_TAR_FILE
AGENT_NAMESPACE: The cluster namespace that the agent will be installed in
NAMESPACE_SCOPED: specify this value if the edge cluster agent is namespace-scoped agent
EndOfMessage
Expand Down Expand Up @@ -1181,6 +1182,8 @@ function get_all_variables() {

local image_arch=$(get_cluster_image_arch)
check_support "${SUPPORTED_EDGE_CLUSTER_ARCH[*]}" "${image_arch}" 'kubernetes edge cluster architectures'
DEFAULT_AGENT_K8S_IMAGE_TAR_FILE=${image_arch}${DEFAULT_AGENT_K8S_IMAGE_TAR_FILE}
DEFAULT_CRONJOB_AUTO_UPGRADE_K8S_TAR_FILE=${image_arch}${DEFAULT_CRONJOB_AUTO_UPGRADE_K8S_TAR_FILE}

if [[ "$USE_EDGE_CLUSTER_REGISTRY" == "true" ]]; then
local default_image_registry_on_edge_cluster
Expand Down
33 changes: 33 additions & 0 deletions anax-in-k8s/Dockerfile.ubi.auto-upgrade-cron.s390x
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
FROM registry.access.redhat.com/ubi9/ubi-minimal:9.1

LABEL vendor="IBM"
LABEL summary="The agent auto upgrade cron job for edge clusters."
LABEL description=""

# add EPEL repo with jq pkg and all deps
COPY EPEL.repo /etc/yum.repos.d

# The build calls adduser (from shadow-utils)
# The auto-upgrade-cronjob.sh calls jq (from jq)
# Download kubectl
# Create required directories
# Create cronjobuser
ARG REQUIRED_RPMS="shadow-utils jq"
RUN microdnf update -y --nodocs --setopt=install_weak_deps=0 --disableplugin=subscription-manager \
&& microdnf install -y --nodocs --setopt=install_weak_deps=0 --disableplugin=subscription-manager ${REQUIRED_RPMS} \
&& microdnf clean all --disableplugin=subscription-manager \
&& rm -rf /mnt/rootfs/var/cache/* /mnt/rootfs/var/log/dnf* /mnt/rootfs/var/log/yum.* \
&& curl -4LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/s390x/kubectl \
&& chmod +x ./kubectl \
&& mv ./kubectl /usr/local/bin \
&& mkdir -p /tmp/agentbackup \
&& adduser cronjobuser -u 1000 -U -f -1 -c "cronjob user,1,2,3"

# Copy cronjob script into container
COPY cronjobs/auto-upgrade-cronjob.sh /usr/local/bin/

# Give user access to cronjob script
RUN chown -R cronjobuser:cronjobuser /usr/local/bin/auto-upgrade-cronjob.sh /tmp/agentbackup

USER cronjobuser
RUN chmod 755 /usr/local/bin/auto-upgrade-cronjob.sh
38 changes: 38 additions & 0 deletions anax-in-k8s/Dockerfile.ubi.s390x
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
FROM registry.access.redhat.com/ubi9/ubi-minimal:9.1

LABEL vendor="IBM"
LABEL summary="The agent for edge clusters."
LABEL description="The agent in a container that is used solely for the purpose of running the agent in a kubernetes edge cluster."

# add EPEL repo with jq pkg and all deps
COPY EPEL.repo /etc/yum.repos.d

# The anax binary (secrets manager code) shells out to groupadd, groupdel (from shadow-utils), pkill (from procps-ng)
# The anax.service calls jq (from jq) and killall (from psmisc)
# anax does not use iptables directly but the github.com/coreos/go-iptables/iptables dependency needs the directory structure
# Create required directories
ARG REQUIRED_RPMS="openssl ca-certificates shadow-utils jq iptables vim-minimal psmisc procps-ng"
RUN microdnf update -y --nodocs --setopt=install_weak_deps=0 --disableplugin=subscription-manager \
&& microdnf install -y --nodocs --setopt=install_weak_deps=0 --disableplugin=subscription-manager ${REQUIRED_RPMS} \
&& microdnf clean all --disableplugin=subscription-manager \
&& rm -rf /mnt/rootfs/var/cache/* /mnt/rootfs/var/log/dnf* /mnt/rootfs/var/log/yum.* \
&& mkdir -p /licenses /usr/horizon/bin /usr/horizon/web /var/horizon \
&& mkdir -p /etc/horizon/agbot/policy.d /etc/horizon/policy.d /etc/horizon/trust /etc/docker/certs.d \
&& adduser agentuser -u 1000 -U -f -1 -c "agent user,1,2,3"

# add license file
COPY LICENSE.txt /licenses

COPY script/* /home/agentuser/
COPY config/* /etc/horizon/

ADD anax /usr/horizon/bin/
ADD hzn /usr/bin/

RUN chown -R agentuser:agentuser /home/agentuser /etc/horizon

USER agentuser
WORKDIR /home/agentuser
RUN mkdir -p /home/agentuser/policy.d

ENTRYPOINT ["/home/agentuser/anax.service", "start"]

0 comments on commit ea438bd

Please sign in to comment.