Merge branch 'master' into issue-4031

open-horizon · Apr 23, 2024 · 7beab88 · 7beab88
2 parents 343b9c8 + f0ce773
commit 7beab88
Show file tree

Hide file tree

Showing 35 changed files with 1,734 additions and 648 deletions.
diff --git a/.github/workflows/E2E-test.yml b/.github/workflows/E2E-test.yml
@@ -6,6 +6,7 @@ on:
   pull_request:
     branches:
       - master
+      - v2.31
       - v2.30
       - v2.29
 

diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml
@@ -7,6 +7,7 @@ on:
   push:
     branches:
       - master
+      - v2.31
       - v2.30
       - v2.29
 

diff --git a/Makefile b/Makefile
@@ -14,7 +14,7 @@ SHELL := /bin/bash
 # DO NOT set this variable to the branch in which you are doing development work.
 BRANCH_NAME ?= ""
 
-export VERSION ?= 2.31.0
+export VERSION ?= 2.32.0
 # BUILD_NUMBER will be added to the version if set. It can be a simple number or something like a numeric timestamp or jenkins hash.
 # It can NOT contain dashes, but can contain: plus, period, and tilde.
 export BUILD_NUMBER
@@ -529,7 +529,7 @@ endif
 
 realclean: i18n-clean clean
 
-mostlyclean: anax-container-clean agbot-container-clean anax-k8s-clean css-clean ess-clean
+mostlyclean: anax-container-clean agbot-container-clean anax-k8s-clean auto-upgrade-cronjob-k8s-clean css-clean ess-clean
 	@echo "Mostlyclean"
 	rm -f $(EXECUTABLE) $(CLI_EXECUTABLE) $(CSS_EXECUTABLE) $(ESS_EXECUTABLE) $(CLI_CONFIG_FILE)
 	rm -Rf vendor

diff --git a/agent-install/agent-install.sh b/agent-install/agent-install.sh
@@ -66,6 +66,7 @@ CRONJOB_AUTO_UPGRADE_NAME="auto-upgrade-cronjob"
 IMAGE_REGISTRY_SECRET_NAME="openhorizon-agent-secrets-docker-cert"
 CONFIGMAP_NAME="openhorizon-agent-config"
 PVC_NAME="openhorizon-agent-pvc"
+DEFAULT_PVC_SIZE="10Gi"
 GET_RESOURCE_MAX_TRY=5
 POD_ID=""
 HZN_ENV_FILE="/tmp/agent-install-horizon-env"
@@ -158,6 +159,7 @@ Additional Edge Cluster Variables (in environment or config file):
     EDGE_CLUSTER_REGISTRY_USERNAME: specify this value if the edge cluster registry requires authentication
     EDGE_CLUSTER_REGISTRY_TOKEN: specify this value if the edge cluster registry requires authentication
     EDGE_CLUSTER_STORAGE_CLASS: the storage class to use for the agent and edge services. Default: gp2
+    EDGE_CLUSTER_PVC_SIZE: the requested size in the agent persistent volume to use for the agent. Default: 10Gi
     AGENT_NAMESPACE: The namespace the agent should run in. Default: openhorizon-agent
     AGENT_WAIT_MAX_SECONDS: Maximum seconds to wait for the Horizon agent to start or stop. Default: 30
     AGENT_DEPLOYMENT_STATUS_TIMEOUT_SECONDS: Maximum seconds to wait for the agent deployment rollout status to be successful. Default: 300
@@ -1254,6 +1256,7 @@ function get_all_variables() {
 
         # get other variables for cluster agent
         get_variable EDGE_CLUSTER_STORAGE_CLASS 'gp2'
+        get_variable EDGE_CLUSTER_PVC_SIZE "$DEFAULT_PVC_SIZE"
         get_variable AGENT_NAMESPACE "$DEFAULT_AGENT_NAMESPACE"
         get_variable NAMESPACE_SCOPED 'false'
         get_variable USE_EDGE_CLUSTER_REGISTRY 'true'
@@ -4029,7 +4032,11 @@ function prepare_k8s_pvc_file() {
         pvc_mode="ReadWriteMany"
     fi
 
-    sed -e "s#__AgentNameSpace__#${AGENT_NAMESPACE}#g" -e "s/__StorageClass__/\"${EDGE_CLUSTER_STORAGE_CLASS}\"/g" -e "s#__PVCAccessMode__#${pvc_mode}#g" persistentClaim-template.yml >persistentClaim.yml
+    if [[ -z $CLUSTER_PVC_SIZE ]]; then
+        CLUSTER_PVC_SIZE=$DEFAULT_PVC_SIZE
+    fi
+
+    sed -e "s#__AgentNameSpace__#${AGENT_NAMESPACE}#g" -e "s/__StorageClass__/\"${EDGE_CLUSTER_STORAGE_CLASS}\"/g" -e "s#__PVCAccessMode__#${pvc_mode}#g" -e "s#__PVCStorageSize__#${CLUSTER_PVC_SIZE}#g" persistentClaim-template.yml >persistentClaim.yml
     chk $? 'creating persistentClaim.yml'
 
     log_debug "prepare_k8s_pvc_file() end"

diff --git a/agent-install/agent-uninstall.sh b/agent-install/agent-uninstall.sh
@@ -230,8 +230,7 @@ function removeNodeFromLocalAndManagementHub() {
     log_debug "removeNodeFromLocalAndManagementHub() begin"
     log_info "Check node status for agent pod: ${POD_ID}"
 
-    EXPORT_EX_USER_AUTH_CMD="export HZN_EXCHANGE_USER_AUTH=${HZN_EXCHANGE_USER_AUTH}"
-    NODE_INFO=$($KUBECTL exec -it ${POD_ID} -n ${AGENT_NAMESPACE} -- bash -c "${EXPORT_EX_USER_AUTH_CMD}; hzn node list")
+    NODE_INFO=$($KUBECTL exec -it ${POD_ID} -n ${AGENT_NAMESPACE} -- bash -c "hzn node list")
     NODE_STATE=$(echo $NODE_INFO | jq -r .configstate.state | sed 's/[^a-z]*//g')
     NODE_ID=$(echo $NODE_INFO | jq -r .id | sed 's/\r//g')
     log_debug "NODE config state for ${NODE_ID} is ${NODE_STATE}"
@@ -241,7 +240,7 @@ function removeNodeFromLocalAndManagementHub() {
             log_info "Process with unregister..."
             unregister $NODE_ID
             sleep 2
-        else 
+        else
             log_info "node state is empty"
         fi
     else
@@ -263,7 +262,6 @@ function unregister() {
     log_debug "unregister() begin"
     log_info "Unregister agent for pod: ${POD_ID}"
 
-    EXPORT_EX_USER_AUTH_CMD="export HZN_EXCHANGE_USER_AUTH=${HZN_EXCHANGE_USER_AUTH}"
     local node_id=$1
 
     if [[ "$DELETE_EX_NODE" == "true" ]]; then
@@ -275,11 +273,11 @@ function unregister() {
     fi
 
     set +e
-    $KUBECTL exec -it ${POD_ID} -n ${AGENT_NAMESPACE} -- bash -c "${EXPORT_EX_USER_AUTH_CMD}; ${HZN_UNREGISTER_CMD}"
+    $KUBECTL exec -it ${POD_ID} -n ${AGENT_NAMESPACE} -- bash -c "${HZN_UNREGISTER_CMD}"
     set -e
 
     # verify the node is unregistered
-    NODE_STATE=$($KUBECTL exec -it ${POD_ID} -n ${AGENT_NAMESPACE} -- bash -c "${EXPORT_EX_USER_AUTH_CMD}; hzn node list | jq -r .configstate.state" | sed 's/[^a-z]*//g')
+    NODE_STATE=$($KUBECTL exec -it ${POD_ID} -n ${AGENT_NAMESPACE} -- bash -c "hzn node list | jq -r .configstate.state" | sed 's/[^a-z]*//g')
     log_debug "NODE config state is ${NODE_STATE}"
 
     if [[ "$NODE_STATE" != "unconfigured" ]] && [[ "$NODE_STATE" != "unconfiguring" ]]; then
@@ -289,10 +287,16 @@ function unregister() {
     log_debug "unregister() end"
 }
 
+function getEscapedExchangeUserAuth() {
+	local escaped_auth=$( echo "${HZN_EXCHANGE_USER_AUTH}" | sed 's/;/\\;/g;s/\$/\\$/g;s/\&/\\&/g;s/|/\\|/g' )
+	echo "${escaped_auth}"
+}
+
 function deleteNodeFromManagementHub() {
     log_debug "deleteNodeFromManagementHub() begin"
 
-    EXPORT_EX_USER_AUTH_CMD="export HZN_EXCHANGE_USER_AUTH=${HZN_EXCHANGE_USER_AUTH}"
+    escaped_USER_AUTH=$(getEscapedExchangeUserAuth)
+    EXPORT_EX_USER_AUTH_CMD="export HZN_EXCHANGE_USER_AUTH=${escaped_USER_AUTH}"
     local node_id=$1
 
     log_info "Deleting node ${node_id} from the management hub..."
@@ -307,10 +311,11 @@ function deleteNodeFromManagementHub() {
 function verifyNodeRemovedFromManagementHub() {
     log_debug "verifyNodeRemovedFromManagementHub() begin"
 
-    EXPORT_EX_USER_AUTH_CMD="export HZN_EXCHANGE_USER_AUTH=${HZN_EXCHANGE_USER_AUTH}"
+    escaped_USER_AUTH=$(getEscapedExchangeUserAuth)
+    EXPORT_EX_USER_AUTH_CMD="export HZN_EXCHANGE_USER_AUTH=${escaped_USER_AUTH}"
     local node_id=$1
 
-    log_info "Verifying node ${node_id} is from the management hub..."
+    log_info "Verifying node ${node_id} is removed from the management hub..."
 
     set +e
     $KUBECTL exec -it ${POD_ID} -n ${AGENT_NAMESPACE} -- bash -c "${EXPORT_EX_USER_AUTH_CMD}; hzn exchange node list ${node_id}" >/dev/null 2>&1
@@ -326,7 +331,7 @@ function deleteAgentResources() {
 
     set +e
     log_info "Deleting agent deployment..."
-    
+
     if [ "$USE_DELETE_FORCE" != true ]; then
         $KUBECTL delete deployment $DEPLOYMENT_NAME -n $AGENT_NAMESPACE --grace-period=$DELETE_TIMEOUT
 
@@ -352,7 +357,7 @@ function deleteAgentResources() {
         if [ "$USE_DELETE_FORCE" != true ]; then
             $KUBECTL delete pods -l app=agent --namespace=$AGENT_NAMESPACE --grace-period=$DELETE_TIMEOUT
 
-            PODS=$($KUBECTL get pod -l app=agent -n $AGENT_NAMESPACE 2>/dev/null) 
+            PODS=$($KUBECTL get pod -l app=agent -n $AGENT_NAMESPACE 2>/dev/null)
             if [[ -n "$PODS" ]]; then
                 log_info "Agent pods still exist"
                 PODS_STILL_EXIST="true"
@@ -416,17 +421,17 @@ function deleteAgentResources() {
 
 function uninstall_cluster() {
     show_config
-    
+
     validate_args
 
     get_agent_pod_id
 
     if [[ "$AGENT_POD_READY" == "true" ]]; then
     	removeNodeFromLocalAndManagementHub
-    else 
+    else
         log_info "agent pod under $AGENT_NAMESPACE is not ready, skip unregister process. Please remove node from management hub later if needed"
     fi
-    
+
     deleteAgentResources
 }
 

diff --git a/agent-install/k8s/auto-upgrade-cronjob-template.yml b/agent-install/k8s/auto-upgrade-cronjob-template.yml
@@ -21,6 +21,16 @@ spec:
             openhorizon.org/component: agent
             type: auto-upgrade-cronjob
         spec:
+          affinity:
+            podAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+              - labelSelector:
+                  matchExpressions:
+                  - key: openhorizon.org/component
+                    operator: In
+                    values:
+                    - agent
+                topologyKey: kubernetes.io/hostname
           volumes:
           - name: agent-pvc-storage
             persistentVolumeClaim:

diff --git a/agent-install/k8s/deployment-template.yml b/agent-install/k8s/deployment-template.yml
@@ -18,6 +18,16 @@ spec:
         app: agent
         openhorizon.org/component: agent
     spec:
+      affinity:
+        podAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: openhorizon.org/component
+                operator: In
+                values:
+                - agent
+            topologyKey: kubernetes.io/hostname
       serviceAccountName: agent-service-account
       volumes:
       - name: agent-etc-vol
@@ -65,7 +75,8 @@ spec:
         - mountPath: /var/horizon
           name: agent-pvc-storage
         ports:
-        - containerPort: 8510
+        - containerPort: 8443
+          name: ess-secure
         securityContext:
           runAsUser: 1000
           runAsGroup: 1000
@@ -90,4 +101,22 @@ spec:
         # START_CERT_VOL
         - name: HZN_MGMT_HUB_CERT_PATH
           value: /etc/default/cert/agent-install.crt
-        # END_CERT_VOL
+        # END_CERT_VOL
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: agent-service
+  namespace: __AgentNameSpace__
+  labels:
+    app: agent
+    openhorizon.org/component: agent
+spec:
+  selector:
+    app: agent
+    openhorizon.org/component: agent
+  ports:
+    - name: ess-secure-port-name
+      protocol: TCP
+      port: 8443
+      targetPort: 8443
diff --git a/agent-install/k8s/persistentClaim-template.yml b/agent-install/k8s/persistentClaim-template.yml
@@ -9,4 +9,4 @@ spec:
     - __PVCAccessMode__
   resources:
     requests:
-      storage: 10Gi
+      storage: __PVCStorageSize__ # need to be configurable
diff --git a/agreementbot/agreementworker.go b/agreementbot/agreementworker.go
@@ -1083,12 +1083,10 @@ func (b *BaseAgreementWorker) HandleAgreementReply(cph ConsumerProtocolHandler,
 
 			// For the purposes of compatibility, skip this function if the agbot config has not been updated to point to the CSS.
 			// Only non-pattern based agreements can use MMS object policy.
-			if agreement.GetDeviceType() == persistence.DEVICE_TYPE_DEVICE {
-				if b.GetCSSURL() != "" && agreement.Pattern == "" {
-					AgreementHandleMMSObjectPolicy(b, b.mmsObjMgr, *agreement, workerId, BAWlogstring)
-				} else if b.GetCSSURL() == "" {
-					glog.Errorf(BAWlogstring(workerId, fmt.Sprintf("unable to evaluate object placement because there is no CSS URL configured in this agbot")))
-				}
+			if b.GetCSSURL() != "" && agreement.Pattern == "" {
+				AgreementHandleMMSObjectPolicy(b, b.mmsObjMgr, *agreement, workerId, BAWlogstring)
+			} else if b.GetCSSURL() == "" {
+				glog.Errorf(BAWlogstring(workerId, fmt.Sprintf("unable to evaluate object placement because there is no CSS URL configured in this agbot")))
 			}
 
 			// Send the reply Ack if it's still valid.

diff --git a/anax-in-k8s/script/anax.service b/anax-in-k8s/script/anax.service
@@ -41,18 +41,17 @@ block() {
 editFSSAPIListen() {
 	echo "Edit FileSyncService.APIListen"
 	anaxJsonFile='/etc/horizon/anax.json'
-	echo "Modifying $anaxJsonFile for anax-in-container..."
+	echo "Modifying $anaxJsonFile for anax-in-k8s..."
 
 	anaxJson=$(jq . $anaxJsonFile)
 	checkrc $? "read anax.json"
 	cp $anaxJsonFile $anaxJsonFile.orig
 	checkrc $? "back up anax.json"
 
-	hostname=$(cat /etc/hostname)
-    anaxJson=$(jq ".Edge.FileSyncService.APIListen = \"$hostname\" " <<< $anaxJson)
+	anaxJson=$(jq ".Edge.FileSyncService.APIListen = \"0.0.0.0\" " <<< $anaxJson)
 	checkrc $? "change FileSyncService.APIListen"
 
-	anaxJson=$(jq ".Edge.FileSyncService.APIProtocol = \"https\" " <<< $anaxJson)
+	anaxJson=$(jq ".Edge.FileSyncService.APIProtocol = \"secure\" " <<< $anaxJson)
 	checkrc $? "change FileSyncService.APIProtocol"
 
 	echo "$anaxJson" > $anaxJsonFile