diff --git a/.github/workflows/E2E-test.yml b/.github/workflows/E2E-test.yml index c2e318eb5..49bac7b16 100644 --- a/.github/workflows/E2E-test.yml +++ b/.github/workflows/E2E-test.yml @@ -6,6 +6,7 @@ on: pull_request: branches: - master + - v2.31 - v2.30 - v2.29 diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 757552b0e..cbffa418f 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -7,6 +7,7 @@ on: push: branches: - master + - v2.31 - v2.30 - v2.29 diff --git a/Makefile b/Makefile index 4df0b47da..47c52a53f 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL := /bin/bash # DO NOT set this variable to the branch in which you are doing development work. BRANCH_NAME ?= "" -export VERSION ?= 2.31.0 +export VERSION ?= 2.32.0 # BUILD_NUMBER will be added to the version if set. It can be a simple number or something like a numeric timestamp or jenkins hash. # It can NOT contain dashes, but can contain: plus, period, and tilde. export BUILD_NUMBER @@ -529,7 +529,7 @@ endif realclean: i18n-clean clean -mostlyclean: anax-container-clean agbot-container-clean anax-k8s-clean css-clean ess-clean +mostlyclean: anax-container-clean agbot-container-clean anax-k8s-clean auto-upgrade-cronjob-k8s-clean css-clean ess-clean @echo "Mostlyclean" rm -f $(EXECUTABLE) $(CLI_EXECUTABLE) $(CSS_EXECUTABLE) $(ESS_EXECUTABLE) $(CLI_CONFIG_FILE) rm -Rf vendor diff --git a/agent-install/agent-install.sh b/agent-install/agent-install.sh index d0b83625c..af0059fe2 100755 --- a/agent-install/agent-install.sh +++ b/agent-install/agent-install.sh @@ -66,6 +66,7 @@ CRONJOB_AUTO_UPGRADE_NAME="auto-upgrade-cronjob" IMAGE_REGISTRY_SECRET_NAME="openhorizon-agent-secrets-docker-cert" CONFIGMAP_NAME="openhorizon-agent-config" PVC_NAME="openhorizon-agent-pvc" +DEFAULT_PVC_SIZE="10Gi" GET_RESOURCE_MAX_TRY=5 POD_ID="" HZN_ENV_FILE="/tmp/agent-install-horizon-env" @@ -158,6 +159,7 @@ Additional Edge Cluster Variables (in environment or config file): EDGE_CLUSTER_REGISTRY_USERNAME: specify this value if the edge cluster registry requires authentication EDGE_CLUSTER_REGISTRY_TOKEN: specify this value if the edge cluster registry requires authentication EDGE_CLUSTER_STORAGE_CLASS: the storage class to use for the agent and edge services. Default: gp2 + EDGE_CLUSTER_PVC_SIZE: the requested size in the agent persistent volume to use for the agent. Default: 10Gi AGENT_NAMESPACE: The namespace the agent should run in. Default: openhorizon-agent AGENT_WAIT_MAX_SECONDS: Maximum seconds to wait for the Horizon agent to start or stop. Default: 30 AGENT_DEPLOYMENT_STATUS_TIMEOUT_SECONDS: Maximum seconds to wait for the agent deployment rollout status to be successful. Default: 300 @@ -1254,6 +1256,7 @@ function get_all_variables() { # get other variables for cluster agent get_variable EDGE_CLUSTER_STORAGE_CLASS 'gp2' + get_variable EDGE_CLUSTER_PVC_SIZE "$DEFAULT_PVC_SIZE" get_variable AGENT_NAMESPACE "$DEFAULT_AGENT_NAMESPACE" get_variable NAMESPACE_SCOPED 'false' get_variable USE_EDGE_CLUSTER_REGISTRY 'true' @@ -4029,7 +4032,11 @@ function prepare_k8s_pvc_file() { pvc_mode="ReadWriteMany" fi - sed -e "s#__AgentNameSpace__#${AGENT_NAMESPACE}#g" -e "s/__StorageClass__/\"${EDGE_CLUSTER_STORAGE_CLASS}\"/g" -e "s#__PVCAccessMode__#${pvc_mode}#g" persistentClaim-template.yml >persistentClaim.yml + if [[ -z $CLUSTER_PVC_SIZE ]]; then + CLUSTER_PVC_SIZE=$DEFAULT_PVC_SIZE + fi + + sed -e "s#__AgentNameSpace__#${AGENT_NAMESPACE}#g" -e "s/__StorageClass__/\"${EDGE_CLUSTER_STORAGE_CLASS}\"/g" -e "s#__PVCAccessMode__#${pvc_mode}#g" -e "s#__PVCStorageSize__#${CLUSTER_PVC_SIZE}#g" persistentClaim-template.yml >persistentClaim.yml chk $? 'creating persistentClaim.yml' log_debug "prepare_k8s_pvc_file() end" diff --git a/agent-install/agent-uninstall.sh b/agent-install/agent-uninstall.sh index f7764bbd8..5709d91d3 100644 --- a/agent-install/agent-uninstall.sh +++ b/agent-install/agent-uninstall.sh @@ -230,8 +230,7 @@ function removeNodeFromLocalAndManagementHub() { log_debug "removeNodeFromLocalAndManagementHub() begin" log_info "Check node status for agent pod: ${POD_ID}" - EXPORT_EX_USER_AUTH_CMD="export HZN_EXCHANGE_USER_AUTH=${HZN_EXCHANGE_USER_AUTH}" - NODE_INFO=$($KUBECTL exec -it ${POD_ID} -n ${AGENT_NAMESPACE} -- bash -c "${EXPORT_EX_USER_AUTH_CMD}; hzn node list") + NODE_INFO=$($KUBECTL exec -it ${POD_ID} -n ${AGENT_NAMESPACE} -- bash -c "hzn node list") NODE_STATE=$(echo $NODE_INFO | jq -r .configstate.state | sed 's/[^a-z]*//g') NODE_ID=$(echo $NODE_INFO | jq -r .id | sed 's/\r//g') log_debug "NODE config state for ${NODE_ID} is ${NODE_STATE}" @@ -241,7 +240,7 @@ function removeNodeFromLocalAndManagementHub() { log_info "Process with unregister..." unregister $NODE_ID sleep 2 - else + else log_info "node state is empty" fi else @@ -263,7 +262,6 @@ function unregister() { log_debug "unregister() begin" log_info "Unregister agent for pod: ${POD_ID}" - EXPORT_EX_USER_AUTH_CMD="export HZN_EXCHANGE_USER_AUTH=${HZN_EXCHANGE_USER_AUTH}" local node_id=$1 if [[ "$DELETE_EX_NODE" == "true" ]]; then @@ -275,11 +273,11 @@ function unregister() { fi set +e - $KUBECTL exec -it ${POD_ID} -n ${AGENT_NAMESPACE} -- bash -c "${EXPORT_EX_USER_AUTH_CMD}; ${HZN_UNREGISTER_CMD}" + $KUBECTL exec -it ${POD_ID} -n ${AGENT_NAMESPACE} -- bash -c "${HZN_UNREGISTER_CMD}" set -e # verify the node is unregistered - NODE_STATE=$($KUBECTL exec -it ${POD_ID} -n ${AGENT_NAMESPACE} -- bash -c "${EXPORT_EX_USER_AUTH_CMD}; hzn node list | jq -r .configstate.state" | sed 's/[^a-z]*//g') + NODE_STATE=$($KUBECTL exec -it ${POD_ID} -n ${AGENT_NAMESPACE} -- bash -c "hzn node list | jq -r .configstate.state" | sed 's/[^a-z]*//g') log_debug "NODE config state is ${NODE_STATE}" if [[ "$NODE_STATE" != "unconfigured" ]] && [[ "$NODE_STATE" != "unconfiguring" ]]; then @@ -289,10 +287,16 @@ function unregister() { log_debug "unregister() end" } +function getEscapedExchangeUserAuth() { + local escaped_auth=$( echo "${HZN_EXCHANGE_USER_AUTH}" | sed 's/;/\\;/g;s/\$/\\$/g;s/\&/\\&/g;s/|/\\|/g' ) + echo "${escaped_auth}" +} + function deleteNodeFromManagementHub() { log_debug "deleteNodeFromManagementHub() begin" - EXPORT_EX_USER_AUTH_CMD="export HZN_EXCHANGE_USER_AUTH=${HZN_EXCHANGE_USER_AUTH}" + escaped_USER_AUTH=$(getEscapedExchangeUserAuth) + EXPORT_EX_USER_AUTH_CMD="export HZN_EXCHANGE_USER_AUTH=${escaped_USER_AUTH}" local node_id=$1 log_info "Deleting node ${node_id} from the management hub..." @@ -307,10 +311,11 @@ function deleteNodeFromManagementHub() { function verifyNodeRemovedFromManagementHub() { log_debug "verifyNodeRemovedFromManagementHub() begin" - EXPORT_EX_USER_AUTH_CMD="export HZN_EXCHANGE_USER_AUTH=${HZN_EXCHANGE_USER_AUTH}" + escaped_USER_AUTH=$(getEscapedExchangeUserAuth) + EXPORT_EX_USER_AUTH_CMD="export HZN_EXCHANGE_USER_AUTH=${escaped_USER_AUTH}" local node_id=$1 - log_info "Verifying node ${node_id} is from the management hub..." + log_info "Verifying node ${node_id} is removed from the management hub..." set +e $KUBECTL exec -it ${POD_ID} -n ${AGENT_NAMESPACE} -- bash -c "${EXPORT_EX_USER_AUTH_CMD}; hzn exchange node list ${node_id}" >/dev/null 2>&1 @@ -326,7 +331,7 @@ function deleteAgentResources() { set +e log_info "Deleting agent deployment..." - + if [ "$USE_DELETE_FORCE" != true ]; then $KUBECTL delete deployment $DEPLOYMENT_NAME -n $AGENT_NAMESPACE --grace-period=$DELETE_TIMEOUT @@ -352,7 +357,7 @@ function deleteAgentResources() { if [ "$USE_DELETE_FORCE" != true ]; then $KUBECTL delete pods -l app=agent --namespace=$AGENT_NAMESPACE --grace-period=$DELETE_TIMEOUT - PODS=$($KUBECTL get pod -l app=agent -n $AGENT_NAMESPACE 2>/dev/null) + PODS=$($KUBECTL get pod -l app=agent -n $AGENT_NAMESPACE 2>/dev/null) if [[ -n "$PODS" ]]; then log_info "Agent pods still exist" PODS_STILL_EXIST="true" @@ -416,17 +421,17 @@ function deleteAgentResources() { function uninstall_cluster() { show_config - + validate_args get_agent_pod_id if [[ "$AGENT_POD_READY" == "true" ]]; then removeNodeFromLocalAndManagementHub - else + else log_info "agent pod under $AGENT_NAMESPACE is not ready, skip unregister process. Please remove node from management hub later if needed" fi - + deleteAgentResources } diff --git a/agent-install/k8s/auto-upgrade-cronjob-template.yml b/agent-install/k8s/auto-upgrade-cronjob-template.yml index 6b296e73f..0b559656b 100644 --- a/agent-install/k8s/auto-upgrade-cronjob-template.yml +++ b/agent-install/k8s/auto-upgrade-cronjob-template.yml @@ -21,6 +21,16 @@ spec: openhorizon.org/component: agent type: auto-upgrade-cronjob spec: + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: openhorizon.org/component + operator: In + values: + - agent + topologyKey: kubernetes.io/hostname volumes: - name: agent-pvc-storage persistentVolumeClaim: diff --git a/agent-install/k8s/deployment-template.yml b/agent-install/k8s/deployment-template.yml index 5206b7648..87016c934 100644 --- a/agent-install/k8s/deployment-template.yml +++ b/agent-install/k8s/deployment-template.yml @@ -18,6 +18,16 @@ spec: app: agent openhorizon.org/component: agent spec: + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: openhorizon.org/component + operator: In + values: + - agent + topologyKey: kubernetes.io/hostname serviceAccountName: agent-service-account volumes: - name: agent-etc-vol @@ -65,7 +75,8 @@ spec: - mountPath: /var/horizon name: agent-pvc-storage ports: - - containerPort: 8510 + - containerPort: 8443 + name: ess-secure securityContext: runAsUser: 1000 runAsGroup: 1000 @@ -90,4 +101,22 @@ spec: # START_CERT_VOL - name: HZN_MGMT_HUB_CERT_PATH value: /etc/default/cert/agent-install.crt - # END_CERT_VOL \ No newline at end of file + # END_CERT_VOL +--- +apiVersion: v1 +kind: Service +metadata: + name: agent-service + namespace: __AgentNameSpace__ + labels: + app: agent + openhorizon.org/component: agent +spec: + selector: + app: agent + openhorizon.org/component: agent + ports: + - name: ess-secure-port-name + protocol: TCP + port: 8443 + targetPort: 8443 \ No newline at end of file diff --git a/agent-install/k8s/persistentClaim-template.yml b/agent-install/k8s/persistentClaim-template.yml index 4df6cb9b6..46e2da09b 100644 --- a/agent-install/k8s/persistentClaim-template.yml +++ b/agent-install/k8s/persistentClaim-template.yml @@ -9,4 +9,4 @@ spec: - __PVCAccessMode__ resources: requests: - storage: 10Gi + storage: __PVCStorageSize__ # need to be configurable diff --git a/agreementbot/agreementworker.go b/agreementbot/agreementworker.go index 27b7a8068..59c26c36b 100644 --- a/agreementbot/agreementworker.go +++ b/agreementbot/agreementworker.go @@ -1083,12 +1083,10 @@ func (b *BaseAgreementWorker) HandleAgreementReply(cph ConsumerProtocolHandler, // For the purposes of compatibility, skip this function if the agbot config has not been updated to point to the CSS. // Only non-pattern based agreements can use MMS object policy. - if agreement.GetDeviceType() == persistence.DEVICE_TYPE_DEVICE { - if b.GetCSSURL() != "" && agreement.Pattern == "" { - AgreementHandleMMSObjectPolicy(b, b.mmsObjMgr, *agreement, workerId, BAWlogstring) - } else if b.GetCSSURL() == "" { - glog.Errorf(BAWlogstring(workerId, fmt.Sprintf("unable to evaluate object placement because there is no CSS URL configured in this agbot"))) - } + if b.GetCSSURL() != "" && agreement.Pattern == "" { + AgreementHandleMMSObjectPolicy(b, b.mmsObjMgr, *agreement, workerId, BAWlogstring) + } else if b.GetCSSURL() == "" { + glog.Errorf(BAWlogstring(workerId, fmt.Sprintf("unable to evaluate object placement because there is no CSS URL configured in this agbot"))) } // Send the reply Ack if it's still valid. diff --git a/anax-in-k8s/script/anax.service b/anax-in-k8s/script/anax.service index 1c8d93582..272791e1f 100755 --- a/anax-in-k8s/script/anax.service +++ b/anax-in-k8s/script/anax.service @@ -41,18 +41,17 @@ block() { editFSSAPIListen() { echo "Edit FileSyncService.APIListen" anaxJsonFile='/etc/horizon/anax.json' - echo "Modifying $anaxJsonFile for anax-in-container..." + echo "Modifying $anaxJsonFile for anax-in-k8s..." anaxJson=$(jq . $anaxJsonFile) checkrc $? "read anax.json" cp $anaxJsonFile $anaxJsonFile.orig checkrc $? "back up anax.json" - hostname=$(cat /etc/hostname) - anaxJson=$(jq ".Edge.FileSyncService.APIListen = \"$hostname\" " <<< $anaxJson) + anaxJson=$(jq ".Edge.FileSyncService.APIListen = \"0.0.0.0\" " <<< $anaxJson) checkrc $? "change FileSyncService.APIListen" - anaxJson=$(jq ".Edge.FileSyncService.APIProtocol = \"https\" " <<< $anaxJson) + anaxJson=$(jq ".Edge.FileSyncService.APIProtocol = \"secure\" " <<< $anaxJson) checkrc $? "change FileSyncService.APIProtocol" echo "$anaxJson" > $anaxJsonFile diff --git a/cli/locales/fr/messages.gotext.json b/cli/locales/fr/messages.gotext.json index 9fe397ca1..c56a5360a 100644 --- a/cli/locales/fr/messages.gotext.json +++ b/cli/locales/fr/messages.gotext.json @@ -1825,6 +1825,30 @@ ], "fuzzy": true }, + { + "id": "Name, or Org is empty string.", + "message": "Name, or Org is empty string.", + "translation": "Le nom ou l'organisation est une chaîne vide.", + "fuzzy": true + }, + { + "id": "The serviceVersions array is empty.", + "message": "The serviceVersions array is empty.", + "translation": "Le tableau serviceVersions est vide.", + "fuzzy": true + }, + { + "id": "retry_durations and retries cannot be zero if priority_value is set to non-zero value", + "message": "retry_durations and retries cannot be zero if priority_value is set to non-zero value", + "translation": "retry_durations et retries ne peuvent pas être égales à zéro si priority_value est défini sur une valeur différente de zéro", + "fuzzy": true + }, + { + "id": "retry_durations, retries and verified_durations cannot be non-zero value if priority_value is zero or not set", + "message": "retry_durations, retries and verified_durations cannot be non-zero value if priority_value is zero or not set", + "translation": "retry_durations, retries et verified_durations ne peuvent pas avoir une valeur différente de zéro si priority_value est égale à zéro ou n'est pas définie", + "fuzzy": true + }, { "id": "properties contains an invalid property: {Err}", "message": "properties contains an invalid property: {Err}", @@ -17201,6 +17225,38 @@ ], "fuzzy": true }, + { + "id": "failed to unmarshal ClusterDeployment in 'hzn exchange service list' output: {Err}", + "message": "failed to unmarshal ClusterDeployment in 'hzn exchange service list' output: {Err}", + "translation": "Echec de la désérialisation de ClusterDeployment dans la sortie 'hzn exchange service list' : {Err}", + "placeholders": [ + { + "id": "Err", + "string": "%[1]v", + "type": "error", + "underlyingType": "interface{Error() string}", + "argNum": 1, + "expr": "err" + } + ], + "fuzzy": true + }, + { + "id": "failed to marshal truncked ClusterDeployment in 'hzn exchange service list' output: {Err}", + "message": "failed to marshal truncked ClusterDeployment in 'hzn exchange service list' output: {Err}", + "translation": "Echec de la sérialisation de ClusterDeployment tronqué dans la sortie 'hzn exchange service list' : {Err}", + "placeholders": [ + { + "id": "Err", + "string": "%[1]v", + "type": "error", + "underlyingType": "interface{Error() string}", + "argNum": 1, + "expr": "err" + } + ], + "fuzzy": true + }, { "id": "Ignoring -f because the clusterDeployment attribute is empty for this service.", "message": "Ignoring -f because the clusterDeployment attribute is empty for this service.", @@ -27696,22 +27752,6 @@ "translation": "L'entrée SecretBindingCheck ne peut pas avoir pour valeur NULL", "fuzzy": true }, - { - "id": "Node type '{NodeType}' does not support secret binding check.", - "message": "Node type '{NodeType}' does not support secret binding check.", - "translation": "Le type de noeud '{NodeType}' ne prend pas en charge la vérification de liaison de secret.", - "placeholders": [ - { - "id": "NodeType", - "string": "%[1]v", - "type": "string", - "underlyingType": "string", - "argNum": 1, - "expr": "nodeType" - } - ], - "fuzzy": true - }, { "id": "No service versions with architecture {NodeArch} specified in the deployment policy or pattern.", "message": "No service versions with architecture {NodeArch} specified in the deployment policy or pattern.", @@ -27746,6 +27786,30 @@ "translation": "Type incompatible", "fuzzy": true }, + { + "id": "{Msgcompatible}, Warning: {Reason}", + "message": "{Msgcompatible}, Warning: {Reason}", + "translation": "{Msgcompatible}, Avertissement : {Reason}", + "placeholders": [ + { + "id": "Msgcompatible", + "string": "%[1]v", + "type": "string", + "underlyingType": "string", + "argNum": 1, + "expr": "msg_compatible" + }, + { + "id": "Reason", + "string": "%[2]v", + "type": "string", + "underlyingType": "string", + "argNum": 2, + "expr": "reason" + } + ], + "fuzzy": true + }, { "id": "Error getting services for all archetctures for {ServiceOrg}/{ServiceURL} version {Version}. {Err}", "message": "Error getting services for all archetctures for {ServiceOrg}/{ServiceURL} version {Version}. {Err}", @@ -27906,12 +27970,6 @@ ], "fuzzy": true }, - { - "id": "Secret binding for a cluster service is not supported.", - "message": "Secret binding for a cluster service is not supported.", - "translation": "La liaison de secret pour un service de cluster n'est pas prise en charge.", - "fuzzy": true - }, { "id": "No secret binding found for the following service secrets: {NbArray}.", "message": "No secret binding found for the following service secrets: {NbArray}.", @@ -28044,6 +28102,22 @@ ], "fuzzy": true }, + { + "id": "{Err}, use non node-level secret", + "message": "{Err}, use non node-level secret", + "translation": "{Err}, utilisent un secret qui n'est pas au niveau du noeud", + "placeholders": [ + { + "id": "Err", + "string": "%[1]v", + "type": "error", + "underlyingType": "interface{Error() string}", + "argNum": 1, + "expr": "err" + } + ], + "fuzzy": true + }, { "id": "Error parsing secret name in the secret binding. {Errparse}", "message": "Error parsing secret name in the secret binding. {Errparse}", @@ -28060,6 +28134,38 @@ ], "fuzzy": true }, + { + "id": "Error checking secret {VaultSecretName} for node {NName} in the secret manager. {Err}", + "message": "Error checking secret {VaultSecretName} for node {NName} in the secret manager. {Err}", + "translation": "Erreur lors de la vérification du secret {VaultSecretName} pour le noeud {NName} dans le gestionnaire de secrets. {Err}", + "placeholders": [ + { + "id": "VaultSecretName", + "string": "%[1]v", + "type": "string", + "underlyingType": "string", + "argNum": 1, + "expr": "vaultSecretName" + }, + { + "id": "NName", + "string": "%[2]v", + "type": "string", + "underlyingType": "string", + "argNum": 2, + "expr": "nName" + }, + { + "id": "Err", + "string": "%[3]v", + "type": "error", + "underlyingType": "interface{Error() string}", + "argNum": 3, + "expr": "err" + } + ], + "fuzzy": true + }, { "id": "Error checking secret {VaultSecretName} in the secret manager. {Err}", "message": "Error checking secret {VaultSecretName} in the secret manager. {Err}", @@ -28084,6 +28190,30 @@ ], "fuzzy": true }, + { + "id": "Node level secret {VaultSecretName} doesn't exist for node {NName}", + "message": "Node level secret {VaultSecretName} doesn't exist for node {NName}", + "translation": "Le secret au niveau du noeud {VaultSecretName} n'existe pas pour le noeud {NName}", + "placeholders": [ + { + "id": "VaultSecretName", + "string": "%[1]v", + "type": "string", + "underlyingType": "string", + "argNum": 1, + "expr": "vaultSecretName" + }, + { + "id": "NName", + "string": "%[2]v", + "type": "string", + "underlyingType": "string", + "argNum": 2, + "expr": "nName" + } + ], + "fuzzy": true + }, { "id": "The binding secret name cannot be an empty string. The valid formats are: '' for the organization level secret and 'user//' for the user level secret.", "message": "The binding secret name cannot be an empty string. The valid formats are: '' for the organization level secret and 'user//' for the user level secret.", diff --git a/common/deploymentconfig.go b/common/deploymentconfig.go index 8eb9c2d4b..45b751852 100644 --- a/common/deploymentconfig.go +++ b/common/deploymentconfig.go @@ -221,6 +221,6 @@ func GetClusterDeploymentMetadata(clusterDeployment interface{}, inspectOperator } func GetKubeOperatorNamespace(tar string) (string, error) { - _, namespace, err := kube_operator.ProcessDeployment(tar, nil, map[string]string{}, map[string]string{}, "", 0) + _, namespace, err := kube_operator.ProcessDeployment(tar, nil, map[string]string{}, "", "", map[string]string{}, "", 0) return namespace, err } diff --git a/cutil/cluster.go b/cutil/cluster.go index 5eaad1601..7658a21a9 100644 --- a/cutil/cluster.go +++ b/cutil/cluster.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/golang/glog" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -12,6 +13,8 @@ import ( "os" ) +const AGENT_PVC_NAME = "openhorizon-agent-pvc" + func NewKubeConfig() (*rest.Config, error) { config, err := rest.InClusterConfig() if err != nil { @@ -93,8 +96,22 @@ func GetClusterNamespace() string { func IsNamespaceScoped() bool { isNamespaceScoped := os.Getenv("HZN_NAMESPACE_SCOPED") - if isNamespaceScoped == "true" { - return true + return isNamespaceScoped == "true" +} + +// pvc name: openhorizon-agent-pvc +func GetAgentPVCInfo() (string, []v1.PersistentVolumeAccessMode, error) { + client, err := NewKubeClient() + if err != nil { + return "", []v1.PersistentVolumeAccessMode{}, err + } + + agentNamespace := GetClusterNamespace() + if agentPVC, err := client.CoreV1().PersistentVolumeClaims(agentNamespace).Get(context.Background(), AGENT_PVC_NAME, metav1.GetOptions{}); err != nil { + return "", []v1.PersistentVolumeAccessMode{}, err + } else { + scName := agentPVC.Spec.StorageClassName + accessMode := agentPVC.Spec.AccessModes + return *scName, accessMode, nil } - return false } diff --git a/cutil/cutil.go b/cutil/cutil.go index 7015c51d8..9caeee48a 100644 --- a/cutil/cutil.go +++ b/cutil/cutil.go @@ -282,17 +282,31 @@ func SetPlatformEnvvars(envAdds map[string]string, prefix string, agreementId st // The name of the mounted file containing the FSS API SSL Certificate that the container should use. envAdds[prefix+"ESS_CERT"] = path.Join(config.HZN_FSS_CERT_MOUNT, config.HZN_FSS_CERT_FILE) - } -// Temporary function to remove ESS env vars for the edge cluster case. -func RemoveESSEnvVars(envAdds map[string]string, prefix string) map[string]string { - delete(envAdds, prefix+"ESS_API_PROTOCOL") - delete(envAdds, prefix+"ESS_API_ADDRESS") - delete(envAdds, prefix+"ESS_API_PORT") - delete(envAdds, prefix+"ESS_AUTH") - delete(envAdds, prefix+"ESS_CERT") - return envAdds +func SetESSEnvVarsForClusterAgent(envAdds map[string]string, prefix string, agreementId string) { + // The address of the file sync service API. + namespace := GetClusterNamespace() + fssAddress := fmt.Sprintf("agent-service.%v.svc.cluster.local", namespace) + envAdds[prefix+"ESS_API_ADDRESS"] = fssAddress + + fssPort := envAdds[prefix+"ESS_API_PORT"] + if strings.Contains(fssPort, "\"") { + fssPort = strings.ReplaceAll(fssPort, "\"", "") + } + envAdds[prefix+"ESS_API_PORT"] = fssPort + + // The secret name of the FSS credentials that the operator should use. + envAdds[prefix+"ESS_AUTH"] = config.HZN_FSS_AUTH_PATH + "-" + agreementId + + // The secret name of FSS API SSL Certificate that the operator should use. + envAdds[prefix+"ESS_CERT"] = config.HZN_FSS_CERT_PATH + "-" + agreementId + + // The cluster agent namesace + envAdds[prefix+"AGENT_NAMESPACE"] = namespace + + agentStorageClassNAME, _, _ := GetAgentPVCInfo() + envAdds[prefix+"AGENT_STORAGE_CLASS_NAME"] = agentStorageClassNAME } // This function is similar to the above, for env vars that are system related. It is only used by workloads. diff --git a/exchangecommon/node_management_policy.go b/exchangecommon/node_management_policy.go index b94a2da37..13153c407 100644 --- a/exchangecommon/node_management_policy.go +++ b/exchangecommon/node_management_policy.go @@ -20,6 +20,7 @@ type ExchangeNodeManagementPolicy struct { PolicyUpgradeTime string `json:"start"` UpgradeWindowDuration int `json:"startWindow"` AgentAutoUpgradePolicy *ExchangeAgentUpgradePolicy `json:"agentUpgradePolicy,omitempty"` + AgentImagePolicy *ExchangeAgentImagePolicy `json:"agentImagePolicy,omitempty"` LastUpdated string `json:"lastUpdated,omitempty"` Created string `json:"created,omitempty"` } @@ -135,3 +136,21 @@ type AgentFileVersions struct { func (a AgentFileVersions) String() string { return fmt.Sprintf("SoftwareVersions: %v, ConfigVersions: %v, CertVersions: %v", a.SoftwareVersions, a.ConfigVersions, a.CertVersions) } + +type ExchangeAgentImagePolicy struct { + Removal []ImageRemovalPolicy `json:"imageRemovalPolicies,omitempty"` +} + +func (e ExchangeAgentImagePolicy) String() string { + str := "Removal: " + for _, pol := range e.Removal { + str = str + fmt.Sprintf(" %v", pol) + } + return str +} + +type ImageRemovalPolicy struct { + ImageId string `json:"imageId"` + DeleteAfterMinutes uint64 `json:"deleteAfterMinutes"` + AgentDownloadedOnly bool `json:"agentDownloadedOnly"` +} diff --git a/governance/governance.go b/governance/governance.go index 862838a9e..b074d7d34 100644 --- a/governance/governance.go +++ b/governance/governance.go @@ -523,6 +523,12 @@ func (w *GovernanceWorker) governAgreements() { // The proposal for this agreement is no longer compatible with the node's policy, so cancel the agreement. glog.V(3).Infof(logString(fmt.Sprintf("current proposal for %v is out of policy: %v", ag.CurrentAgreementId, err))) + glog.V(3).Infof(logString(fmt.Sprintf("terminating agreement %v because it cannot be verified by the agreement bot.", ag.CurrentAgreementId))) + reason := w.producerPH[ag.AgreementProtocol].GetTerminationCode(producer.TERM_REASON_POLICY_CHANGED) + eventlog.LogAgreementEvent(w.db, persistence.SEVERITY_INFO, + persistence.NewMessageMeta(EL_GOV_START_TERM_AG_WITH_REASON, ag.RunningWorkload.URL, w.producerPH[ag.AgreementProtocol].GetTerminationReason(reason)), + persistence.EC_CANCEL_AGREEMENT, ag) + w.cancelGovernedAgreement(&ag, reason) } else { glog.V(5).Infof(logString(fmt.Sprintf("agreement %v is still in policy.", ag.CurrentAgreementId))) diff --git a/i18n_translation/anax.out.gotext.json b/i18n_translation/anax.out.gotext.json index 543d8fa9d..d12248648 100644 --- a/i18n_translation/anax.out.gotext.json +++ b/i18n_translation/anax.out.gotext.json @@ -471,6 +471,22 @@ "id": "Error converting the selections into Selectors: {Err}", "translation": "Error converting the selections into Selectors: {Err}" }, + { + "id": "Name, or Org is empty string.", + "translation": "Name, or Org is empty string." + }, + { + "id": "The serviceVersions array is empty.", + "translation": "The serviceVersions array is empty." + }, + { + "id": "retry_durations and retries cannot be zero if priority_value is set to non-zero value", + "translation": "retry_durations and retries cannot be zero if priority_value is set to non-zero value" + }, + { + "id": "retry_durations, retries and verified_durations cannot be non-zero value if priority_value is zero or not set", + "translation": "retry_durations, retries and verified_durations cannot be non-zero value if priority_value is zero or not set" + }, { "id": "properties contains an invalid property: {Err}", "translation": "properties contains an invalid property: {Err}" @@ -1315,10 +1331,6 @@ "id": "The SecretBindingCheck input cannot be null", "translation": "The SecretBindingCheck input cannot be null" }, - { - "id": "Node type '{NodeType}' does not support secret binding check.", - "translation": "Node type '{NodeType}' does not support secret binding check." - }, { "id": "No service versions with architecture {NodeArch} specified in the deployment policy or pattern.", "translation": "No service versions with architecture {NodeArch} specified in the deployment policy or pattern." @@ -1335,6 +1347,10 @@ "id": "Type Incompatible", "translation": "Type Incompatible" }, + { + "id": "{Msgcompatible}, Warning: {Reason}", + "translation": "{Msgcompatible}, Warning: {Reason}" + }, { "id": "Error getting services for all archetctures for {ServiceOrg}/{ServiceURL} version {Version}. {Err}", "translation": "Error getting services for all archetctures for {ServiceOrg}/{ServiceURL} version {Version}. {Err}" @@ -1371,10 +1387,6 @@ "id": "Failed to find the dependent services for {Org}/{URL} {Arch} {Version}. {Err}", "translation": "Failed to find the dependent services for {Org}/{URL} {Arch} {Version}. {Err}" }, - { - "id": "Secret binding for a cluster service is not supported.", - "translation": "Secret binding for a cluster service is not supported." - }, { "id": "No secret binding found for the following service secrets: {NbArray}.", "translation": "No secret binding found for the following service secrets: {NbArray}." @@ -1399,14 +1411,26 @@ "id": "Secret {VaultSecretName} does not exist in the secret manager for either org level or user level.", "translation": "Secret {VaultSecretName} does not exist in the secret manager for either org level or user level." }, + { + "id": "{Err}, use non node-level secret", + "translation": "{Err}, use non node-level secret" + }, { "id": "Error parsing secret name in the secret binding. {Errparse}", "translation": "Error parsing secret name in the secret binding. {Errparse}" }, + { + "id": "Error checking secret {VaultSecretName} for node {NName} in the secret manager. {Err}", + "translation": "Error checking secret {VaultSecretName} for node {NName} in the secret manager. {Err}" + }, { "id": "Error checking secret {VaultSecretName} in the secret manager. {Err}", "translation": "Error checking secret {VaultSecretName} in the secret manager. {Err}" }, + { + "id": "Node level secret {VaultSecretName} doesn't exist for node {NName}", + "translation": "Node level secret {VaultSecretName} doesn't exist for node {NName}" + }, { "id": "The binding secret name cannot be an empty string. The valid formats are: '' for the organization level secret and 'user//' for the user level secret.", "translation": "The binding secret name cannot be an empty string. The valid formats are: '' for the organization level secret and 'user//' for the user level secret." diff --git a/i18n_translation/hzn.out.gotext.json b/i18n_translation/hzn.out.gotext.json index 4c19eb4eb..a800a7707 100644 --- a/i18n_translation/hzn.out.gotext.json +++ b/i18n_translation/hzn.out.gotext.json @@ -403,6 +403,22 @@ "id": "Error converting the selections into Selectors: {Err}", "translation": "Error converting the selections into Selectors: {Err}" }, + { + "id": "Name, or Org is empty string.", + "translation": "Name, or Org is empty string." + }, + { + "id": "The serviceVersions array is empty.", + "translation": "The serviceVersions array is empty." + }, + { + "id": "retry_durations and retries cannot be zero if priority_value is set to non-zero value", + "translation": "retry_durations and retries cannot be zero if priority_value is set to non-zero value" + }, + { + "id": "retry_durations, retries and verified_durations cannot be non-zero value if priority_value is zero or not set", + "translation": "retry_durations, retries and verified_durations cannot be non-zero value if priority_value is zero or not set" + }, { "id": "properties contains an invalid property: {Err}", "translation": "properties contains an invalid property: {Err}" @@ -5367,6 +5383,14 @@ "id": "service '{Service}' not found in org {SvcOrg}", "translation": "service '{Service}' not found in org {SvcOrg}" }, + { + "id": "failed to unmarshal ClusterDeployment in 'hzn exchange service list' output: {Err}", + "translation": "failed to unmarshal ClusterDeployment in 'hzn exchange service list' output: {Err}" + }, + { + "id": "failed to marshal truncked ClusterDeployment in 'hzn exchange service list' output: {Err}", + "translation": "failed to marshal truncked ClusterDeployment in 'hzn exchange service list' output: {Err}" + }, { "id": "Ignoring -f because the clusterDeployment attribute is empty for this service.", "translation": "Ignoring -f because the clusterDeployment attribute is empty for this service." @@ -8091,10 +8115,6 @@ "id": "The SecretBindingCheck input cannot be null", "translation": "The SecretBindingCheck input cannot be null" }, - { - "id": "Node type '{NodeType}' does not support secret binding check.", - "translation": "Node type '{NodeType}' does not support secret binding check." - }, { "id": "No service versions with architecture {NodeArch} specified in the deployment policy or pattern.", "translation": "No service versions with architecture {NodeArch} specified in the deployment policy or pattern." @@ -8111,6 +8131,10 @@ "id": "Type Incompatible", "translation": "Type Incompatible" }, + { + "id": "{Msgcompatible}, Warning: {Reason}", + "translation": "{Msgcompatible}, Warning: {Reason}" + }, { "id": "Error getting services for all archetctures for {ServiceOrg}/{ServiceURL} version {Version}. {Err}", "translation": "Error getting services for all archetctures for {ServiceOrg}/{ServiceURL} version {Version}. {Err}" @@ -8147,10 +8171,6 @@ "id": "Failed to find the dependent services for {Org}/{URL} {Arch} {Version}. {Err}", "translation": "Failed to find the dependent services for {Org}/{URL} {Arch} {Version}. {Err}" }, - { - "id": "Secret binding for a cluster service is not supported.", - "translation": "Secret binding for a cluster service is not supported." - }, { "id": "No secret binding found for the following service secrets: {NbArray}.", "translation": "No secret binding found for the following service secrets: {NbArray}." @@ -8175,14 +8195,26 @@ "id": "Secret {VaultSecretName} does not exist in the secret manager for either org level or user level.", "translation": "Secret {VaultSecretName} does not exist in the secret manager for either org level or user level." }, + { + "id": "{Err}, use non node-level secret", + "translation": "{Err}, use non node-level secret" + }, { "id": "Error parsing secret name in the secret binding. {Errparse}", "translation": "Error parsing secret name in the secret binding. {Errparse}" }, + { + "id": "Error checking secret {VaultSecretName} for node {NName} in the secret manager. {Err}", + "translation": "Error checking secret {VaultSecretName} for node {NName} in the secret manager. {Err}" + }, { "id": "Error checking secret {VaultSecretName} in the secret manager. {Err}", "translation": "Error checking secret {VaultSecretName} in the secret manager. {Err}" }, + { + "id": "Node level secret {VaultSecretName} doesn't exist for node {NName}", + "translation": "Node level secret {VaultSecretName} doesn't exist for node {NName}" + }, { "id": "The binding secret name cannot be an empty string. The valid formats are: '' for the organization level secret and 'user//' for the user level secret.", "translation": "The binding secret name cannot be an empty string. The valid formats are: '' for the organization level secret and 'user//' for the user level secret." diff --git a/i18n_translation/original/anax.out.gotext.json.auto b/i18n_translation/original/anax.out.gotext.json.auto index 65544126a..dfd510371 100644 --- a/i18n_translation/original/anax.out.gotext.json.auto +++ b/i18n_translation/original/anax.out.gotext.json.auto @@ -2287,6 +2287,34 @@ ], "fuzzy": true }, + { + "id": "Name, or Org is empty string.", + "message": "Name, or Org is empty string.", + "translation": "Name, or Org is empty string.", + "translatorComment": "Copied from source.", + "fuzzy": true + }, + { + "id": "The serviceVersions array is empty.", + "message": "The serviceVersions array is empty.", + "translation": "The serviceVersions array is empty.", + "translatorComment": "Copied from source.", + "fuzzy": true + }, + { + "id": "retry_durations and retries cannot be zero if priority_value is set to non-zero value", + "message": "retry_durations and retries cannot be zero if priority_value is set to non-zero value", + "translation": "retry_durations and retries cannot be zero if priority_value is set to non-zero value", + "translatorComment": "Copied from source.", + "fuzzy": true + }, + { + "id": "retry_durations, retries and verified_durations cannot be non-zero value if priority_value is zero or not set", + "message": "retry_durations, retries and verified_durations cannot be non-zero value if priority_value is zero or not set", + "translation": "retry_durations, retries and verified_durations cannot be non-zero value if priority_value is zero or not set", + "translatorComment": "Copied from source.", + "fuzzy": true + }, { "id": "properties contains an invalid property: {Err}", "message": "properties contains an invalid property: {Err}", @@ -6504,23 +6532,6 @@ "translatorComment": "Copied from source.", "fuzzy": true }, - { - "id": "Node type '{NodeType}' does not support secret binding check.", - "message": "Node type '{NodeType}' does not support secret binding check.", - "translation": "Node type '{NodeType}' does not support secret binding check.", - "translatorComment": "Copied from source.", - "placeholders": [ - { - "id": "NodeType", - "string": "%[1]v", - "type": "string", - "underlyingType": "string", - "argNum": 1, - "expr": "nodeType" - } - ], - "fuzzy": true - }, { "id": "No service versions with architecture {NodeArch} specified in the deployment policy or pattern.", "message": "No service versions with architecture {NodeArch} specified in the deployment policy or pattern.", @@ -6559,6 +6570,31 @@ "translatorComment": "Copied from source.", "fuzzy": true }, + { + "id": "{Msgcompatible}, Warning: {Reason}", + "message": "{Msgcompatible}, Warning: {Reason}", + "translation": "{Msgcompatible}, Warning: {Reason}", + "translatorComment": "Copied from source.", + "placeholders": [ + { + "id": "Msgcompatible", + "string": "%[1]v", + "type": "string", + "underlyingType": "string", + "argNum": 1, + "expr": "msg_compatible" + }, + { + "id": "Reason", + "string": "%[2]v", + "type": "string", + "underlyingType": "string", + "argNum": 2, + "expr": "reason" + } + ], + "fuzzy": true + }, { "id": "Error getting services for all archetctures for {ServiceOrg}/{ServiceURL} version {Version}. {Err}", "message": "Error getting services for all archetctures for {ServiceOrg}/{ServiceURL} version {Version}. {Err}", @@ -6728,13 +6764,6 @@ ], "fuzzy": true }, - { - "id": "Secret binding for a cluster service is not supported.", - "message": "Secret binding for a cluster service is not supported.", - "translation": "Secret binding for a cluster service is not supported.", - "translatorComment": "Copied from source.", - "fuzzy": true - }, { "id": "No secret binding found for the following service secrets: {NbArray}.", "message": "No secret binding found for the following service secrets: {NbArray}.", @@ -6873,6 +6902,23 @@ ], "fuzzy": true }, + { + "id": "{Err}, use non node-level secret", + "message": "{Err}, use non node-level secret", + "translation": "{Err}, use non node-level secret", + "translatorComment": "Copied from source.", + "placeholders": [ + { + "id": "Err", + "string": "%[1]v", + "type": "error", + "underlyingType": "interface{Error() string}", + "argNum": 1, + "expr": "err" + } + ], + "fuzzy": true + }, { "id": "Error parsing secret name in the secret binding. {Errparse}", "message": "Error parsing secret name in the secret binding. {Errparse}", @@ -6890,6 +6936,39 @@ ], "fuzzy": true }, + { + "id": "Error checking secret {VaultSecretName} for node {NName} in the secret manager. {Err}", + "message": "Error checking secret {VaultSecretName} for node {NName} in the secret manager. {Err}", + "translation": "Error checking secret {VaultSecretName} for node {NName} in the secret manager. {Err}", + "translatorComment": "Copied from source.", + "placeholders": [ + { + "id": "VaultSecretName", + "string": "%[1]v", + "type": "string", + "underlyingType": "string", + "argNum": 1, + "expr": "vaultSecretName" + }, + { + "id": "NName", + "string": "%[2]v", + "type": "string", + "underlyingType": "string", + "argNum": 2, + "expr": "nName" + }, + { + "id": "Err", + "string": "%[3]v", + "type": "error", + "underlyingType": "interface{Error() string}", + "argNum": 3, + "expr": "err" + } + ], + "fuzzy": true + }, { "id": "Error checking secret {VaultSecretName} in the secret manager. {Err}", "message": "Error checking secret {VaultSecretName} in the secret manager. {Err}", @@ -6915,6 +6994,31 @@ ], "fuzzy": true }, + { + "id": "Node level secret {VaultSecretName} doesn't exist for node {NName}", + "message": "Node level secret {VaultSecretName} doesn't exist for node {NName}", + "translation": "Node level secret {VaultSecretName} doesn't exist for node {NName}", + "translatorComment": "Copied from source.", + "placeholders": [ + { + "id": "VaultSecretName", + "string": "%[1]v", + "type": "string", + "underlyingType": "string", + "argNum": 1, + "expr": "vaultSecretName" + }, + { + "id": "NName", + "string": "%[2]v", + "type": "string", + "underlyingType": "string", + "argNum": 2, + "expr": "nName" + } + ], + "fuzzy": true + }, { "id": "The binding secret name cannot be an empty string. The valid formats are: '\u003csecretname\u003e' for the organization level secret and 'user/\u003cusername\u003e/\u003csecretname\u003e' for the user level secret.", "message": "The binding secret name cannot be an empty string. The valid formats are: '\u003csecretname\u003e' for the organization level secret and 'user/\u003cusername\u003e/\u003csecretname\u003e' for the user level secret.", diff --git a/i18n_translation/original/hzn.out.gotext.json.auto b/i18n_translation/original/hzn.out.gotext.json.auto index b0cfa9fcd..223141552 100644 --- a/i18n_translation/original/hzn.out.gotext.json.auto +++ b/i18n_translation/original/hzn.out.gotext.json.auto @@ -1926,6 +1926,34 @@ ], "fuzzy": true }, + { + "id": "Name, or Org is empty string.", + "message": "Name, or Org is empty string.", + "translation": "Name, or Org is empty string.", + "translatorComment": "Copied from source.", + "fuzzy": true + }, + { + "id": "The serviceVersions array is empty.", + "message": "The serviceVersions array is empty.", + "translation": "The serviceVersions array is empty.", + "translatorComment": "Copied from source.", + "fuzzy": true + }, + { + "id": "retry_durations and retries cannot be zero if priority_value is set to non-zero value", + "message": "retry_durations and retries cannot be zero if priority_value is set to non-zero value", + "translation": "retry_durations and retries cannot be zero if priority_value is set to non-zero value", + "translatorComment": "Copied from source.", + "fuzzy": true + }, + { + "id": "retry_durations, retries and verified_durations cannot be non-zero value if priority_value is zero or not set", + "message": "retry_durations, retries and verified_durations cannot be non-zero value if priority_value is zero or not set", + "translation": "retry_durations, retries and verified_durations cannot be non-zero value if priority_value is zero or not set", + "translatorComment": "Copied from source.", + "fuzzy": true + }, { "id": "properties contains an invalid property: {Err}", "message": "properties contains an invalid property: {Err}", @@ -18543,6 +18571,40 @@ ], "fuzzy": true }, + { + "id": "failed to unmarshal ClusterDeployment in 'hzn exchange service list' output: {Err}", + "message": "failed to unmarshal ClusterDeployment in 'hzn exchange service list' output: {Err}", + "translation": "failed to unmarshal ClusterDeployment in 'hzn exchange service list' output: {Err}", + "translatorComment": "Copied from source.", + "placeholders": [ + { + "id": "Err", + "string": "%[1]v", + "type": "error", + "underlyingType": "interface{Error() string}", + "argNum": 1, + "expr": "err" + } + ], + "fuzzy": true + }, + { + "id": "failed to marshal truncked ClusterDeployment in 'hzn exchange service list' output: {Err}", + "message": "failed to marshal truncked ClusterDeployment in 'hzn exchange service list' output: {Err}", + "translation": "failed to marshal truncked ClusterDeployment in 'hzn exchange service list' output: {Err}", + "translatorComment": "Copied from source.", + "placeholders": [ + { + "id": "Err", + "string": "%[1]v", + "type": "error", + "underlyingType": "interface{Error() string}", + "argNum": 1, + "expr": "err" + } + ], + "fuzzy": true + }, { "id": "Ignoring -f because the clusterDeployment attribute is empty for this service.", "message": "Ignoring -f because the clusterDeployment attribute is empty for this service.", @@ -29719,23 +29781,6 @@ "translatorComment": "Copied from source.", "fuzzy": true }, - { - "id": "Node type '{NodeType}' does not support secret binding check.", - "message": "Node type '{NodeType}' does not support secret binding check.", - "translation": "Node type '{NodeType}' does not support secret binding check.", - "translatorComment": "Copied from source.", - "placeholders": [ - { - "id": "NodeType", - "string": "%[1]v", - "type": "string", - "underlyingType": "string", - "argNum": 1, - "expr": "nodeType" - } - ], - "fuzzy": true - }, { "id": "No service versions with architecture {NodeArch} specified in the deployment policy or pattern.", "message": "No service versions with architecture {NodeArch} specified in the deployment policy or pattern.", @@ -29774,6 +29819,31 @@ "translatorComment": "Copied from source.", "fuzzy": true }, + { + "id": "{Msgcompatible}, Warning: {Reason}", + "message": "{Msgcompatible}, Warning: {Reason}", + "translation": "{Msgcompatible}, Warning: {Reason}", + "translatorComment": "Copied from source.", + "placeholders": [ + { + "id": "Msgcompatible", + "string": "%[1]v", + "type": "string", + "underlyingType": "string", + "argNum": 1, + "expr": "msg_compatible" + }, + { + "id": "Reason", + "string": "%[2]v", + "type": "string", + "underlyingType": "string", + "argNum": 2, + "expr": "reason" + } + ], + "fuzzy": true + }, { "id": "Error getting services for all archetctures for {ServiceOrg}/{ServiceURL} version {Version}. {Err}", "message": "Error getting services for all archetctures for {ServiceOrg}/{ServiceURL} version {Version}. {Err}", @@ -29943,13 +30013,6 @@ ], "fuzzy": true }, - { - "id": "Secret binding for a cluster service is not supported.", - "message": "Secret binding for a cluster service is not supported.", - "translation": "Secret binding for a cluster service is not supported.", - "translatorComment": "Copied from source.", - "fuzzy": true - }, { "id": "No secret binding found for the following service secrets: {NbArray}.", "message": "No secret binding found for the following service secrets: {NbArray}.", @@ -30088,6 +30151,23 @@ ], "fuzzy": true }, + { + "id": "{Err}, use non node-level secret", + "message": "{Err}, use non node-level secret", + "translation": "{Err}, use non node-level secret", + "translatorComment": "Copied from source.", + "placeholders": [ + { + "id": "Err", + "string": "%[1]v", + "type": "error", + "underlyingType": "interface{Error() string}", + "argNum": 1, + "expr": "err" + } + ], + "fuzzy": true + }, { "id": "Error parsing secret name in the secret binding. {Errparse}", "message": "Error parsing secret name in the secret binding. {Errparse}", @@ -30105,6 +30185,39 @@ ], "fuzzy": true }, + { + "id": "Error checking secret {VaultSecretName} for node {NName} in the secret manager. {Err}", + "message": "Error checking secret {VaultSecretName} for node {NName} in the secret manager. {Err}", + "translation": "Error checking secret {VaultSecretName} for node {NName} in the secret manager. {Err}", + "translatorComment": "Copied from source.", + "placeholders": [ + { + "id": "VaultSecretName", + "string": "%[1]v", + "type": "string", + "underlyingType": "string", + "argNum": 1, + "expr": "vaultSecretName" + }, + { + "id": "NName", + "string": "%[2]v", + "type": "string", + "underlyingType": "string", + "argNum": 2, + "expr": "nName" + }, + { + "id": "Err", + "string": "%[3]v", + "type": "error", + "underlyingType": "interface{Error() string}", + "argNum": 3, + "expr": "err" + } + ], + "fuzzy": true + }, { "id": "Error checking secret {VaultSecretName} in the secret manager. {Err}", "message": "Error checking secret {VaultSecretName} in the secret manager. {Err}", @@ -30130,6 +30243,31 @@ ], "fuzzy": true }, + { + "id": "Node level secret {VaultSecretName} doesn't exist for node {NName}", + "message": "Node level secret {VaultSecretName} doesn't exist for node {NName}", + "translation": "Node level secret {VaultSecretName} doesn't exist for node {NName}", + "translatorComment": "Copied from source.", + "placeholders": [ + { + "id": "VaultSecretName", + "string": "%[1]v", + "type": "string", + "underlyingType": "string", + "argNum": 1, + "expr": "vaultSecretName" + }, + { + "id": "NName", + "string": "%[2]v", + "type": "string", + "underlyingType": "string", + "argNum": 2, + "expr": "nName" + } + ], + "fuzzy": true + }, { "id": "The binding secret name cannot be an empty string. The valid formats are: '\u003csecretname\u003e' for the organization level secret and 'user/\u003cusername\u003e/\u003csecretname\u003e' for the user level secret.", "message": "The binding secret name cannot be an empty string. The valid formats are: '\u003csecretname\u003e' for the organization level secret and 'user/\u003cusername\u003e/\u003csecretname\u003e' for the user level secret.", diff --git a/i18n_translation/translated/anax.out.gotext_fr.json b/i18n_translation/translated/anax.out.gotext_fr.json index 3107dbadc..eaf700d96 100644 --- a/i18n_translation/translated/anax.out.gotext_fr.json +++ b/i18n_translation/translated/anax.out.gotext_fr.json @@ -471,6 +471,22 @@ "id": "Error converting the selections into Selectors: {Err}", "translation": "Erreur lors de la conversion des sélections en sélecteurs : {Err}" }, + { + "id": "Name, or Org is empty string.", + "translation": "Le nom ou l'organisation est une chaîne vide." + }, + { + "id": "The serviceVersions array is empty.", + "translation": "Le tableau serviceVersions est vide." + }, + { + "id": "retry_durations and retries cannot be zero if priority_value is set to non-zero value", + "translation": "retry_durations et retries ne peuvent pas être égales à zéro si priority_value est défini sur une valeur différente de zéro" + }, + { + "id": "retry_durations, retries and verified_durations cannot be non-zero value if priority_value is zero or not set", + "translation": "retry_durations, retries et verified_durations ne peuvent pas avoir une valeur différente de zéro si priority_value est égale à zéro ou n'est pas définie" + }, { "id": "properties contains an invalid property: {Err}", "translation": "Les propriétés comportent une propriété non valide : {Err}" @@ -1315,10 +1331,6 @@ "id": "The SecretBindingCheck input cannot be null", "translation": "L'entrée SecretBindingCheck ne peut pas avoir pour valeur NULL" }, - { - "id": "Node type '{NodeType}' does not support secret binding check.", - "translation": "Le type de noeud '{NodeType}' ne prend pas en charge la vérification de liaison de secret." - }, { "id": "No service versions with architecture {NodeArch} specified in the deployment policy or pattern.", "translation": "Aucune version de service avec l'architecture {NodeArch} n'est spécifiée dans la règle de déploiement ou le pattern." @@ -1335,6 +1347,10 @@ "id": "Type Incompatible", "translation": "Type incompatible" }, + { + "id": "{Msgcompatible}, Warning: {Reason}", + "translation": "{Msgcompatible}, Avertissement : {Reason}" + }, { "id": "Error getting services for all archetctures for {ServiceOrg}/{ServiceURL} version {Version}. {Err}", "translation": "Erreur lors de l'obtention des services pour toutes les architectures de {ServiceOrg}/{ServiceURL} version {Version}. {Err}" @@ -1371,10 +1387,6 @@ "id": "Failed to find the dependent services for {Org}/{URL} {Arch} {Version}. {Err}", "translation": "Les services dépendants sont introuvables pour {Org}/{URL} {Arch} {Version}. {Err}" }, - { - "id": "Secret binding for a cluster service is not supported.", - "translation": "La liaison de secret pour un service de cluster n'est pas prise en charge." - }, { "id": "No secret binding found for the following service secrets: {NbArray}.", "translation": "Aucune liaison de secret n'a été trouvée pour les secrets de service suivants : {NbArray}." @@ -1399,14 +1411,26 @@ "id": "Secret {VaultSecretName} does not exist in the secret manager for either org level or user level.", "translation": "Le secret {VaultSecretName} n'existe pas dans le gestionnaire de secrets au niveau de l'organisation ou de l'utilisateur." }, + { + "id": "{Err}, use non node-level secret", + "translation": "{Err}, utilisent un secret qui n'est pas au niveau du noeud" + }, { "id": "Error parsing secret name in the secret binding. {Errparse}", "translation": "Erreur lors de l'analyse syntaxique du nom de secret dans la liaison de secret. {Errparse}" }, + { + "id": "Error checking secret {VaultSecretName} for node {NName} in the secret manager. {Err}", + "translation": "Erreur lors de la vérification du secret {VaultSecretName} pour le noeud {NName} dans le gestionnaire de secrets. {Err}" + }, { "id": "Error checking secret {VaultSecretName} in the secret manager. {Err}", "translation": "Erreur lors de la vérification du secret {VaultSecretName} dans le gestionnaire de secrets. {Err}" }, + { + "id": "Node level secret {VaultSecretName} doesn't exist for node {NName}", + "translation": "Le secret au niveau du noeud {VaultSecretName} n'existe pas pour le noeud {NName}" + }, { "id": "The binding secret name cannot be an empty string. The valid formats are: '' for the organization level secret and 'user//' for the user level secret.", "translation": "Le nom du secret de liaison ne peut pas être une chaîne vide. Les formats valides sont les suivants : ''pour le secret de niveau organisation et 'user//' pour le secret de niveau utilisateur." diff --git a/i18n_translation/translated/hzn.out.gotext_fr.json b/i18n_translation/translated/hzn.out.gotext_fr.json index d139a79c1..8f01521e8 100644 --- a/i18n_translation/translated/hzn.out.gotext_fr.json +++ b/i18n_translation/translated/hzn.out.gotext_fr.json @@ -403,6 +403,22 @@ "id": "Error converting the selections into Selectors: {Err}", "translation": "Erreur lors de la conversion des sélections en sélecteurs : {Err}" }, + { + "id": "Name, or Org is empty string.", + "translation": "Le nom ou l'organisation est une chaîne vide." + }, + { + "id": "The serviceVersions array is empty.", + "translation": "Le tableau serviceVersions est vide." + }, + { + "id": "retry_durations and retries cannot be zero if priority_value is set to non-zero value", + "translation": "retry_durations et retries ne peuvent pas être égales à zéro si priority_value est défini sur une valeur différente de zéro" + }, + { + "id": "retry_durations, retries and verified_durations cannot be non-zero value if priority_value is zero or not set", + "translation": "retry_durations, retries et verified_durations ne peuvent pas avoir une valeur différente de zéro si priority_value est égale à zéro ou n'est pas définie" + }, { "id": "properties contains an invalid property: {Err}", "translation": "Les propriétés comportent une propriété non valide : {Err}" @@ -5367,6 +5383,14 @@ "id": "service '{Service}' not found in org {SvcOrg}", "translation": "Service '{Service}' introuvable dans l'organisation {SvcOrg}" }, + { + "id": "failed to unmarshal ClusterDeployment in 'hzn exchange service list' output: {Err}", + "translation": "Echec de la désérialisation de ClusterDeployment dans la sortie 'hzn exchange service list' : {Err}" + }, + { + "id": "failed to marshal truncked ClusterDeployment in 'hzn exchange service list' output: {Err}", + "translation": "Echec de la sérialisation de ClusterDeployment tronqué dans la sortie 'hzn exchange service list' : {Err}" + }, { "id": "Ignoring -f because the clusterDeployment attribute is empty for this service.", "translation": "-f est ignoré car l'attribut clusterDeployment est vide pour ce service." @@ -8091,10 +8115,6 @@ "id": "The SecretBindingCheck input cannot be null", "translation": "L'entrée SecretBindingCheck ne peut pas avoir pour valeur NULL" }, - { - "id": "Node type '{NodeType}' does not support secret binding check.", - "translation": "Le type de noeud '{NodeType}' ne prend pas en charge la vérification de liaison de secret." - }, { "id": "No service versions with architecture {NodeArch} specified in the deployment policy or pattern.", "translation": "Aucune version de service avec l'architecture {NodeArch} n'est spécifiée dans la règle de déploiement ou le pattern." @@ -8111,6 +8131,10 @@ "id": "Type Incompatible", "translation": "Type incompatible" }, + { + "id": "{Msgcompatible}, Warning: {Reason}", + "translation": "{Msgcompatible}, Avertissement : {Reason}" + }, { "id": "Error getting services for all archetctures for {ServiceOrg}/{ServiceURL} version {Version}. {Err}", "translation": "Erreur lors de l'obtention des services pour toutes les architectures de {ServiceOrg}/{ServiceURL} version {Version}. {Err}" @@ -8147,10 +8171,6 @@ "id": "Failed to find the dependent services for {Org}/{URL} {Arch} {Version}. {Err}", "translation": "Les services dépendants sont introuvables pour {Org}/{URL} {Arch} {Version}. {Err}" }, - { - "id": "Secret binding for a cluster service is not supported.", - "translation": "La liaison de secret pour un service de cluster n'est pas prise en charge." - }, { "id": "No secret binding found for the following service secrets: {NbArray}.", "translation": "Aucune liaison de secret n'a été trouvée pour les secrets de service suivants : {NbArray}." @@ -8175,14 +8195,26 @@ "id": "Secret {VaultSecretName} does not exist in the secret manager for either org level or user level.", "translation": "Le secret {VaultSecretName} n'existe pas dans le gestionnaire de secrets au niveau de l'organisation ou de l'utilisateur." }, + { + "id": "{Err}, use non node-level secret", + "translation": "{Err}, utilisent un secret qui n'est pas au niveau du noeud" + }, { "id": "Error parsing secret name in the secret binding. {Errparse}", "translation": "Erreur lors de l'analyse syntaxique du nom de secret dans la liaison de secret. {Errparse}" }, + { + "id": "Error checking secret {VaultSecretName} for node {NName} in the secret manager. {Err}", + "translation": "Erreur lors de la vérification du secret {VaultSecretName} pour le noeud {NName} dans le gestionnaire de secrets. {Err}" + }, { "id": "Error checking secret {VaultSecretName} in the secret manager. {Err}", "translation": "Erreur lors de la vérification du secret {VaultSecretName} dans le gestionnaire de secrets. {Err}" }, + { + "id": "Node level secret {VaultSecretName} doesn't exist for node {NName}", + "translation": "Le secret au niveau du noeud {VaultSecretName} n'existe pas pour le noeud {NName}" + }, { "id": "The binding secret name cannot be an empty string. The valid formats are: '' for the organization level secret and 'user//' for the user level secret.", "translation": "Le nom du secret de liaison ne peut pas être une chaîne vide. Les formats valides sont les suivants : ''pour le secret de niveau organisation et 'user//' pour le secret de niveau utilisateur." diff --git a/kube_operator/api_objects.go b/kube_operator/api_objects.go index 842de9ad9..c6d13a2dd 100644 --- a/kube_operator/api_objects.go +++ b/kube_operator/api_objects.go @@ -36,7 +36,7 @@ type APIObjectInterface interface { // Sort a slice of k8s api objects by kind of object // Returns a map of object type names to api object interfaces types, the namespace to be used for the operator, and an error if one occurs // Also verifies that all objects are named so they can be found and uninstalled -func sortAPIObjects(allObjects []APIObjects, customResources map[string][]*unstructured.Unstructured, metadata map[string]interface{}, envVarMap map[string]string, secretsMap map[string]string, agreementId string, crInstallTimeout int64) (map[string][]APIObjectInterface, string, error) { +func sortAPIObjects(allObjects []APIObjects, customResources map[string][]*unstructured.Unstructured, metadata map[string]interface{}, envVarMap map[string]string, fssAuthFilePath string, fssCertFilePath string, secretsMap map[string]string, agreementId string, crInstallTimeout int64) (map[string][]APIObjectInterface, string, error) { namespace := "" // get the namespace from metadata @@ -123,7 +123,7 @@ func sortAPIObjects(allObjects []APIObjects, customResources map[string][]*unstr return objMap, namespace, fmt.Errorf(kwlog(fmt.Sprintf("Error: multiple namespaces specified in operator: %s and %s", namespace, typedDeployment.ObjectMeta.Namespace))) } } - newDeployment := DeploymentAppsV1{DeploymentObject: typedDeployment, EnvVarMap: envVarMap, ServiceSecrets: secretsMap, AgreementId: agreementId} + newDeployment := DeploymentAppsV1{DeploymentObject: typedDeployment, EnvVarMap: envVarMap, FssAuthFilePath: fssAuthFilePath, FssCertFilePath: fssCertFilePath, ServiceSecrets: secretsMap, AgreementId: agreementId} if newDeployment.Name() != "" { glog.V(4).Infof(kwlog(fmt.Sprintf("Found kubernetes deployment object %s.", newDeployment.Name()))) objMap[K8S_DEPLOYMENT_TYPE] = append(objMap[K8S_DEPLOYMENT_TYPE], newDeployment) @@ -385,20 +385,18 @@ type ClusterRolebindingRbacV1 struct { func (crb ClusterRolebindingRbacV1) Install(c KubeClient, namespace string) error { glog.V(3).Infof(kwlog(fmt.Sprintf("creating cluster role binding %v", crb))) - // checking the serviceaccount for clusterrolebinding if it is namespace-scoped agent: - // - If the namespace of serviceaccount is defined in yaml, but is different from namespace for operator, replace the sa namespace with namespace to deploy operator. - if cutil.IsNamespaceScoped() { - // normalize the namespace of service account for namespace scoped agent - subs := []rbacv1.Subject{} - for _, sub := range crb.ClusterRolebindingObject.Subjects { - rb_sub := &sub - if sub.Namespace != "" && sub.Namespace != namespace { - rb_sub.Namespace = namespace - } - subs = append(subs, *rb_sub) + // checking the serviceaccount for clusterrolebinding: + // - namespace-scoped agent: Normalize the namespace of service account for namespace scoped agent. If the namespace of serviceaccount is defined in yaml, but is different from namespace for operator, replace the sa namespace with namespace to deploy operator. + // - cluster-scoped agent: If the namespace of the serviceaccount is absent, add namespace + subs := []rbacv1.Subject{} + for _, sub := range crb.ClusterRolebindingObject.Subjects { + rb_sub := &sub + if (cutil.IsNamespaceScoped() && sub.Namespace != "" && sub.Namespace != namespace) || (!cutil.IsNamespaceScoped() && sub.Namespace == "") { + rb_sub.Namespace = namespace } - crb.ClusterRolebindingObject.Subjects = subs + subs = append(subs, *rb_sub) } + crb.ClusterRolebindingObject.Subjects = subs // get clusterrolebinding existingCRB, err := c.Client.RbacV1().ClusterRoleBindings().Get(context.Background(), crb.Name(), metav1.GetOptions{}) @@ -631,6 +629,8 @@ func (sa ServiceAccountCoreV1) Namespace() string { type DeploymentAppsV1 struct { DeploymentObject *appsv1.Deployment EnvVarMap map[string]string + FssAuthFilePath string + FssCertFilePath string ServiceSecrets map[string]string AgreementId string } @@ -643,21 +643,50 @@ func (d DeploymentAppsV1) Install(c KubeClient, namespace string) error { } // The ESS is not supported in edge cluster services, so for now, remove the ESS env vars. - envAdds := cutil.RemoveESSEnvVars(d.EnvVarMap, config.ENVVAR_PREFIX) + //envAdds := cutil.RemoveESSEnvVars(d.EnvVarMap, config.ENVVAR_PREFIX) + cutil.SetESSEnvVarsForClusterAgent(d.EnvVarMap, config.ENVVAR_PREFIX, d.AgreementId) // Create the config map. - mapName, err := c.CreateConfigMap(envAdds, d.AgreementId, namespace) + mapName, err := c.CreateConfigMap(d.EnvVarMap, d.AgreementId, namespace) if err != nil && errors.IsAlreadyExists(err) { - d.Uninstall(c, namespace) - mapName, err = c.CreateConfigMap(envAdds, d.AgreementId, namespace) + c.DeleteConfigMap(d.AgreementId, namespace) + mapName, err = c.CreateConfigMap(d.EnvVarMap, d.AgreementId, namespace) } if err != nil { return err } + // create k8s secrets object from ess auth file. d.FssAuthFilePath == "" if kubeworker is updating service vault secret + if d.FssAuthFilePath != "" { + essAuthSecretName, err := c.CreateESSAuthSecrets(d.FssAuthFilePath, d.AgreementId, namespace) + if err != nil && errors.IsAlreadyExists(err) { + c.DeleteESSAuthSecrets(d.AgreementId, namespace) + essAuthSecretName, _ = c.CreateESSAuthSecrets(d.FssAuthFilePath, d.AgreementId, namespace) + } + glog.V(3).Infof(kwlog(fmt.Sprintf("ess auth secret %v is created under namespace: %v", essAuthSecretName, namespace))) + } + + if d.FssCertFilePath != "" { + essCertSecretName, err := c.CreateESSCertSecrets(d.FssCertFilePath, d.AgreementId, namespace) + if err != nil && errors.IsAlreadyExists(err) { + c.DeleteESSCertSecrets(d.AgreementId, namespace) + essCertSecretName, _ = c.CreateESSCertSecrets(d.FssCertFilePath, d.AgreementId, namespace) + } + glog.V(3).Infof(kwlog(fmt.Sprintf("ess cert secret %v is created under namespace: %v", essCertSecretName, namespace))) + } + + // create MMS pvc + pvcName, err := c.CreateMMSPVC(d.EnvVarMap, d.AgreementId, namespace) + if err != nil && errors.IsAlreadyExists(err) { + c.DeleteMMSPVC(d.AgreementId, namespace) + pvcName, _ = c.CreateMMSPVC(d.EnvVarMap, d.AgreementId, namespace) + } + glog.V(3).Infof(kwlog(fmt.Sprintf("MMS pvc %v is created under namespace: %v", pvcName, namespace))) + // Let the operator know about the config map dWithEnv := addConfigMapVarToDeploymentObject(*d.DeploymentObject, mapName) + // handle service vault secrets if len(d.ServiceSecrets) > 0 { glog.V(3).Infof(kwlog(fmt.Sprintf("creating k8s secrets for service secret %v", d.ServiceSecrets))) @@ -669,7 +698,7 @@ func (d DeploymentAppsV1) Install(c KubeClient, namespace string) error { secretsName, err := c.CreateK8SSecrets(decodedSecrets, d.AgreementId, namespace) if err != nil && errors.IsAlreadyExists(err) { - d.Uninstall(c, namespace) + c.DeleteK8SSecrets(d.AgreementId, namespace) secretsName, err = c.CreateK8SSecrets(d.ServiceSecrets, d.AgreementId, namespace) } if err != nil { @@ -682,7 +711,7 @@ func (d DeploymentAppsV1) Install(c KubeClient, namespace string) error { _, err = c.Client.AppsV1().Deployments(namespace).Create(context.Background(), &dWithEnv, metav1.CreateOptions{}) if err != nil && errors.IsAlreadyExists(err) { d.Uninstall(c, namespace) - mapName, err = c.CreateConfigMap(envAdds, d.AgreementId, namespace) + _, _ = c.CreateConfigMap(d.EnvVarMap, d.AgreementId, namespace) _, err = c.Client.AppsV1().Deployments(namespace).Create(context.Background(), &dWithEnv, metav1.CreateOptions{}) } if err != nil { @@ -733,21 +762,31 @@ func (d DeploymentAppsV1) Uninstall(c KubeClient, namespace string) { glog.Errorf(kwlog(fmt.Sprintf("unable to delete deployment %s. Error: %v", d.DeploymentObject.ObjectMeta.Name, err))) } - configMapName := fmt.Sprintf("%s-%s", HZN_ENV_VARS, d.AgreementId) - glog.V(3).Infof(kwlog(fmt.Sprintf("deleting config map %v in namespace %v", configMapName, namespace))) - // Delete the agreement config map - err = c.Client.CoreV1().ConfigMaps(namespace).Delete(context.Background(), configMapName, metav1.DeleteOptions{}) - if err != nil { - glog.Errorf(kwlog(fmt.Sprintf("unable to delete config map %s. Error: %v", configMapName, err))) + glog.V(3).Infof(kwlog(fmt.Sprintf("deleting config map for agreement %v in namespace %v", d.AgreementId, namespace))) + if err = c.DeleteConfigMap(d.AgreementId, namespace); err != nil { + glog.Errorf(kwlog(err.Error())) } - // delete the secrets contains agreement service vault secrets - secretsName := fmt.Sprintf("%s-%s", HZN_SERVICE_SECRETS, d.AgreementId) - glog.V(3).Infof(kwlog(fmt.Sprintf("deleting secrets %v in namespace %v", secretsName, namespace))) - err = c.Client.CoreV1().Secrets(namespace).Delete(context.Background(), secretsName, metav1.DeleteOptions{}) - if err != nil { - glog.Errorf(kwlog(fmt.Sprintf("unable to delete secrets %s in namespace %v. Error: %v", secretsName, namespace, err))) + glog.V(3).Infof(kwlog(fmt.Sprintf("deleting ess auth secret for agreement %v in namespace %v", d.AgreementId, namespace))) + if err = c.DeleteESSAuthSecrets(d.AgreementId, namespace); err != nil { + glog.Errorf(kwlog(err.Error())) + } + + glog.V(3).Infof(kwlog(fmt.Sprintf("deleting ess cert secret for agreement %v in namespace %v", d.AgreementId, namespace))) + if err = c.DeleteESSCertSecrets(d.AgreementId, namespace); err != nil { + glog.Errorf(kwlog(err.Error())) + } + + glog.V(3).Infof(kwlog(fmt.Sprintf("deleting secrets for agreement %v in namespace %v", d.AgreementId, namespace))) + if err = c.DeleteK8SSecrets(d.AgreementId, namespace); err != nil { + glog.Errorf(kwlog(err.Error())) + } + + glog.V(3).Infof(kwlog(fmt.Sprintf("deleting mms pvc for agreement %v in namespace %v", d.AgreementId, namespace))) + if err = c.DeleteMMSPVC(d.AgreementId, namespace); err != nil { + glog.Errorf(kwlog(err.Error())) } + } // Status will be the status of the operator pod @@ -758,7 +797,7 @@ func (d DeploymentAppsV1) Status(c KubeClient, namespace string) (interface{}, e return nil, err } else if podList == nil || len(podList.Items) == 0 { labelSelector := metav1.LabelSelector{MatchLabels: d.DeploymentObject.Spec.Selector.MatchLabels} - podList, err = c.Client.CoreV1().Pods(namespace).List(context.Background(), metav1.ListOptions{LabelSelector: labels.Set(labelSelector.MatchLabels).String()}) + podList, _ = c.Client.CoreV1().Pods(namespace).List(context.Background(), metav1.ListOptions{LabelSelector: labels.Set(labelSelector.MatchLabels).String()}) } return podList, nil } diff --git a/kube_operator/client.go b/kube_operator/client.go index b986b5836..f31efb534 100644 --- a/kube_operator/client.go +++ b/kube_operator/client.go @@ -22,6 +22,8 @@ import ( rbacv1 "k8s.io/api/rbac/v1" v1scheme "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" v1beta1scheme "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/conversion" @@ -31,6 +33,7 @@ import ( dynamic "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" + "os" "reflect" "strings" ) @@ -46,20 +49,34 @@ const ( SECRETS_VOLUME_NAME = "service-secrets-vol" - K8S_CLUSTER_ROLE_TYPE = "ClusterRole" - K8S_CLUSTER_ROLEBINDING_TYPE = "ClusterRoleBinding" - K8S_ROLE_TYPE = "Role" - K8S_ROLEBINDING_TYPE = "RoleBinding" - K8S_DEPLOYMENT_TYPE = "Deployment" - K8S_SERVICEACCOUNT_TYPE = "ServiceAccount" - K8S_CRD_TYPE = "CustomResourceDefinition" - K8S_NAMESPACE_TYPE = "Namespace" - K8S_UNSTRUCTURED_TYPE = "Unstructured" - K8S_OLM_OPERATOR_GROUP_TYPE = "OperatorGroup" + MMS_VOLUME_NAME = "mms-shared-storage" + + K8S_CLUSTER_ROLE_TYPE = "ClusterRole" + K8S_CLUSTER_ROLEBINDING_TYPE = "ClusterRoleBinding" + K8S_ROLE_TYPE = "Role" + K8S_ROLEBINDING_TYPE = "RoleBinding" + K8S_DEPLOYMENT_TYPE = "Deployment" + K8S_SERVICEACCOUNT_TYPE = "ServiceAccount" + K8S_CRD_TYPE = "CustomResourceDefinition" + K8S_NAMESPACE_TYPE = "Namespace" + K8S_UNSTRUCTURED_TYPE = "Unstructured" + K8S_OLM_OPERATOR_GROUP_TYPE = "OperatorGroup" + K8S_MMS_SHARED_PVC_NAME = "mms-shared-storage-pvc" + STORAGE_CLASS_USERINPUT_NAME = "MMS_K8S_STORAGE_CLASS" + PVC_SIZE_USERINPUT_NAME = "MMS_K8S_STORAGE_SIZE" + PVC_ACCESS_MODE_USERINPUT_NAME = "MMS_K8S_PVC_ACCESS_MODE" + DEFAULT_PVC_SIZE_IN_STRING = "10" +) + +var ( + accessModeMap = map[string]corev1.PersistentVolumeAccessMode{ + "ReadWriteOnce": corev1.ReadWriteOnce, + "ReadWriteMany": corev1.ReadWriteMany, + } ) func getBaseK8sKinds() []string { - return []string{K8S_NAMESPACE_TYPE, K8S_CLUSTER_ROLE_TYPE, K8S_CLUSTER_ROLEBINDING_TYPE, K8S_ROLE_TYPE, K8S_ROLEBINDING_TYPE, K8S_DEPLOYMENT_TYPE, K8S_SERVICEACCOUNT_TYPE, K8S_CRD_TYPE} + return []string{K8S_NAMESPACE_TYPE, K8S_CLUSTER_ROLE_TYPE, K8S_CLUSTER_ROLEBINDING_TYPE, K8S_ROLE_TYPE, K8S_ROLEBINDING_TYPE, K8S_SERVICEACCOUNT_TYPE, K8S_CRD_TYPE, K8S_DEPLOYMENT_TYPE} } func getDangerKinds() []string { @@ -130,9 +147,9 @@ func NewDynamicKubeClient() (dynamic.Interface, error) { } // Install creates the objects specified in the operator deployment in the cluster and creates the custom resource to start the operator -func (c KubeClient) Install(tar string, metadata map[string]interface{}, envVars map[string]string, secretsMap map[string]string, agId string, reqNamespace string, crInstallTimeout int64) error { +func (c KubeClient) Install(tar string, metadata map[string]interface{}, envVars map[string]string, fssAuthFilePath string, fssCertFilePath string, secretsMap map[string]string, agId string, reqNamespace string, crInstallTimeout int64) error { - apiObjMap, opNamespace, err := ProcessDeployment(tar, metadata, envVars, secretsMap, agId, crInstallTimeout) + apiObjMap, opNamespace, err := ProcessDeployment(tar, metadata, envVars, fssAuthFilePath, fssCertFilePath, secretsMap, agId, crInstallTimeout) if err != nil { return err } @@ -181,7 +198,7 @@ func (c KubeClient) Install(tar string, metadata map[string]interface{}, envVars // Install creates the objects specified in the operator deployment in the cluster and creates the custom resource to start the operator func (c KubeClient) Uninstall(tar string, metadata map[string]interface{}, agId string, reqNamespace string) error { - apiObjMap, opNamespace, err := ProcessDeployment(tar, metadata, map[string]string{}, map[string]string{}, agId, 0) + apiObjMap, opNamespace, err := ProcessDeployment(tar, metadata, map[string]string{}, "", "", map[string]string{}, agId, 0) if err != nil { return err } @@ -220,7 +237,7 @@ func (c KubeClient) Uninstall(tar string, metadata map[string]interface{}, agId return nil } func (c KubeClient) OperatorStatus(tar string, metadata map[string]interface{}, agId string, reqNamespace string) (interface{}, error) { - apiObjMap, opNamespace, err := ProcessDeployment(tar, metadata, map[string]string{}, map[string]string{}, agId, 0) + apiObjMap, opNamespace, err := ProcessDeployment(tar, metadata, map[string]string{}, "", "", map[string]string{}, agId, 0) if err != nil { return nil, err } @@ -238,7 +255,7 @@ func (c KubeClient) OperatorStatus(tar string, metadata map[string]interface{}, } func (c KubeClient) Status(tar string, metadata map[string]interface{}, agId string, reqNamespace string) ([]ContainerStatus, error) { - apiObjMap, opNamespace, err := ProcessDeployment(tar, metadata, map[string]string{}, map[string]string{}, agId, 0) + apiObjMap, opNamespace, err := ProcessDeployment(tar, metadata, map[string]string{}, "", "", map[string]string{}, agId, 0) if err != nil { return nil, err } @@ -294,7 +311,7 @@ func (c KubeClient) Update(tar string, metadata map[string]interface{}, agId str } // Current implementaion only updatedSecrets will be passed into this function - apiObjMap, opNamespace, err := ProcessDeployment(tar, metadata, updatedEnv, updatedSecretsMap, agId, 0) + apiObjMap, opNamespace, err := ProcessDeployment(tar, metadata, updatedEnv, "", "", updatedSecretsMap, agId, 0) if err != nil { return err } @@ -315,7 +332,7 @@ func (c KubeClient) Update(tar string, metadata map[string]interface{}, agId str } // processDeployment takes the deployment string and converts it to a map with the k8s objects, the namespace to be used, and an error if one occurs -func ProcessDeployment(tar string, metadata map[string]interface{}, envVars map[string]string, secretsMap map[string]string, agId string, crInstallTimeout int64) (map[string][]APIObjectInterface, string, error) { +func ProcessDeployment(tar string, metadata map[string]interface{}, envVars map[string]string, fssAuthFilePath string, fssCertFilePath string, secretsMap map[string]string, agId string, crInstallTimeout int64) (map[string][]APIObjectInterface, string, error) { // Read the yaml files from the commpressed tar files yamls, err := getYamlFromTarGz(tar) if err != nil { @@ -338,7 +355,7 @@ func ProcessDeployment(tar string, metadata map[string]interface{}, envVars map[ } // Sort the k8s api objects by kind - return sortAPIObjects(k8sObjs, customResourceKindMap, metadata, envVars, secretsMap, agId, crInstallTimeout) + return sortAPIObjects(k8sObjs, customResourceKindMap, metadata, envVars, fssAuthFilePath, fssCertFilePath, secretsMap, agId, crInstallTimeout) } // CreateConfigMap will create a config map with the provided environment variable map @@ -350,6 +367,7 @@ func (c KubeClient) CreateConfigMap(envVars map[string]string, agId string, name } delete(envVars, "") } + // hzn-env-vars- hznEnvConfigMap := corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("%s-%s", HZN_ENV_VARS, agId)}, Data: envVars} res, err := c.Client.CoreV1().ConfigMaps(namespace).Create(context.Background(), &hznEnvConfigMap, metav1.CreateOptions{}) if err != nil { @@ -358,6 +376,87 @@ func (c KubeClient) CreateConfigMap(envVars map[string]string, agId string, name return res.ObjectMeta.Name, nil } +// CreateConfigMap will create a config map with the provided environment variable map +func (c KubeClient) DeleteConfigMap(agId string, namespace string) error { + // hzn-env-vars- + hznEnvConfigmapName := fmt.Sprintf("%s-%s", HZN_ENV_VARS, agId) + err := c.Client.CoreV1().ConfigMaps(namespace).Delete(context.Background(), hznEnvConfigmapName, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("Error: failed to delete config map for %s: %v", agId, err) + } + return nil +} + +// CreateESSSecret will create a k8s secrets object from the ess auth file +func (c KubeClient) CreateESSAuthSecrets(fssAuthFilePath string, agId string, namespace string) (string, error) { + if essAuth, err := os.Open(fssAuthFilePath); err != nil { + return "", err + } else if essAuthBytes, err := ioutil.ReadAll(essAuth); err != nil { + return "", err + } else { + secretData := make(map[string][]byte) + secretData[config.HZN_FSS_AUTH_FILE] = essAuthBytes + fssSecret := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", config.HZN_FSS_AUTH_PATH, agId), // ess-auth- + Namespace: namespace, + }, + Data: secretData, + } + res, err := c.Client.CoreV1().Secrets(namespace).Create(context.Background(), &fssSecret, metav1.CreateOptions{}) + if err != nil { + return "", fmt.Errorf("Error: failed to create ess auth secret for %s: %v", agId, err) + } + return res.ObjectMeta.Name, nil + } + +} + +func (c KubeClient) DeleteESSAuthSecrets(agId string, namespace string) error { + essAuthSecretName := fmt.Sprintf("%s-%s", config.HZN_FSS_AUTH_PATH, agId) + err := c.Client.CoreV1().Secrets(namespace).Delete(context.Background(), essAuthSecretName, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("Error: failed to delete ess auth secret for %s: %v", agId, err) + } + return nil +} + +func (c KubeClient) CreateESSCertSecrets(fssCertFilePath string, agId string, namespace string) (string, error) { + if essCert, err := os.Open(fssCertFilePath); err != nil { + return "", err + } else if essCertBytes, err := ioutil.ReadAll(essCert); err != nil { + return "", err + } else { + secretData := make(map[string][]byte) + secretData[config.HZN_FSS_CERT_FILE] = essCertBytes + certSecret := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s", config.HZN_FSS_CERT_PATH, agId), // ess-cert- + Namespace: namespace, + }, + Data: secretData, + } + + res, err := c.Client.CoreV1().Secrets(namespace).Create(context.Background(), &certSecret, metav1.CreateOptions{}) + if err != nil && errors.IsAlreadyExists(err) { + _, err = c.Client.CoreV1().Secrets(namespace).Update(context.Background(), &certSecret, metav1.UpdateOptions{}) + } + if err != nil { + return "", fmt.Errorf("Error: failed to create ess cert secret for %s: %v", agId, err) + } + return res.ObjectMeta.Name, nil + } +} + +func (c KubeClient) DeleteESSCertSecrets(agId string, namespace string) error { + essCertSecretName := fmt.Sprintf("%s-%s", config.HZN_FSS_CERT_PATH, agId) + err := c.Client.CoreV1().Secrets(namespace).Delete(context.Background(), essCertSecretName, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("Error: failed to delete ess cert secret for %s: %v", agId, err) + } + return nil +} + // CreateK8SSecrets will create a k8s secrets object which contains the service secret name and value func (c KubeClient) CreateK8SSecrets(serviceSecretsMap map[string]string, agId string, namespace string) (string, error) { secretsLabel := map[string]string{"name": HZN_SERVICE_SECRETS} @@ -369,6 +468,71 @@ func (c KubeClient) CreateK8SSecrets(serviceSecretsMap map[string]string, agId s return res.ObjectMeta.Name, nil } +// DeleteK8SSecrets will delete k8s secrets object which contains the service secret name and value +func (c KubeClient) DeleteK8SSecrets(agId string, namespace string) error { + // delete the secrets contains agreement service vault secrets + secretsName := fmt.Sprintf("%s-%s", HZN_SERVICE_SECRETS, agId) + err := c.Client.CoreV1().Secrets(namespace).Delete(context.Background(), secretsName, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("Error: failed to delete k8s secrets that contains service secrets for %s: %v", agId, err) + } + return nil +} + +func (c KubeClient) CreateMMSPVC(envVars map[string]string, agId string, namespace string) (string, error) { + storageClass, accessModes, _ := cutil.GetAgentPVCInfo() + if scInUserinput, ok := envVars[STORAGE_CLASS_USERINPUT_NAME]; ok { + storageClass = scInUserinput + } + + if accessModeInUserinput, ok := envVars[PVC_ACCESS_MODE_USERINPUT_NAME]; ok { + if m, ok := accessModeMap[accessModeInUserinput]; ok { + accessModes = []corev1.PersistentVolumeAccessMode{m} + } + } + + pvcSizeInString := DEFAULT_PVC_SIZE_IN_STRING + if pvcSizeInUserInput, ok := envVars[PVC_SIZE_USERINPUT_NAME]; ok { + pvcSizeInString = pvcSizeInUserInput + } + + mmsPvcName := fmt.Sprintf("%s-%s", K8S_MMS_SHARED_PVC_NAME, agId) + mmsPVC := corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: mmsPvcName, + Namespace: namespace, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: &storageClass, + AccessModes: accessModes, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse(fmt.Sprintf("%vGi", pvcSizeInString)), + }, + }, + }, + } + + res, err := c.Client.CoreV1().PersistentVolumeClaims(namespace).Create(context.Background(), &mmsPVC, metav1.CreateOptions{}) + if err != nil && errors.IsAlreadyExists(err) { + _, err = c.Client.CoreV1().PersistentVolumeClaims(namespace).Update(context.Background(), &mmsPVC, metav1.UpdateOptions{}) + } + if err != nil { + return "", fmt.Errorf("Error: failed to create mms pvc for %s: %v", agId, err) + } + return res.ObjectMeta.Name, nil +} + +func (c KubeClient) DeleteMMSPVC(agId string, namespace string) error { + mmsPvcName := fmt.Sprintf("%s-%s", K8S_MMS_SHARED_PVC_NAME, agId) + + err := c.Client.CoreV1().PersistentVolumeClaims(namespace).Delete(context.Background(), mmsPvcName, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("Error: failed to delete mms pvc for %s: %v", agId, err) + } + return nil +} + func unstructuredObjectFromYaml(crStr YamlFile) (*unstructured.Unstructured, error) { cr := make(map[string]interface{}) err := yaml.UnmarshalStrict([]byte(crStr.Body), &cr) diff --git a/kube_operator/kubeworker.go b/kube_operator/kubeworker.go index cfe96f3ff..713ec04bb 100644 --- a/kube_operator/kubeworker.go +++ b/kube_operator/kubeworker.go @@ -5,22 +5,28 @@ import ( "github.com/boltdb/bolt" "github.com/golang/glog" "github.com/open-horizon/anax/config" + "github.com/open-horizon/anax/cutil" "github.com/open-horizon/anax/events" "github.com/open-horizon/anax/persistence" "github.com/open-horizon/anax/resource" "github.com/open-horizon/anax/worker" + "path" ) type KubeWorker struct { worker.BaseWorker + config *config.HorizonConfig db *bolt.DB + authMgr *resource.AuthenticationManager secretMgr *resource.SecretsManager } -func NewKubeWorker(name string, config *config.HorizonConfig, db *bolt.DB, sm *resource.SecretsManager) *KubeWorker { +func NewKubeWorker(name string, config *config.HorizonConfig, db *bolt.DB, am *resource.AuthenticationManager, sm *resource.SecretsManager) *KubeWorker { worker := &KubeWorker{ BaseWorker: worker.NewBaseWorker(name, config, nil), + config: config, db: db, + authMgr: am, secretMgr: sm, } glog.Info(kwlog(fmt.Sprintf("Starting Kubernetes Worker"))) @@ -32,6 +38,10 @@ func (w *KubeWorker) Messages() chan events.Message { return w.BaseWorker.Manager.Messages } +func (kw *KubeWorker) GetAuthenticationManager() *resource.AuthenticationManager { + return kw.authMgr +} + func (w *KubeWorker) GetSecretManager() *resource.SecretsManager { return w.secretMgr } @@ -181,13 +191,39 @@ func (w *KubeWorker) processKubeOperator(lc *events.AgreementLaunchContext, kd * } // eg: secretsMap is map[secret1:eyJrZXki...] - client, err := NewKubeClient() - if err != nil { - return err - } - err = client.Install(kd.OperatorYamlArchive, kd.Metadata, *(lc.EnvironmentAdditions), secretsMap, lc.AgreementId, lc.Configure.ClusterNamespace, crInstallTimeout) - if err != nil { - return err + // create auth in agent pod and mount it to service pod + if ags, err := persistence.FindEstablishedAgreements(w.db, lc.AgreementProtocol, []persistence.EAFilter{persistence.UnarchivedEAFilter(), persistence.IdEAFilter(lc.AgreementId)}); err != nil { + glog.Errorf("Unable to retrieve agreement %v from database, error %v", lc.AgreementId, err) + } else if len(ags) != 1 { + glog.V(3).Infof(kwlog(fmt.Sprintf("Ignoring the configure event for agreement %v, the agreement is no longer active.", lc.AgreementId))) + return nil + } else if ags[0].AgreementTerminatedTime != 0 { + glog.V(3).Infof(kwlog(fmt.Sprintf("Received configure command for agreement %v. Ignoring it because this agreement has been terminated.", lc.AgreementId))) + return nil + } else if ags[0].AgreementExecutionStartTime != 0 { + glog.V(3).Infof(kwlog(fmt.Sprintf("Received configure command for agreement %v. Ignoring it because the containers for this agreement has been configured.", lc.AgreementId))) + return nil + } else { + serviceIdentity := cutil.FormOrgSpecUrl(cutil.NormalizeURL(ags[0].RunningWorkload.URL), ags[0].RunningWorkload.Org) + sVer := ags[0].RunningWorkload.Version + glog.V(3).Infof(kwlog(fmt.Sprintf("Creating ESS creds for svc: %v svcVer: %v", serviceIdentity, sVer))) + + _, err := w.GetAuthenticationManager().CreateCredential(lc.AgreementId, serviceIdentity, sVer, false) + if err != nil { + return err + } + + client, err := NewKubeClient() + if err != nil { + return err + } + + fssAuthFilePath := path.Join(w.GetAuthenticationManager().GetCredentialPath(lc.AgreementId), config.HZN_FSS_AUTH_FILE) // /var/horizon/ess-auth//auth.json + fssCertFilePath := path.Join(w.config.GetESSSSLClientCertPath(), config.HZN_FSS_CERT_FILE) // /var/horizon/ess-auth/SSL/cert/cert.pem + err = client.Install(kd.OperatorYamlArchive, kd.Metadata, *(lc.EnvironmentAdditions), fssAuthFilePath, fssCertFilePath, secretsMap, lc.AgreementId, lc.Configure.ClusterNamespace, crInstallTimeout) + if err != nil { + return err + } } return nil } diff --git a/locales/fr/messages.gotext.json b/locales/fr/messages.gotext.json index 25b0069d1..493953437 100644 --- a/locales/fr/messages.gotext.json +++ b/locales/fr/messages.gotext.json @@ -2169,6 +2169,30 @@ ], "fuzzy": true }, + { + "id": "Name, or Org is empty string.", + "message": "Name, or Org is empty string.", + "translation": "Le nom ou l'organisation est une chaîne vide.", + "fuzzy": true + }, + { + "id": "The serviceVersions array is empty.", + "message": "The serviceVersions array is empty.", + "translation": "Le tableau serviceVersions est vide.", + "fuzzy": true + }, + { + "id": "retry_durations and retries cannot be zero if priority_value is set to non-zero value", + "message": "retry_durations and retries cannot be zero if priority_value is set to non-zero value", + "translation": "retry_durations et retries ne peuvent pas être égales à zéro si priority_value est défini sur une valeur différente de zéro", + "fuzzy": true + }, + { + "id": "retry_durations, retries and verified_durations cannot be non-zero value if priority_value is zero or not set", + "message": "retry_durations, retries and verified_durations cannot be non-zero value if priority_value is zero or not set", + "translation": "retry_durations, retries et verified_durations ne peuvent pas avoir une valeur différente de zéro si priority_value est égale à zéro ou n'est pas définie", + "fuzzy": true + }, { "id": "properties contains an invalid property: {Err}", "message": "properties contains an invalid property: {Err}", @@ -6175,22 +6199,6 @@ "translation": "L'entrée SecretBindingCheck ne peut pas avoir pour valeur NULL", "fuzzy": true }, - { - "id": "Node type '{NodeType}' does not support secret binding check.", - "message": "Node type '{NodeType}' does not support secret binding check.", - "translation": "Le type de noeud '{NodeType}' ne prend pas en charge la vérification de liaison de secret.", - "placeholders": [ - { - "id": "NodeType", - "string": "%[1]v", - "type": "string", - "underlyingType": "string", - "argNum": 1, - "expr": "nodeType" - } - ], - "fuzzy": true - }, { "id": "No service versions with architecture {NodeArch} specified in the deployment policy or pattern.", "message": "No service versions with architecture {NodeArch} specified in the deployment policy or pattern.", @@ -6225,6 +6233,30 @@ "translation": "Type incompatible", "fuzzy": true }, + { + "id": "{Msgcompatible}, Warning: {Reason}", + "message": "{Msgcompatible}, Warning: {Reason}", + "translation": "{Msgcompatible}, Avertissement : {Reason}", + "placeholders": [ + { + "id": "Msgcompatible", + "string": "%[1]v", + "type": "string", + "underlyingType": "string", + "argNum": 1, + "expr": "msg_compatible" + }, + { + "id": "Reason", + "string": "%[2]v", + "type": "string", + "underlyingType": "string", + "argNum": 2, + "expr": "reason" + } + ], + "fuzzy": true + }, { "id": "Error getting services for all archetctures for {ServiceOrg}/{ServiceURL} version {Version}. {Err}", "message": "Error getting services for all archetctures for {ServiceOrg}/{ServiceURL} version {Version}. {Err}", @@ -6385,12 +6417,6 @@ ], "fuzzy": true }, - { - "id": "Secret binding for a cluster service is not supported.", - "message": "Secret binding for a cluster service is not supported.", - "translation": "La liaison de secret pour un service de cluster n'est pas prise en charge.", - "fuzzy": true - }, { "id": "No secret binding found for the following service secrets: {NbArray}.", "message": "No secret binding found for the following service secrets: {NbArray}.", @@ -6523,6 +6549,22 @@ ], "fuzzy": true }, + { + "id": "{Err}, use non node-level secret", + "message": "{Err}, use non node-level secret", + "translation": "{Err}, utilisent un secret qui n'est pas au niveau du noeud", + "placeholders": [ + { + "id": "Err", + "string": "%[1]v", + "type": "error", + "underlyingType": "interface{Error() string}", + "argNum": 1, + "expr": "err" + } + ], + "fuzzy": true + }, { "id": "Error parsing secret name in the secret binding. {Errparse}", "message": "Error parsing secret name in the secret binding. {Errparse}", @@ -6539,6 +6581,38 @@ ], "fuzzy": true }, + { + "id": "Error checking secret {VaultSecretName} for node {NName} in the secret manager. {Err}", + "message": "Error checking secret {VaultSecretName} for node {NName} in the secret manager. {Err}", + "translation": "Erreur lors de la vérification du secret {VaultSecretName} pour le noeud {NName} dans le gestionnaire de secrets. {Err}", + "placeholders": [ + { + "id": "VaultSecretName", + "string": "%[1]v", + "type": "string", + "underlyingType": "string", + "argNum": 1, + "expr": "vaultSecretName" + }, + { + "id": "NName", + "string": "%[2]v", + "type": "string", + "underlyingType": "string", + "argNum": 2, + "expr": "nName" + }, + { + "id": "Err", + "string": "%[3]v", + "type": "error", + "underlyingType": "interface{Error() string}", + "argNum": 3, + "expr": "err" + } + ], + "fuzzy": true + }, { "id": "Error checking secret {VaultSecretName} in the secret manager. {Err}", "message": "Error checking secret {VaultSecretName} in the secret manager. {Err}", @@ -6563,6 +6637,30 @@ ], "fuzzy": true }, + { + "id": "Node level secret {VaultSecretName} doesn't exist for node {NName}", + "message": "Node level secret {VaultSecretName} doesn't exist for node {NName}", + "translation": "Le secret au niveau du noeud {VaultSecretName} n'existe pas pour le noeud {NName}", + "placeholders": [ + { + "id": "VaultSecretName", + "string": "%[1]v", + "type": "string", + "underlyingType": "string", + "argNum": 1, + "expr": "vaultSecretName" + }, + { + "id": "NName", + "string": "%[2]v", + "type": "string", + "underlyingType": "string", + "argNum": 2, + "expr": "nName" + } + ], + "fuzzy": true + }, { "id": "The binding secret name cannot be an empty string. The valid formats are: '' for the organization level secret and 'user//' for the user level secret.", "message": "The binding secret name cannot be an empty string. The valid formats are: '' for the organization level secret and 'user//' for the user level secret.", diff --git a/main.go b/main.go index f07e42a35..47d91e1f5 100644 --- a/main.go +++ b/main.go @@ -188,7 +188,7 @@ func main() { if imageWorker := imagefetch.NewImageFetchWorker("ImageFetch", cfg, db); imageWorker != nil { workers.Add(imageWorker) } - workers.Add(kube_operator.NewKubeWorker("Kube", cfg, db, secretm)) + workers.Add(kube_operator.NewKubeWorker("Kube", cfg, db, authm, secretm)) workers.Add(resource.NewResourceWorker("Resource", cfg, db, authm)) workers.Add(changes.NewChangesWorker("ExchangeChanges", cfg, db)) workers.Add(nodemanagement.NewNodeManagementWorker("NodeManagement", cfg, db)) diff --git a/nodemanagement/agent_upgrade.go b/nodemanagement/agent_upgrade.go new file mode 100644 index 000000000..b5b33ccfa --- /dev/null +++ b/nodemanagement/agent_upgrade.go @@ -0,0 +1,402 @@ +package nodemanagement + +import ( + "encoding/json" + "fmt" + "github.com/boltdb/bolt" + "github.com/golang/glog" + "github.com/open-horizon/anax/common" + "github.com/open-horizon/anax/eventlog" + "github.com/open-horizon/anax/events" + "github.com/open-horizon/anax/exchange" + "github.com/open-horizon/anax/exchangecommon" + "github.com/open-horizon/anax/persistence" + "github.com/open-horizon/anax/semanticversion" + "github.com/open-horizon/anax/version" + "os" + "path" + "sort" +) + +const STATUS_FILE_NAME = "status.json" +const NMP_MONITOR = "NMPMonitor" + +// this function will set the status of any nmp in "download started" to "waiting" +// run this when the node starts or is registered so a partial download that ended unexpectedly will be restarted +func (w *NodeManagementWorker) ResetDownloadStartedStatuses() error { + downloadStartedStatuses, err := persistence.FindDownloadStartedNMPStatuses(w.db) + if err != nil { + return err + } + for statusName, status := range downloadStartedStatuses { + status.SetStatus(exchangecommon.STATUS_NEW) + if err := w.UpdateStatus(statusName, status, exchange.GetPutNodeManagementPolicyStatusHandler(w), persistence.NewMessageMeta(EL_NMP_STATUS_CHANGED, statusName, exchangecommon.STATUS_NEW), persistence.EC_NMP_STATUS_UPDATE_NEW); err != nil { + return err + } + } + + return nil +} + +// this returns the name and status struct of the status with the eariest scheduled start time and deletes that earliest status from the map passed in +func getLatest(statusMap *map[string]*exchangecommon.NodeManagementPolicyStatus) (string, *exchangecommon.NodeManagementPolicyStatus) { + latestNmpName := "" + latestNmpStatus := &exchangecommon.NodeManagementPolicyStatus{} + if statusMap == nil { + return "", nil + } + + for nmpName, nmpStatus := range *statusMap { + if nmpStatus != nil && nmpStatus.TimeToStart() { + if latestNmpName == "" || !nmpStatus.AgentUpgradeInternal.ScheduledUnixTime.Before(latestNmpStatus.AgentUpgradeInternal.ScheduledUnixTime) { + latestNmpStatus = nmpStatus + latestNmpName = nmpName + } + } + } + delete(*statusMap, latestNmpName) + return latestNmpName, latestNmpStatus +} + +// After a successful download, update the node status in the db and the exchange and create an eventlog event for the change +func (n *NodeManagementWorker) DownloadComplete(cmd *NMPDownloadCompleteCommand) { + status, err := persistence.FindNMPStatus(n.db, cmd.Msg.NMPName) + if err != nil { + glog.Errorf(nmwlog(fmt.Sprintf("Failed to get nmp status %v from the database: %v", cmd.Msg.NMPName, err))) + return + } else if status == nil { + glog.Errorf(nmwlog(fmt.Sprintf("Failed to find status for nmp %v in the database.", cmd.Msg.NMPName))) + return + } + var msgMeta *persistence.MessageMeta + eventCode := "" + + if cmd.Msg.Status == exchangecommon.STATUS_NO_ACTION { + glog.Infof(nmwlog(fmt.Sprintf("Already in compliance with nmp %v. Download skipped.", cmd.Msg.NMPName))) + status.SetStatus(exchangecommon.STATUS_NO_ACTION) + msgMeta = persistence.NewMessageMeta(EL_NMP_STATUS_CHANGED, cmd.Msg.NMPName, exchangecommon.STATUS_NO_ACTION) + eventCode = persistence.EC_NMP_STATUS_DOWNLOAD_SUCCESSFUL + } else if cmd.Msg.Status == exchangecommon.STATUS_DOWNLOADED { + glog.Infof(nmwlog(fmt.Sprintf("Sucessfully downloaded packages for nmp %v.", cmd.Msg.NMPName))) + if dev, err := exchange.GetExchangeDevice(n.GetHTTPFactory(), n.GetExchangeId(), n.GetExchangeId(), n.GetExchangeToken(), n.GetExchangeURL()); err != nil { + glog.Errorf(nmwlog(fmt.Sprintf("Failed to get device from the db: %v", err))) + return + } else if dev.HAGroup != "" { + status.SetStatus(exchangecommon.STATUS_HA_WAITING) + msgMeta = persistence.NewMessageMeta(EL_NMP_STATUS_CHANGED, cmd.Msg.NMPName, exchangecommon.STATUS_HA_WAITING) + eventCode = persistence.EC_NMP_STATUS_CHANGED + } else { + status.SetStatus(exchangecommon.STATUS_DOWNLOADED) + msgMeta = persistence.NewMessageMeta(EL_NMP_STATUS_CHANGED, cmd.Msg.NMPName, exchangecommon.STATUS_DOWNLOADED) + eventCode = persistence.EC_NMP_STATUS_DOWNLOAD_SUCCESSFUL + } + } else if cmd.Msg.Status == exchangecommon.STATUS_PRECHECK_FAILED { + glog.Infof(nmwlog(fmt.Sprintf("Node management policy %v failed precheck conditions. %v", cmd.Msg.NMPName, cmd.Msg.ErrorMessage))) + status.SetStatus(exchangecommon.STATUS_PRECHECK_FAILED) + status.SetErrorMessage(cmd.Msg.ErrorMessage) + msgMeta = persistence.NewMessageMeta(EL_NMP_STATUS_CHANGED_WITH_ERROR, cmd.Msg.NMPName, exchangecommon.STATUS_PRECHECK_FAILED, cmd.Msg.ErrorMessage) + eventCode = persistence.EC_NMP_STATUS_CHANGED + } else { + if status.AgentUpgradeInternal.DownloadAttempts < 4 { + glog.Infof(nmwlog(fmt.Sprintf("Resetting status for %v to waiting to retry failed download.", cmd.Msg.NMPName))) + status.AgentUpgradeInternal.DownloadAttempts = status.AgentUpgradeInternal.DownloadAttempts + 1 + status.SetStatus(exchangecommon.STATUS_NEW) + msgMeta = persistence.NewMessageMeta(EL_NMP_STATUS_CHANGED, cmd.Msg.NMPName, exchangecommon.STATUS_NEW) + eventCode = persistence.EC_NMP_STATUS_CHANGED + } else { + glog.Infof(nmwlog(fmt.Sprintf("Download attempted 3 times already for %v. Download will not be tried again.", cmd.Msg.NMPName))) + glog.Errorf(nmwlog(fmt.Sprintf("Failed to download packages for nmp %v. %v", cmd.Msg.NMPName, cmd.Msg.ErrorMessage))) + status.SetStatus(cmd.Msg.Status) + status.SetErrorMessage(cmd.Msg.ErrorMessage) + msgMeta = persistence.NewMessageMeta(EL_NMP_STATUS_CHANGED_WITH_ERROR, cmd.Msg.NMPName, cmd.Msg.Status, cmd.Msg.ErrorMessage) + eventCode = persistence.EC_NMP_STATUS_CHANGED + } + } + if cmd.Msg.Versions != nil { + status.AgentUpgrade.UpgradedVersions = *cmd.Msg.Versions + } + if cmd.Msg.Latests != nil { + status.AgentUpgradeInternal.LatestMap = *cmd.Msg.Latests + } + err = n.UpdateStatus(cmd.Msg.NMPName, status, exchange.GetPutNodeManagementPolicyStatusHandler(n), msgMeta, eventCode) + if err != nil { + glog.Errorf(nmwlog(fmt.Sprintf("Failed to update nmp status %v: %v", cmd.Msg.NMPName, err))) + } + + if cmd.Msg.Status == exchangecommon.STATUS_DOWNLOADED { + n.Messages() <- events.NewAgentPackageDownloadedMessage(events.AGENT_PACKAGE_DOWNLOADED, events.StartDownloadMessage{NMPStatus: status, NMPName: cmd.Msg.NMPName}) + } +} + +// Read and persist the status out of the file +// Update status in the exchange +// If everything is successful, delete the job working dir +func (n *NodeManagementWorker) CollectStatus(workingFolderPath string, policyName string, dbStatus *exchangecommon.NodeManagementPolicyStatus) error { + filePath := path.Join(workingFolderPath, policyName, STATUS_FILE_NAME) + // Read in the status file + if _, err := os.Stat(filePath); err != nil { + return fmt.Errorf("Failed to open status file %v for management job %v. Error was: %v", filePath, policyName, err) + } + if openPath, err := os.Open(filePath); err != nil { + return fmt.Errorf("Failed to open status file %v for management job %v. Errorf was: %v", filePath, policyName, err) + } else { + contents := exchangecommon.NodeManagementPolicyStatus{} + err = json.NewDecoder(openPath).Decode(&contents) + if err != nil { + return fmt.Errorf("Failed to decode status file %v for management job %v. Error was %v.", filePath, policyName, err) + } + + exchDev, err := persistence.FindExchangeDevice(n.db) + if err != nil { + glog.Errorf(nmwlog(fmt.Sprintf("Error getting device from database: %v", err))) + exchDev = nil + } + + status_changed, err := common.SetNodeManagementPolicyStatus(n.db, exchDev, policyName, &contents, dbStatus, + exchange.GetPutNodeManagementPolicyStatusHandler(n), + exchange.GetHTTPDeviceHandler(n), + exchange.GetHTTPPatchDeviceHandler(n)) + if err != nil { + glog.Errorf(nmwlog(fmt.Sprintf("Error saving nmp status for %v: %v", policyName, err))) + return err + } else { + // log the event + if status_changed { + pattern := "" + configState := "" + if exchDev != nil { + pattern = exchDev.Pattern + configState = exchDev.Config.State + } + status_string := contents.AgentUpgrade.Status + if status_string == "" { + status_string = exchangecommon.STATUS_UNKNOWN + } + if contents.AgentUpgrade.ErrorMessage != "" { + status_string += fmt.Sprintf(", ErrorMessage: %v", contents.AgentUpgrade.ErrorMessage) + } + eventlog.LogNodeEvent(n.db, persistence.SEVERITY_INFO, persistence.NewMessageMeta(EL_NMP_STATUS_CHANGED, policyName, status_string), persistence.EC_NMP_STATUS_CHANGED, exchange.GetId(n.GetExchangeId()), exchange.GetOrg(n.GetExchangeId()), pattern, configState) + } + } + } + return nil +} + +// Check if the current agent versions are up to date for software, cert and config according to +// the specification of the nmp. The NMP must have at least one 'latest' as the version string. +func IsAgentUpToDate(status *exchangecommon.NodeManagementPolicyStatus, exchAFVs *exchangecommon.AgentFileVersions, db *bolt.DB) (bool, error) { + // get local device info + dev, err := persistence.FindExchangeDevice(db) + if err != nil || dev == nil { + return false, fmt.Errorf("Failed to get device from the local db: %v", err) + } + + if exchAFVs != nil { + // check software version + if status.AgentUpgradeInternal.LatestMap.SoftwareLatest { + versions := exchAFVs.SoftwareVersions + if !IsVersionLatest(versions, version.HORIZON_VERSION) { + return false, nil + } + } + // check config version + if status.AgentUpgradeInternal.LatestMap.ConfigLatest { + versions := exchAFVs.ConfigVersions + + devConfigVer := "" + if dev.SoftwareVersions != nil { + if ver, ok := dev.SoftwareVersions[persistence.CONFIG_VERSION]; ok { + devConfigVer = ver + } + } + + if !IsVersionLatest(versions, devConfigVer) { + return false, nil + } + } + // check certificate version + if status.AgentUpgradeInternal.LatestMap.CertLatest { + versions := exchAFVs.CertVersions + + devCertVer := "" + if dev.SoftwareVersions != nil { + if ver, ok := dev.SoftwareVersions[persistence.CERT_VERSION]; ok { + devCertVer = ver + } + } + + if !IsVersionLatest(versions, devCertVer) { + return false, nil + } + } + } + return true, nil +} + +// Compare status.UpgradedVersions with the AgentFileVersions. +// It returns true if all the versions are up to date. This means +// that the nmp has been processed before with the latest versions. +func IsLatestVersionHandled(status *exchangecommon.NodeManagementPolicyStatus, exchAFVs *exchangecommon.AgentFileVersions) (bool, error) { + + // not handled + if status.AgentUpgrade == nil { + return false, nil + } + + upgradedVersions := status.AgentUpgrade.UpgradedVersions + + if exchAFVs != nil { + // check software version + if status.AgentUpgradeInternal.LatestMap.SoftwareLatest { + versions := exchAFVs.SoftwareVersions + if !IsVersionLatest(versions, upgradedVersions.SoftwareVersion) { + return false, nil + } + } + // check config version + if status.AgentUpgradeInternal.LatestMap.ConfigLatest { + versions := exchAFVs.ConfigVersions + if !IsVersionLatest(versions, upgradedVersions.ConfigVersion) { + return false, nil + } + } + // check certificate version + if status.AgentUpgradeInternal.LatestMap.CertLatest { + versions := exchAFVs.CertVersions + if !IsVersionLatest(versions, upgradedVersions.CertVersion) { + return false, nil + } + } + } + return true, nil +} + +// check if current version is the latest available version. If the number of +// available versions is zero, the current version is considered the latest. +func IsVersionLatest(availibleVers []string, currentVersion string) bool { + if availibleVers != nil && len(availibleVers) != 0 { + sort.Slice(availibleVers, func(i, j int) bool { + comp, _ := semanticversion.CompareVersions(availibleVers[i], availibleVers[j]) + return comp > 0 + }) + + return currentVersion == availibleVers[0] + } + return true +} + +// Check all nmp statuses that specify "latest" for a version, if status is not "downloaded", "download started" or "initiated", then change to "waiting" as there is a new version availible +// If there is no new version for whatever the status has "latest" for, it will be marked successful without executing +func (n *NodeManagementWorker) HandleAgentFilesVersionChange(cmd *AgentFileVersionChangeCommand) { + glog.V(3).Infof(nmwlog(fmt.Sprintf("HandleAgentFilesVersionChange re-evaluating NMPs that request the 'latest' versions."))) + if latestStatuses, err := persistence.FindNMPWithLatestKeywordVersion(n.db); err != nil { + glog.Errorf(nmwlog(fmt.Sprintf("Error getting nmp statuses from db to change to \"waiting\". Error was: %v", err))) + return + } else { + // get agent file versions + exchAFVs, err := exchange.GetNodeUpgradeVersionsHandler(n)() + if err != nil { + glog.Errorf("Failed to get the AgentFileVersion from the exchange. %v", err) + return + } + + needDeferCommand := false + for statusName, status := range latestStatuses { + setStatusToWaiting := false + nmpStatus := status.AgentUpgrade.Status + if nmpStatus == exchangecommon.STATUS_NEW { + glog.V(3).Infof(nmwlog(fmt.Sprintf("The nmp %v is already in 'waiting' status. do nothing.", statusName))) + continue + } else if nmpStatus == exchangecommon.STATUS_DOWNLOADED || nmpStatus == exchangecommon.STATUS_DOWNLOAD_STARTED || nmpStatus == exchangecommon.STATUS_INITIATED || nmpStatus == exchangecommon.STATUS_ROLLBACK_STARTED { + glog.V(3).Infof(nmwlog(fmt.Sprintf("The nmp %v with latest keyword is currently being executed or downloaded (status is %v). Exiting without changing status to \"waiting\", checking this nmp later", statusName, nmpStatus))) + needDeferCommand = true + } else if nmpStatus == exchangecommon.STATUS_DOWNLOAD_FAILED || nmpStatus == exchangecommon.STATUS_FAILED_JOB || nmpStatus == exchangecommon.STATUS_PRECHECK_FAILED || nmpStatus == exchangecommon.STATUS_ROLLBACK_FAILED || nmpStatus == exchangecommon.STATUS_ROLLBACK_SUCCESSFUL { + if isHandled, err := IsLatestVersionHandled(status, exchAFVs); err != nil { + glog.Errorf(nmwlog(fmt.Sprintf("Error checking if the latest versions are previously handled for nmp %v. %v", statusName, err))) + } else if isHandled { + glog.V(3).Infof(nmwlog(fmt.Sprintf("The latest agent versions are previously handled for nmp %v. The status was %v. Exiting without changing status to \"waiting\".", statusName, nmpStatus))) + } else { + setStatusToWaiting = true + } + } else { + if isUpToDate, err := IsAgentUpToDate(status, exchAFVs, n.db); err != nil { + glog.Errorf(nmwlog(fmt.Sprintf("Error checking if the agent versions are up to date for nmp %v. %v", statusName, err))) + } else if isUpToDate { + glog.V(3).Infof(nmwlog(fmt.Sprintf("The agent versions are up to date for nmp %v. Exiting without changing status to \"waiting\".", statusName))) + } else { + setStatusToWaiting = true + } + } + + // set the status to waiting for this nmp + if setStatusToWaiting { + glog.V(3).Infof(nmwlog(fmt.Sprintf("Change status to \"waiting\" for the nmp %v", statusName))) + + // Add startWindow to current time to randomize upgrade start times just like what occurs when an NMP first executes + if status.TimeToStart() { + nmp, err := persistence.FindNodeManagementPolicy(n.db, statusName) + if err != nil { + glog.Errorf(nmwlog(fmt.Sprintf("Error getting nmp from db to check the startWindow value. Error was: %v", err))) + } + if nmp != nil { + status.SetScheduledStartTime(exchangecommon.TIME_NOW_KEYWORD, nmp.LastUpdated, nmp.UpgradeWindowDuration) + } + } + + status.AgentUpgrade.Status = exchangecommon.STATUS_NEW + err = n.UpdateStatus(statusName, status, exchange.GetPutNodeManagementPolicyStatusHandler(n), persistence.NewMessageMeta(EL_NMP_STATUS_CHANGED, statusName, exchangecommon.STATUS_NEW), persistence.EC_NMP_STATUS_UPDATE_NEW) + if err != nil { + glog.Errorf(nmwlog(fmt.Sprintf("Error changing nmp status for %v to \"waiting\". Error was %v.", statusName, err))) + } + } + + } // end for + + if needDeferCommand && cmd != nil { + n.AddDeferredCommand(cmd) + } + } +} + +// This function gets all the 'reset' nmp status from the exchange and set them to +// 'waiting' so that the agent can start re-evaluating them. +func (w *NodeManagementWorker) HandleNmpStatusReset() { + glog.V(3).Infof(nmwlog(fmt.Sprintf("HandleNmpStatusReset re-evaluating NMPs that has the status 'reset'."))) + + // get all the nmps that applies to this node from the exchange + allNmpStatus, err := exchange.GetNodeManagementAllStatuses(w, exchange.GetOrg(w.GetExchangeId()), exchange.GetId(w.GetExchangeId())) + if err != nil { + glog.Errorf(nmwlog(fmt.Sprintf("Error getting all nmp statuses for node %v from the exchange. %v", w.GetExchangeId(), err))) + } else { + glog.V(5).Infof(nmwlog(fmt.Sprintf("GetNodeManagementAllStatuses returns: %v", allNmpStatus))) + } + + // find all nmp status from local db + allLocalStatuses, err := persistence.FindAllNMPStatus(w.db) + if err != nil { + glog.Errorf(nmwlog(fmt.Sprintf("Error getting all nmp statuses from the local database. %v", err))) + } + + // change the status to 'waiting' + if allNmpStatus != nil { + for nmp_name, nmp_status := range allNmpStatus.PolicyStatuses { + if nmp_status.Status() == exchangecommon.STATUS_RESET { + if local_status, ok := allLocalStatuses[nmp_name]; ok { + glog.V(3).Infof(nmwlog(fmt.Sprintf("Change status from \"reset\" to \"waiting\" for the nmp %v", nmp_name))) + + local_status.AgentUpgrade.Status = exchangecommon.STATUS_NEW + if local_status.AgentUpgradeInternal != nil { + local_status.AgentUpgradeInternal.DownloadAttempts = 0 + } + + err = w.UpdateStatus(nmp_name, local_status, exchange.GetPutNodeManagementPolicyStatusHandler(w), persistence.NewMessageMeta(EL_NMP_STATUS_CHANGED, nmp_name, exchangecommon.STATUS_NEW), persistence.EC_NMP_STATUS_UPDATE_NEW) + if err != nil { + glog.Errorf(nmwlog(fmt.Sprintf("Error changing nmp status for %v from \"reset\" to \"waiting\". Error was %v.", nmp_name, err))) + } + } else { + glog.V(3).Infof(nmwlog(fmt.Sprintf("node management status for nmp %v for node %v is set to \"reset\" but the status cannot be found from the local db. Skiping it.", nmp_name, w.GetExchangeId()))) + } + } + } + } +} diff --git a/nodemanagement/commands.go b/nodemanagement/commands.go index 38a744406..8580c51d2 100644 --- a/nodemanagement/commands.go +++ b/nodemanagement/commands.go @@ -2,6 +2,7 @@ package nodemanagement import ( "fmt" + "github.com/open-horizon/anax/containermessage" "github.com/open-horizon/anax/events" ) @@ -142,3 +143,21 @@ func NewNmpStatusChangeCommand(msg *events.ExchangeChangeMessage) *NmpStatusChan Msg: msg, } } + +type ImageFetchedCommand struct { + DeploymentDescription *containermessage.DeploymentDescription +} + +func NewImageFetchedCommand(dd *containermessage.DeploymentDescription) *ImageFetchedCommand { + return &ImageFetchedCommand{ + DeploymentDescription: dd, + } +} + +func (i ImageFetchedCommand) String() string { + return fmt.Sprintf("DeploymentDescription: %v", i.DeploymentDescription) +} + +func (i ImageFetchedCommand) ShortString() string { + return i.String() +} diff --git a/nodemanagement/node_management_worker.go b/nodemanagement/node_management_worker.go index f89e4356d..e743ab4de 100644 --- a/nodemanagement/node_management_worker.go +++ b/nodemanagement/node_management_worker.go @@ -1,11 +1,9 @@ package nodemanagement import ( - "encoding/json" "fmt" "github.com/boltdb/bolt" "github.com/golang/glog" - "github.com/open-horizon/anax/common" "github.com/open-horizon/anax/config" "github.com/open-horizon/anax/cutil" "github.com/open-horizon/anax/eventlog" @@ -14,19 +12,11 @@ import ( "github.com/open-horizon/anax/exchangecommon" "github.com/open-horizon/anax/externalpolicy" "github.com/open-horizon/anax/persistence" - "github.com/open-horizon/anax/semanticversion" - "github.com/open-horizon/anax/version" "github.com/open-horizon/anax/worker" - "os" - "path" - "sort" "strings" "sync" ) -const STATUS_FILE_NAME = "status.json" -const NMP_MONITOR = "NMPMonitor" - var statusUpdateLock sync.Mutex type NodeManagementWorker struct { @@ -165,43 +155,6 @@ func (w *NodeManagementWorker) checkNMPTimeToRun() int { return 60 } -// this function will set the status of any nmp in "download started" to "waiting" -// run this when the node starts or is registered so a partial download that ended unexpectedly will be restarted -func (w *NodeManagementWorker) ResetDownloadStartedStatuses() error { - downloadStartedStatuses, err := persistence.FindDownloadStartedNMPStatuses(w.db) - if err != nil { - return err - } - for statusName, status := range downloadStartedStatuses { - status.SetStatus(exchangecommon.STATUS_NEW) - if err := w.UpdateStatus(statusName, status, exchange.GetPutNodeManagementPolicyStatusHandler(w), persistence.NewMessageMeta(EL_NMP_STATUS_CHANGED, statusName, exchangecommon.STATUS_NEW), persistence.EC_NMP_STATUS_UPDATE_NEW); err != nil { - return err - } - } - - return nil -} - -// this returns the name and status struct of the status with the eariest scheduled start time and deletes that earliest status from the map passed in -func getLatest(statusMap *map[string]*exchangecommon.NodeManagementPolicyStatus) (string, *exchangecommon.NodeManagementPolicyStatus) { - latestNmpName := "" - latestNmpStatus := &exchangecommon.NodeManagementPolicyStatus{} - if statusMap == nil { - return "", nil - } - - for nmpName, nmpStatus := range *statusMap { - if nmpStatus != nil && nmpStatus.TimeToStart() { - if latestNmpName == "" || !nmpStatus.AgentUpgradeInternal.ScheduledUnixTime.Before(latestNmpStatus.AgentUpgradeInternal.ScheduledUnixTime) { - latestNmpStatus = nmpStatus - latestNmpName = nmpName - } - } - } - delete(*statusMap, latestNmpName) - return latestNmpName, latestNmpStatus -} - func getEC(config *config.HorizonConfig, db *bolt.DB) *worker.BaseExchangeContext { var ec *worker.BaseExchangeContext if dev, _ := persistence.FindExchangeDevice(db); dev != nil { @@ -229,74 +182,13 @@ func (n *NodeManagementWorker) HandleRegistration() { return } -// After a successful download, update the node status in the db and the exchange and create an eventlog event for the change -func (n *NodeManagementWorker) DownloadComplete(cmd *NMPDownloadCompleteCommand) { - status, err := persistence.FindNMPStatus(n.db, cmd.Msg.NMPName) - if err != nil { - glog.Errorf(nmwlog(fmt.Sprintf("Failed to get nmp status %v from the database: %v", cmd.Msg.NMPName, err))) - return - } else if status == nil { - glog.Errorf(nmwlog(fmt.Sprintf("Failed to find status for nmp %v in the database.", cmd.Msg.NMPName))) - return - } - var msgMeta *persistence.MessageMeta - eventCode := "" - - if cmd.Msg.Status == exchangecommon.STATUS_NO_ACTION { - glog.Infof(nmwlog(fmt.Sprintf("Already in compliance with nmp %v. Download skipped.", cmd.Msg.NMPName))) - status.SetStatus(exchangecommon.STATUS_NO_ACTION) - msgMeta = persistence.NewMessageMeta(EL_NMP_STATUS_CHANGED, cmd.Msg.NMPName, exchangecommon.STATUS_NO_ACTION) - eventCode = persistence.EC_NMP_STATUS_DOWNLOAD_SUCCESSFUL - } else if cmd.Msg.Status == exchangecommon.STATUS_DOWNLOADED { - glog.Infof(nmwlog(fmt.Sprintf("Sucessfully downloaded packages for nmp %v.", cmd.Msg.NMPName))) - if dev, err := exchange.GetExchangeDevice(n.GetHTTPFactory(), n.GetExchangeId(), n.GetExchangeId(), n.GetExchangeToken(), n.GetExchangeURL()); err != nil { - glog.Errorf(nmwlog(fmt.Sprintf("Failed to get device from the db: %v", err))) - return - } else if dev.HAGroup != "" { - status.SetStatus(exchangecommon.STATUS_HA_WAITING) - msgMeta = persistence.NewMessageMeta(EL_NMP_STATUS_CHANGED, cmd.Msg.NMPName, exchangecommon.STATUS_HA_WAITING) - eventCode = persistence.EC_NMP_STATUS_CHANGED - } else { - status.SetStatus(exchangecommon.STATUS_DOWNLOADED) - msgMeta = persistence.NewMessageMeta(EL_NMP_STATUS_CHANGED, cmd.Msg.NMPName, exchangecommon.STATUS_DOWNLOADED) - eventCode = persistence.EC_NMP_STATUS_DOWNLOAD_SUCCESSFUL - } - } else if cmd.Msg.Status == exchangecommon.STATUS_PRECHECK_FAILED { - glog.Infof(nmwlog(fmt.Sprintf("Node management policy %v failed precheck conditions. %v", cmd.Msg.NMPName, cmd.Msg.ErrorMessage))) - status.SetStatus(exchangecommon.STATUS_PRECHECK_FAILED) - status.SetErrorMessage(cmd.Msg.ErrorMessage) - msgMeta = persistence.NewMessageMeta(EL_NMP_STATUS_CHANGED_WITH_ERROR, cmd.Msg.NMPName, exchangecommon.STATUS_PRECHECK_FAILED, cmd.Msg.ErrorMessage) - eventCode = persistence.EC_NMP_STATUS_CHANGED - } else { - if status.AgentUpgradeInternal.DownloadAttempts < 4 { - glog.Infof(nmwlog(fmt.Sprintf("Resetting status for %v to waiting to retry failed download.", cmd.Msg.NMPName))) - status.AgentUpgradeInternal.DownloadAttempts = status.AgentUpgradeInternal.DownloadAttempts + 1 - status.SetStatus(exchangecommon.STATUS_NEW) - msgMeta = persistence.NewMessageMeta(EL_NMP_STATUS_CHANGED, cmd.Msg.NMPName, exchangecommon.STATUS_NEW) - eventCode = persistence.EC_NMP_STATUS_CHANGED - } else { - glog.Infof(nmwlog(fmt.Sprintf("Download attempted 3 times already for %v. Download will not be tried again.", cmd.Msg.NMPName))) - glog.Errorf(nmwlog(fmt.Sprintf("Failed to download packages for nmp %v. %v", cmd.Msg.NMPName, cmd.Msg.ErrorMessage))) - status.SetStatus(cmd.Msg.Status) - status.SetErrorMessage(cmd.Msg.ErrorMessage) - msgMeta = persistence.NewMessageMeta(EL_NMP_STATUS_CHANGED_WITH_ERROR, cmd.Msg.NMPName, cmd.Msg.Status, cmd.Msg.ErrorMessage) - eventCode = persistence.EC_NMP_STATUS_CHANGED +func (n *NodeManagementWorker) HandleImageFetchedCommand(cmd *ImageFetchedCommand) { + for _, svc := range cmd.DeploymentDescription.Services { + imgRec := persistence.NewServiceImageUsage(svc.Image) + if err := persistence.SaveOrUpdateServiceImage(n.db, imgRec); err != nil { + glog.Errorf(nmwlog(fmt.Sprintf("Failed to save image download record to db: %v", err))) } } - if cmd.Msg.Versions != nil { - status.AgentUpgrade.UpgradedVersions = *cmd.Msg.Versions - } - if cmd.Msg.Latests != nil { - status.AgentUpgradeInternal.LatestMap = *cmd.Msg.Latests - } - err = n.UpdateStatus(cmd.Msg.NMPName, status, exchange.GetPutNodeManagementPolicyStatusHandler(n), msgMeta, eventCode) - if err != nil { - glog.Errorf(nmwlog(fmt.Sprintf("Failed to update nmp status %v: %v", cmd.Msg.NMPName, err))) - } - - if cmd.Msg.Status == exchangecommon.STATUS_DOWNLOADED { - n.Messages() <- events.NewAgentPackageDownloadedMessage(events.AGENT_PACKAGE_DOWNLOADED, events.StartDownloadMessage{NMPStatus: status, NMPName: cmd.Msg.NMPName}) - } } func (n *NodeManagementWorker) CommandHandler(command worker.Command) bool { @@ -330,6 +222,9 @@ func (n *NodeManagementWorker) CommandHandler(command worker.Command) bool { n.HandleAgentFilesVersionChange(cmd) case *NmpStatusChangeCommand: n.HandleNmpStatusReset() + case *ImageFetchedCommand: + cmd := command.(*ImageFetchedCommand) + n.HandleImageFetchedCommand(cmd) default: return false } @@ -520,6 +415,12 @@ func (n *NodeManagementWorker) NewEvent(incoming events.Message) { case events.CHANGE_NMP_STATUS: n.Commands <- NewNmpStatusChangeCommand(msg) } + case *events.ImageFetchMessage: + msg, _ := incoming.(*events.ImageFetchMessage) + switch msg.Event().Id { + case events.IMAGE_FETCHED: + n.Commands <- NewImageFetchedCommand(msg.DeploymentDescription) + } } } @@ -567,60 +468,6 @@ func (n *NodeManagementWorker) CheckNMPStatus(baseWorkingFile string, statusFile return nil } -// Read and persist the status out of the file -// Update status in the exchange -// If everything is successful, delete the job working dir -func (n *NodeManagementWorker) CollectStatus(workingFolderPath string, policyName string, dbStatus *exchangecommon.NodeManagementPolicyStatus) error { - filePath := path.Join(workingFolderPath, policyName, STATUS_FILE_NAME) - // Read in the status file - if _, err := os.Stat(filePath); err != nil { - return fmt.Errorf("Failed to open status file %v for management job %v. Error was: %v", filePath, policyName, err) - } - if openPath, err := os.Open(filePath); err != nil { - return fmt.Errorf("Failed to open status file %v for management job %v. Errorf was: %v", filePath, policyName, err) - } else { - contents := exchangecommon.NodeManagementPolicyStatus{} - err = json.NewDecoder(openPath).Decode(&contents) - if err != nil { - return fmt.Errorf("Failed to decode status file %v for management job %v. Error was %v.", filePath, policyName, err) - } - - exchDev, err := persistence.FindExchangeDevice(n.db) - if err != nil { - glog.Errorf(nmwlog(fmt.Sprintf("Error getting device from database: %v", err))) - exchDev = nil - } - - status_changed, err := common.SetNodeManagementPolicyStatus(n.db, exchDev, policyName, &contents, dbStatus, - exchange.GetPutNodeManagementPolicyStatusHandler(n), - exchange.GetHTTPDeviceHandler(n), - exchange.GetHTTPPatchDeviceHandler(n)) - if err != nil { - glog.Errorf(nmwlog(fmt.Sprintf("Error saving nmp status for %v: %v", policyName, err))) - return err - } else { - // log the event - if status_changed { - pattern := "" - configState := "" - if exchDev != nil { - pattern = exchDev.Pattern - configState = exchDev.Config.State - } - status_string := contents.AgentUpgrade.Status - if status_string == "" { - status_string = exchangecommon.STATUS_UNKNOWN - } - if contents.AgentUpgrade.ErrorMessage != "" { - status_string += fmt.Sprintf(", ErrorMessage: %v", contents.AgentUpgrade.ErrorMessage) - } - eventlog.LogNodeEvent(n.db, persistence.SEVERITY_INFO, persistence.NewMessageMeta(EL_NMP_STATUS_CHANGED, policyName, status_string), persistence.EC_NMP_STATUS_CHANGED, exchange.GetId(n.GetExchangeId()), exchange.GetOrg(n.GetExchangeId()), pattern, configState) - } - } - } - return nil -} - // Update a given nmp status in the db and the exchange func (n *NodeManagementWorker) UpdateStatus(policyName string, status *exchangecommon.NodeManagementPolicyStatus, putStatusHandler exchange.PutNodeManagementPolicyStatusHandler, eventLogMessageMeta *persistence.MessageMeta, eventCode string) error { org, nodeId := cutil.SplitOrgSpecUrl(n.GetExchangeId()) @@ -643,225 +490,6 @@ func (n *NodeManagementWorker) UpdateStatus(policyName string, status *exchangec return nil } -// Check if the current agent versions are up to date for software, cert and config according to -// the specification of the nmp. The NMP must have at least one 'latest' as the version string. -func IsAgentUpToDate(status *exchangecommon.NodeManagementPolicyStatus, exchAFVs *exchangecommon.AgentFileVersions, db *bolt.DB) (bool, error) { - // get local device info - dev, err := persistence.FindExchangeDevice(db) - if err != nil || dev == nil { - return false, fmt.Errorf("Failed to get device from the local db: %v", err) - } - - if exchAFVs != nil { - // check software version - if status.AgentUpgradeInternal.LatestMap.SoftwareLatest { - versions := exchAFVs.SoftwareVersions - if !IsVersionLatest(versions, version.HORIZON_VERSION) { - return false, nil - } - } - // check config version - if status.AgentUpgradeInternal.LatestMap.ConfigLatest { - versions := exchAFVs.ConfigVersions - - devConfigVer := "" - if dev.SoftwareVersions != nil { - if ver, ok := dev.SoftwareVersions[persistence.CONFIG_VERSION]; ok { - devConfigVer = ver - } - } - - if !IsVersionLatest(versions, devConfigVer) { - return false, nil - } - } - // check certificate version - if status.AgentUpgradeInternal.LatestMap.CertLatest { - versions := exchAFVs.CertVersions - - devCertVer := "" - if dev.SoftwareVersions != nil { - if ver, ok := dev.SoftwareVersions[persistence.CERT_VERSION]; ok { - devCertVer = ver - } - } - - if !IsVersionLatest(versions, devCertVer) { - return false, nil - } - } - } - return true, nil -} - -// Compare status.UpgradedVersions with the AgentFileVersions. -// It returns true if all the versions are up to date. This means -// that the nmp has been processed before with the latest versions. -func IsLatestVersionHandled(status *exchangecommon.NodeManagementPolicyStatus, exchAFVs *exchangecommon.AgentFileVersions) (bool, error) { - - // not handled - if status.AgentUpgrade == nil { - return false, nil - } - - upgradedVersions := status.AgentUpgrade.UpgradedVersions - - if exchAFVs != nil { - // check software version - if status.AgentUpgradeInternal.LatestMap.SoftwareLatest { - versions := exchAFVs.SoftwareVersions - if !IsVersionLatest(versions, upgradedVersions.SoftwareVersion) { - return false, nil - } - } - // check config version - if status.AgentUpgradeInternal.LatestMap.ConfigLatest { - versions := exchAFVs.ConfigVersions - if !IsVersionLatest(versions, upgradedVersions.ConfigVersion) { - return false, nil - } - } - // check certificate version - if status.AgentUpgradeInternal.LatestMap.CertLatest { - versions := exchAFVs.CertVersions - if !IsVersionLatest(versions, upgradedVersions.CertVersion) { - return false, nil - } - } - } - return true, nil -} - -// check if current version is the latest available version. If the number of -// available versions is zero, the current version is considered the latest. -func IsVersionLatest(availibleVers []string, currentVersion string) bool { - if availibleVers != nil && len(availibleVers) != 0 { - sort.Slice(availibleVers, func(i, j int) bool { - comp, _ := semanticversion.CompareVersions(availibleVers[i], availibleVers[j]) - return comp > 0 - }) - - return currentVersion == availibleVers[0] - } - return true -} - -// Check all nmp statuses that specify "latest" for a version, if status is not "downloaded", "download started" or "initiated", then change to "waiting" as there is a new version availible -// If there is no new version for whatever the status has "latest" for, it will be marked successful without executing -func (n *NodeManagementWorker) HandleAgentFilesVersionChange(cmd *AgentFileVersionChangeCommand) { - glog.V(3).Infof(nmwlog(fmt.Sprintf("HandleAgentFilesVersionChange re-evaluating NMPs that request the 'latest' versions."))) - if latestStatuses, err := persistence.FindNMPWithLatestKeywordVersion(n.db); err != nil { - glog.Errorf(nmwlog(fmt.Sprintf("Error getting nmp statuses from db to change to \"waiting\". Error was: %v", err))) - return - } else { - // get agent file versions - exchAFVs, err := exchange.GetNodeUpgradeVersionsHandler(n)() - if err != nil { - glog.Errorf("Failed to get the AgentFileVersion from the exchange. %v", err) - return - } - - needDeferCommand := false - for statusName, status := range latestStatuses { - setStatusToWaiting := false - nmpStatus := status.AgentUpgrade.Status - if nmpStatus == exchangecommon.STATUS_NEW { - glog.V(3).Infof(nmwlog(fmt.Sprintf("The nmp %v is already in 'waiting' status. do nothing.", statusName))) - continue - } else if nmpStatus == exchangecommon.STATUS_DOWNLOADED || nmpStatus == exchangecommon.STATUS_DOWNLOAD_STARTED || nmpStatus == exchangecommon.STATUS_INITIATED || nmpStatus == exchangecommon.STATUS_ROLLBACK_STARTED { - glog.V(3).Infof(nmwlog(fmt.Sprintf("The nmp %v with latest keyword is currently being executed or downloaded (status is %v). Exiting without changing status to \"waiting\", checking this nmp later", statusName, nmpStatus))) - needDeferCommand = true - } else if nmpStatus == exchangecommon.STATUS_DOWNLOAD_FAILED || nmpStatus == exchangecommon.STATUS_FAILED_JOB || nmpStatus == exchangecommon.STATUS_PRECHECK_FAILED || nmpStatus == exchangecommon.STATUS_ROLLBACK_FAILED || nmpStatus == exchangecommon.STATUS_ROLLBACK_SUCCESSFUL { - if isHandled, err := IsLatestVersionHandled(status, exchAFVs); err != nil { - glog.Errorf(nmwlog(fmt.Sprintf("Error checking if the latest versions are previously handled for nmp %v. %v", statusName, err))) - } else if isHandled { - glog.V(3).Infof(nmwlog(fmt.Sprintf("The latest agent versions are previously handled for nmp %v. The status was %v. Exiting without changing status to \"waiting\".", statusName, nmpStatus))) - } else { - setStatusToWaiting = true - } - } else { - if isUpToDate, err := IsAgentUpToDate(status, exchAFVs, n.db); err != nil { - glog.Errorf(nmwlog(fmt.Sprintf("Error checking if the agent versions are up to date for nmp %v. %v", statusName, err))) - } else if isUpToDate { - glog.V(3).Infof(nmwlog(fmt.Sprintf("The agent versions are up to date for nmp %v. Exiting without changing status to \"waiting\".", statusName))) - } else { - setStatusToWaiting = true - } - } - - // set the status to waiting for this nmp - if setStatusToWaiting { - glog.V(3).Infof(nmwlog(fmt.Sprintf("Change status to \"waiting\" for the nmp %v", statusName))) - - // Add startWindow to current time to randomize upgrade start times just like what occurs when an NMP first executes - if status.TimeToStart() { - nmp, err := persistence.FindNodeManagementPolicy(n.db, statusName) - if err != nil { - glog.Errorf(nmwlog(fmt.Sprintf("Error getting nmp from db to check the startWindow value. Error was: %v", err))) - } - if nmp != nil { - status.SetScheduledStartTime(exchangecommon.TIME_NOW_KEYWORD, nmp.LastUpdated, nmp.UpgradeWindowDuration) - } - } - - status.AgentUpgrade.Status = exchangecommon.STATUS_NEW - err = n.UpdateStatus(statusName, status, exchange.GetPutNodeManagementPolicyStatusHandler(n), persistence.NewMessageMeta(EL_NMP_STATUS_CHANGED, statusName, exchangecommon.STATUS_NEW), persistence.EC_NMP_STATUS_UPDATE_NEW) - if err != nil { - glog.Errorf(nmwlog(fmt.Sprintf("Error changing nmp status for %v to \"waiting\". Error was %v.", statusName, err))) - } - } - - } // end for - - if needDeferCommand && cmd != nil { - n.AddDeferredCommand(cmd) - } - } -} - -// This function gets all the 'reset' nmp status from the exchange and set them to -// 'waiting' so that the agent can start re-evaluating them. -func (w *NodeManagementWorker) HandleNmpStatusReset() { - glog.V(3).Infof(nmwlog(fmt.Sprintf("HandleNmpStatusReset re-evaluating NMPs that has the status 'reset'."))) - - // get all the nmps that applies to this node from the exchange - allNmpStatus, err := exchange.GetNodeManagementAllStatuses(w, exchange.GetOrg(w.GetExchangeId()), exchange.GetId(w.GetExchangeId())) - if err != nil { - glog.Errorf(nmwlog(fmt.Sprintf("Error getting all nmp statuses for node %v from the exchange. %v", w.GetExchangeId(), err))) - } else { - glog.V(5).Infof(nmwlog(fmt.Sprintf("GetNodeManagementAllStatuses returns: %v", allNmpStatus))) - } - - // find all nmp status from local db - allLocalStatuses, err := persistence.FindAllNMPStatus(w.db) - if err != nil { - glog.Errorf(nmwlog(fmt.Sprintf("Error getting all nmp statuses from the local database. %v", err))) - } - - // change the status to 'waiting' - if allNmpStatus != nil { - for nmp_name, nmp_status := range allNmpStatus.PolicyStatuses { - if nmp_status.Status() == exchangecommon.STATUS_RESET { - if local_status, ok := allLocalStatuses[nmp_name]; ok { - glog.V(3).Infof(nmwlog(fmt.Sprintf("Change status from \"reset\" to \"waiting\" for the nmp %v", nmp_name))) - - local_status.AgentUpgrade.Status = exchangecommon.STATUS_NEW - if local_status.AgentUpgradeInternal != nil { - local_status.AgentUpgradeInternal.DownloadAttempts = 0 - } - - err = w.UpdateStatus(nmp_name, local_status, exchange.GetPutNodeManagementPolicyStatusHandler(w), persistence.NewMessageMeta(EL_NMP_STATUS_CHANGED, nmp_name, exchangecommon.STATUS_NEW), persistence.EC_NMP_STATUS_UPDATE_NEW) - if err != nil { - glog.Errorf(nmwlog(fmt.Sprintf("Error changing nmp status for %v from \"reset\" to \"waiting\". Error was %v.", nmp_name, err))) - } - } else { - glog.V(3).Infof(nmwlog(fmt.Sprintf("node management status for nmp %v for node %v is set to \"reset\" but the status cannot be found from the local db. Skiping it.", nmp_name, w.GetExchangeId()))) - } - } - } - } -} - func nmwlog(message string) string { return fmt.Sprintf("Node management worker: %v", message) } diff --git a/persistence/service_images.go b/persistence/service_images.go new file mode 100644 index 000000000..5f5a5ad4a --- /dev/null +++ b/persistence/service_images.go @@ -0,0 +1,98 @@ +package persistence + +import ( + "encoding/json" + "fmt" + "github.com/boltdb/bolt" + "regexp" + "time" +) + +// service image table name +const SERVICE_IMAGES = "service_images" + +type ServiceImageUsage struct { + ImageId string `json:"image_id"` + TimeLastUsed uint64 `json:"time_last_used"` +} + +func NewServiceImageUsage(imageId string) *ServiceImageUsage { + return &ServiceImageUsage{ImageId: imageId, TimeLastUsed: uint64(time.Now().Unix())} +} + +func (s ServiceImageUsage) String() string { + return fmt.Sprintf("ImageId: %v, "+ + "TimeLastUsed: %v", + s.ImageId, s.TimeLastUsed) +} + +func (s ServiceImageUsage) ShortString() string { + return s.String() +} + +// save or update the given image info +// image use info is keyed with ImageName:ImageTag +func SaveOrUpdateServiceImage(db *bolt.DB, serviceImage *ServiceImageUsage) error { + writeErr := db.Update(func(tx *bolt.Tx) error { + if bucket, err := tx.CreateBucketIfNotExists([]byte(SERVICE_IMAGES)); err != nil { + return err + } else if serial, err := json.Marshal(serviceImage); err != nil { + return fmt.Errorf("Failed to serialize service image usage: %v", err) + } else { + return bucket.Put([]byte(serviceImage.ImageId), serial) + } + }) + + return writeErr +} + +func DeleteServiceImage(db *bolt.DB, imageId string) error { + return db.Update(func(tx *bolt.Tx) error { + if bucket, err := tx.CreateBucketIfNotExists([]byte(SERVICE_IMAGES)); err != nil { + return err + } else if err := bucket.Delete([]byte(imageId)); err != nil { + return fmt.Errorf("Unable to delete service image usage record for %v: %v.", imageId, err) + } + return nil + }) +} + +func FindServiceImageUsageWithFilters(db *bolt.DB, filters []IUFilter) ([]ServiceImageUsage, error) { + imgUsages := make([]ServiceImageUsage, 0) + + readErr := db.View(func(tx *bolt.Tx) error { + if bucket := tx.Bucket([]byte(SERVICE_IMAGES)); bucket != nil { + bucket.ForEach(func(k, v []byte) error { + imgRec := ServiceImageUsage{} + if err := json.Unmarshal(v, &imgRec); err != nil { + return fmt.Errorf("Unable to deserialize service image usage record %v: %v", k, err) + } else { + exclude := false + for _, filter := range filters { + if !filter(imgRec) { + exclude = true + } + } + if !exclude { + imgUsages = append(imgUsages, imgRec) + } + } + return nil + }) + } + return nil + }) + + return imgUsages, readErr +} + +type IUFilter func(ServiceImageUsage) bool + +func ImageNameRegexFilter(imageId string) IUFilter { + regEx, err := regexp.Compile(imageId) + if err != nil { + return func(iu ServiceImageUsage) bool { return false } + } + + return func(iu ServiceImageUsage) bool { return regEx.Match([]byte(iu.ImageId)) } +} diff --git a/resource/certificate.go b/resource/certificate.go index 51796eb83..6f7d854b3 100644 --- a/resource/certificate.go +++ b/resource/certificate.go @@ -10,6 +10,7 @@ import ( "fmt" "github.com/golang/glog" "github.com/open-horizon/anax/config" + "github.com/open-horizon/anax/cutil" "github.com/open-horizon/anax/i18n" "github.com/open-horizon/edge-sync-service/common" "math/big" @@ -52,6 +53,12 @@ func CreateCertificate(org string, keyPath string, certPath string) error { return errors.New(msgPrinter.Sprintf("unable to generate private key for MMS API certificate, error %v", err)) } + ipAddress := []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")} + + agentNS := cutil.GetClusterNamespace() + dnsName1 := fmt.Sprintf("agent-service.%v.svc.cluster.local", agentNS) + dnsName2 := fmt.Sprintf("agent-service.%v.svc", agentNS) + template := x509.Certificate{ SerialNumber: serialNumber, Subject: pkix.Name{ @@ -65,8 +72,8 @@ func CreateCertificate(org string, keyPath string, certPath string) error { KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, BasicConstraintsValid: true, - DNSNames: []string{"localhost"}, - IPAddresses: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}, + DNSNames: []string{"localhost", "e2edevtest", dnsName1, "agent-service", dnsName2}, + IPAddresses: ipAddress, } derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) diff --git a/resource/resource_manager.go b/resource/resource_manager.go index 25a3b3138..375a79b9f 100644 --- a/resource/resource_manager.go +++ b/resource/resource_manager.go @@ -7,6 +7,7 @@ import ( "github.com/golang/glog" "github.com/open-horizon/anax/config" "github.com/open-horizon/anax/exchange" + "github.com/open-horizon/anax/persistence" "github.com/open-horizon/edge-sync-service/common" "github.com/open-horizon/edge-sync-service/core/base" "github.com/open-horizon/edge-sync-service/core/security" @@ -20,23 +21,25 @@ import ( ) type ResourceManager struct { - config *config.HorizonConfig - org string - pattern string - id string - token string + config *config.HorizonConfig + org string + pattern string + id string + token string + nodeType string } -func NewResourceManager(cfg *config.HorizonConfig, org string, pattern string, id string, token string) *ResourceManager { +func NewResourceManager(cfg *config.HorizonConfig, org string, pattern string, id string, token string, nodeType string) *ResourceManager { if id != "" && pattern == "" { pattern = "openhorizon/openhorizon.edgenode" } return &ResourceManager{ - config: cfg, - pattern: pattern, - id: id, - org: org, - token: token, + config: cfg, + pattern: pattern, + id: id, + org: org, + token: token, + nodeType: nodeType, } } @@ -44,19 +47,21 @@ func (r *ResourceManager) Configured() bool { return r.id != "" } -func (r *ResourceManager) NodeConfigUpdate(org string, pattern string, id string, token string) { +func (r *ResourceManager) NodeConfigUpdate(org string, pattern string, id string, token string, nodeType string) { r.pattern = pattern r.id = id r.org = org r.token = token + r.nodeType = nodeType } func (r ResourceManager) String() string { return fmt.Sprintf("ResourceManager: Org %v"+ ", Pattern: %v"+ ", ID: %v"+ - ", Token: %v", - r.org, r.pattern, r.id, r.token) + ", Token: %v"+ + ", NodeType: %v", + r.org, r.pattern, r.id, r.token, r.nodeType) } func (r ResourceManager) setupFileSyncService(am *AuthenticationManager) error { @@ -85,7 +90,7 @@ func (r ResourceManager) setupFileSyncService(am *AuthenticationManager) error { } else if essCertKeyBytes, err := ioutil.ReadAll(essCertKey); err != nil { return errors.New(fmt.Sprintf("unable to read ESS SSL Certificate Key file %v, error %v", r.config.GetESSSSLCertKeyPath(), err)) } else { - // create path for ListeningAddress if it does not exist + // For device agent, create path for ListeningAddress if it does not exist listenAddrPath := r.config.GetFileSyncServiceAPIUnixDomainSocketPath() if listenAddrPath != "" { if _, err := os.Stat(listenAddrPath); os.IsNotExist(err) { @@ -98,7 +103,7 @@ func (r ResourceManager) setupFileSyncService(am *AuthenticationManager) error { common.Configuration.DestinationType = exchange.GetId(r.pattern) common.Configuration.DestinationID = r.id common.Configuration.OrgID = r.org - common.Configuration.ListeningType = r.config.GetFileSyncServiceProtocol() + common.Configuration.ListeningType = r.config.GetFileSyncServiceProtocol() // secure for cluster, unix-secure for e2edev and device common.Configuration.ListeningAddress = r.config.GetFileSyncServiceAPIListen() common.Configuration.SecureListeningPort = r.config.GetFileSyncServiceAPIPort() common.Configuration.ServerCertificate = string(essCertBytes) @@ -107,7 +112,7 @@ func (r ResourceManager) setupFileSyncService(am *AuthenticationManager) error { common.Configuration.EnableDataChunk = r.config.IsDataChunkEnabled() common.Configuration.MaxDataChunkSize = r.config.GetFileSyncServiceMaxDataChunkSize() common.Configuration.HTTPPollingInterval = r.config.GetESSPollingRate() - common.Configuration.PersistenceRootPath = r.config.GetFileSyncServiceStoragePath() + common.Configuration.PersistenceRootPath = r.config.GetFileSyncServiceStoragePath() // /var/horizon/ess-store/, the db file will be inside: /var/horizon/ess-store/sync/db/ess-sync.db common.Configuration.HTTPCSSUseSSL = true common.Configuration.HTTPCSSCACertificate = r.config.GetCSSSSLCert() common.Configuration.LogTraceDestination = "glog" @@ -116,6 +121,11 @@ func (r ResourceManager) setupFileSyncService(am *AuthenticationManager) error { common.Configuration.HTTPESSObjClientTimeout = r.config.GetHTTPESSObjClientTimeout() } + if r.nodeType == persistence.DEVICE_TYPE_CLUSTER { + common.Configuration.ServerCertificate = certFile + common.Configuration.ServerKey = certKeyFile + } + if glog.V(5) { common.Configuration.LogLevel = "TRACE" common.Configuration.TraceLevel = "TRACE" @@ -156,9 +166,7 @@ func (r ResourceManager) setupFileSyncService(am *AuthenticationManager) error { // Set the authenticator that we're going to use. security.SetAuthentication(&FSSAuthenticate{nodeOrg: r.org, nodeID: r.id, nodeToken: r.token, AuthMgr: am}) - return nil - } func (r ResourceManager) setupSecretsAPI(am *AuthenticationManager, db *bolt.DB) { diff --git a/resource/resource_worker.go b/resource/resource_worker.go index 49daed5b1..32c18dd9b 100644 --- a/resource/resource_worker.go +++ b/resource/resource_worker.go @@ -25,19 +25,18 @@ func NewResourceWorker(name string, config *config.HorizonConfig, db *bolt.DB, a dev, _ := persistence.FindExchangeDevice(db) if dev != nil { ec = worker.NewExchangeContext(fmt.Sprintf("%v/%v", dev.Org, dev.Id), dev.Token, config.Edge.ExchangeURL, config.GetCSSURL(), config.Edge.AgbotURL, config.Collaborators.HTTPClientFactory) - if !dev.IsEdgeCluster() { - if config == nil || config.GetCSSURL() == "" { - term_string := "Terminating, unable to start model management resource manager. Please set either CSSURL in the anax configuration file or HZN_FSS_CSSURL in /etc/default/horizon file." - glog.Errorf(term_string) - panic(term_string) - } - - rm = NewResourceManager(config, dev.Org, dev.Pattern, dev.Id, dev.Token) + if config == nil || config.GetCSSURL() == "" { + term_string := "Terminating, unable to start model management resource manager. Please set either CSSURL in the anax configuration file or HZN_FSS_CSSURL in /etc/default/horizon file." + glog.Errorf(term_string) + panic(term_string) } + + rm = NewResourceManager(config, dev.Org, dev.Pattern, dev.Id, dev.Token, dev.NodeType) + } if rm == nil { - rm = NewResourceManager(config, "", "", "", "") + rm = NewResourceManager(config, "", "", "", "", "") } worker := &ResourceWorker{ @@ -139,8 +138,6 @@ func (w *ResourceWorker) handleNodeConfigCommand(cmd *NodeConfigCommand) error { return err } else if dev == nil { return errors.New("no device object in local DB") - } else if dev.IsEdgeCluster() { - return nil } else { if w.Config == nil || w.Config.GetCSSURL() == "" { term_string := "Terminating, unable to start model management resource manager. Please set either CSSURL in the anax configuration file or HZN_FSS_CSSURL in /etc/default/horizon file." @@ -154,7 +151,7 @@ func (w *ResourceWorker) handleNodeConfigCommand(cmd *NodeConfigCommand) error { if destinationType == "" { destinationType = "openhorizon/openhorizon.edgenode" } - w.rm.NodeConfigUpdate(cmd.msg.Org(), destinationType, cmd.msg.DeviceId(), cmd.msg.Token()) + w.rm.NodeConfigUpdate(cmd.msg.Org(), destinationType, cmd.msg.DeviceId(), cmd.msg.Token(), cmd.msg.DeviceType()) return w.rm.StartFileSyncServiceAndSecretsAPI(w.am, w.db) } diff --git a/test/gov/hzn_dev_services.sh b/test/gov/hzn_dev_services.sh index 94b5795ec..591b810c8 100755 --- a/test/gov/hzn_dev_services.sh +++ b/test/gov/hzn_dev_services.sh @@ -268,7 +268,7 @@ if [ "${startedServices}" != "${NUMBER_SERVICES}" ]; then fi echo -e "Waiting for services to run a bit before stopping them." -sleep 15 +sleep 60 containers=$(docker ps -a) restarting=$(echo ${containers} | grep "Restarting")