Merge pull request #233 from chengxiangdong/fix_node_addr

Fix the issues of querying node external IPs

go.mod

@@ -5,7 +5,6 @@ go 1.19
 require (
 	github.com/fsnotify/fsnotify v1.6.0
 	github.com/huaweicloud/huaweicloud-sdk-go-v3 v0.1.16
-	github.com/mitchellh/mapstructure v1.4.1
 	github.com/onsi/ginkgo/v2 v2.6.1
 	github.com/onsi/gomega v1.24.1
 	github.com/patrickmn/go-cache v2.1.0+incompatible

go.sum

@@ -255,8 +255,6 @@ github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM=
 github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
-github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
-github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae h1:O4SWKdcHVCvYqyDV+9CJA1fcDN2L11Bule0iFy3YlAI=
 github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=

hack/deploy/huawei-cloud-controller-manager.yaml

@@ -186,6 +186,11 @@ spec:
             - --cloud-provider=huaweicloud
             - --use-service-account-credentials=true
             - --bind-address=127.0.0.1
+            - --node-status-update-frequency=5s
+            - --node-monitor-period=5s
+            - --leader-elect-lease-duration=30s
+            - --leader-elect-renew-deadline=20s
+            - --leader-elect-retry-period=2s
           volumeMounts:
             - mountPath: /etc/kubernetes
               name: k8s-certs

manifests/huawei-cloud-controller-manager.yaml

@@ -51,6 +51,11 @@ spec:
             - --cloud-config=/etc/config/cloud-config
             - --cloud-provider=huaweicloud
             - --use-service-account-credentials=true
+            - --node-status-update-frequency=5s
+            - --node-monitor-period=5s
+            - --leader-elect-lease-duration=30s
+            - --leader-elect-renew-deadline=20s
+            - --leader-elect-retry-period=2s
           volumeMounts:
             - mountPath: /etc/kubernetes
               name: k8s-certs

pkg/cloudprovider/huaweicloud/instances.go

@@ -53,7 +53,7 @@ func (i *Instances) NodeAddresses(ctx context.Context, name types.NodeName) ([]v
 
 // NodeAddressesByProviderID returns the addresses of the specified instance.
 func (i *Instances) NodeAddressesByProviderID(_ context.Context, providerID string) ([]v1.NodeAddress, error) {
-	klog.Infof("NodeAddressesByProviderID is called witd provider ID %s", providerID)
+	klog.Infof("NodeAddressesByProviderID is called with provider ID %s", providerID)
 	instanceID, err := parseInstanceID(providerID)
 	if err != nil {
 		return nil, err
@@ -224,6 +224,8 @@ func (i *Instances) InstanceMetadata(ctx context.Context, node *v1.Node) (*cloud
 	}
 
 	return &cloudprovider.InstanceMetadata{
+		Region:        i.cloudConfig.AuthOpts.Region,
+		Zone:          instance.OSEXTAZavailabilityZone,
 		ProviderID:    providerID,
 		InstanceType:  instanceFlavor,
 		NodeAddresses: addresses,

pkg/cloudprovider/huaweicloud/wrapper/ecs.go

@@ -26,7 +26,6 @@ import (
 	"github.com/huaweicloud/huaweicloud-sdk-go-v3/core/sdkerr"
 	ecs "github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ecs/v2"
 	"github.com/huaweicloud/huaweicloud-sdk-go-v3/services/ecs/v2/model"
-	"github.com/mitchellh/mapstructure"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	v1 "k8s.io/api/core/v1"
@@ -128,14 +127,14 @@ func (e *EcsClient) ListInterfaces(req *model.ListServerInterfacesRequest) ([]mo
 
 func (e *EcsClient) BuildAddresses(server *model.ServerDetail, interfaces []model.InterfaceAttachment,
 	networkingOpts *config.NetworkingOptions) ([]v1.NodeAddress, error) {
-	addrs := []v1.NodeAddress{}
+	nodeAddresses := make([]v1.NodeAddress, 0)
 
 	// parse private IP addresses first in an ordered manner
-	for _, iface := range interfaces {
-		for _, fixedIP := range *iface.FixedIps {
-			if *iface.PortState == "ACTIVE" {
+	for _, inter := range interfaces {
+		if *inter.PortState == "ACTIVE" {
+			for _, fixedIP := range *inter.FixedIps {
 				if net.ParseIP(*fixedIP.IpAddress).To4() != nil {
-					addToNodeAddresses(&addrs,
+					addToNodeAddresses(&nodeAddresses,
 						v1.NodeAddress{
 							Type:    v1.NodeInternalIP,
 							Address: *fixedIP.IpAddress,
@@ -148,75 +147,63 @@ func (e *EcsClient) BuildAddresses(server *model.ServerDetail, interfaces []mode
 
 	// process public IP addresses
 	if server.AccessIPv4 != "" {
-		addToNodeAddresses(&addrs,
+		addToNodeAddresses(&nodeAddresses,
 			v1.NodeAddress{
 				Type:    v1.NodeExternalIP,
 				Address: server.AccessIPv4,
 			},
 		)
 	}
 
-	// process the rest
-	type Address struct {
-		IPType string `mapstructure:"OS-EXT-IPS:type"`
-		Addr   string
+	nicIDs := make([]string, 0)
+	for nicID := range server.Addresses {
+		nicIDs = append(nicIDs, nicID)
 	}
+	sort.Strings(nicIDs)
 
-	var addresses map[string][]Address
-	err := mapstructure.Decode(server.Addresses, &addresses)
-	if err != nil {
-		return nil, err
-	}
-
-	var networks []string
-	for k := range addresses {
-		networks = append(networks, k)
-	}
-	sort.Strings(networks)
-
-	for _, network := range networks {
-		for _, props := range addresses[network] {
+	for _, nicID := range nicIDs {
+		for _, serverAddr := range server.Addresses[nicID] {
 			var addressType v1.NodeAddressType
-			if props.IPType == "floating" {
+			if serverAddr.OSEXTIPStype != nil && serverAddr.OSEXTIPStype.Value() == "floating" {
 				addressType = v1.NodeExternalIP
-			} else if utils.IsStrSliceContains(networkingOpts.PublicNetworkName, network) {
+			} else if utils.IsStrSliceContains(networkingOpts.PublicNetworkName, nicID) {
 				addressType = v1.NodeExternalIP
 				// removing already added address to avoid listing it as both ExternalIP and InternalIP
 				// may happen due to listing "private" network as "public" in CCM's CloudConfig
-				removeFromNodeAddresses(&addrs,
+				removeFromNodeAddresses(&nodeAddresses,
 					v1.NodeAddress{
-						Address: props.Addr,
+						Address: serverAddr.Addr,
 					},
 				)
 			} else {
 				if len(networkingOpts.InternalNetworkName) == 0 ||
-					utils.IsStrSliceContains(networkingOpts.InternalNetworkName, network) {
+					utils.IsStrSliceContains(networkingOpts.InternalNetworkName, nicID) {
 					addressType = v1.NodeInternalIP
 				} else {
-					klog.V(4).Infof("[DEBUG] Node '%s' address '%s' "+
-						"ignored due to 'internal-network-name' option", server.Name, props.Addr)
+					klog.V(4).Infof("[DEBUG] Node '%s' address '%s' ignored due to 'internal-network-name' option",
+						server.Name, serverAddr.Addr)
 
-					removeFromNodeAddresses(&addrs,
+					removeFromNodeAddresses(&nodeAddresses,
 						v1.NodeAddress{
-							Address: props.Addr,
+							Address: serverAddr.Addr,
 						},
 					)
 					continue
 				}
 			}
 
-			if net.ParseIP(props.Addr).To4() != nil {
-				addToNodeAddresses(&addrs,
+			if net.ParseIP(serverAddr.Addr).To4() != nil {
+				addToNodeAddresses(&nodeAddresses,
 					v1.NodeAddress{
 						Type:    addressType,
-						Address: props.Addr,
+						Address: serverAddr.Addr,
 					},
 				)
 			}
 		}
 	}
-
-	return addrs, nil
+	klog.V(6).Infof("server: %s/%s, network addresses: %s", server.Name, server.Id, utils.ToString(nodeAddresses))
+	return nodeAddresses, nil
 }
 
 func (e *EcsClient) ListSecurityGroups(instanceID string) ([]model.NovaSecurityGroup, error) {