From 3dd3fe732b6d61928867c8d6a6c8957636991cc0 Mon Sep 17 00:00:00 2001 From: bzsuni Date: Tue, 25 Jul 2023 18:44:54 +0800 Subject: [PATCH] update egci crd Signed-off-by: bzsuni --- charts/README.md | 38 +- ...teway.spidernet.io_egressclusterinfos.yaml | 90 ++- charts/templates/egressClusterInfo.yaml | 10 + charts/values.yaml | 14 +- pkg/agent/police.go | 31 +- pkg/config/config.go | 42 +- pkg/controller/controller.go | 2 +- pkg/controller/egress_cluster_info.go | 698 ++++++++++-------- .../apis/v1beta1/egressclusterinfo_types.go | 41 +- pkg/k8s/apis/v1beta1/zz_generated.deepcopy.go | 122 +-- test/e2e/common/egressconfigmap.go | 14 +- 11 files changed, 617 insertions(+), 485 deletions(-) create mode 100644 charts/templates/egressClusterInfo.yaml diff --git a/charts/README.md b/charts/README.md index 01de38e67..fda152aa0 100644 --- a/charts/README.md +++ b/charts/README.md @@ -23,25 +23,25 @@ helm install egressgateway egressgateway/egressgateway --namespace kube-system ### Feature parameters -| Name | Description | Value | -| ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | ----------------------- | -| `feature.enableIPv4` | Enable IPv4 | `true` | -| `feature.enableIPv6` | Enable IPv6 | `true` | -| `feature.datapathMode` | iptables mode, [`iptables`, `ebpf`] | `iptables` | -| `feature.tunnelIpv4Subnet` | Tunnel IPv4 subnet | `172.31.0.0/16` | -| `feature.tunnelIpv6Subnet` | Tunnel IPv6 subnet | `fd11::/112` | -| `feature.tunnelDetectMethod` | Tunnel base on which interface [`defaultRouteInterface`, `interface=eth0`] | `defaultRouteInterface` | -| `feature.iptables.backendMode` | Iptables mode can be specified as `nft` or `legacy`, with `auto` meaning automatic detection. The default value is `auto`. | `auto` | -| `feature.vxlan.name` | The name of VXLAN device | `egress.vxlan` | -| `feature.vxlan.port` | VXLAN port | `7789` | -| `feature.vxlan.id` | VXLAN ID | `100` | -| `feature.vxlan.disableChecksumOffload` | Disable checksum offload | `true` | -| `feature.egressIgnoreCIDR.autoDetect.podCIDR` | cni cluster used | `calico` | -| `feature.egressIgnoreCIDR.autoDetect.clusterIP` | if ignore service ip | `true` | -| `feature.egressIgnoreCIDR.autoDetect.nodeIP` | if ignore node ip | `true` | -| `feature.egressIgnoreCIDR.custom` | CIDRs provided manually | `[]` | -| `feature.maxNumberEndpointPerSlice` | max number of endpoints per slice | `100` | -| `feature.announcedInterfacesToExclude` | The list of network interface excluded for announcing Egress IP. | `["^cali.*","br-*"]` | +| Name | Description | Value | +| -------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------- | ----------------------- | +| `feature.enableIPv4` | Enable IPv4 | `true` | +| `feature.enableIPv6` | Enable IPv6 | `true` | +| `feature.datapathMode` | iptables mode, [`iptables`, `ebpf`] | `iptables` | +| `feature.tunnelIpv4Subnet` | Tunnel IPv4 subnet | `172.31.0.0/16` | +| `feature.tunnelIpv6Subnet` | Tunnel IPv6 subnet | `fd11::/112` | +| `feature.tunnelDetectMethod` | Tunnel base on which interface [`defaultRouteInterface`, `interface=eth0`] | `defaultRouteInterface` | +| `feature.iptables.backendMode` | Iptables mode can be specified as `nft` or `legacy`, with `auto` meaning automatic detection. The default value is `auto`. | `auto` | +| `feature.vxlan.name` | The name of VXLAN device | `egress.vxlan` | +| `feature.vxlan.port` | VXLAN port | `7789` | +| `feature.vxlan.id` | VXLAN ID | `100` | +| `feature.vxlan.disableChecksumOffload` | Disable checksum offload | `true` | +| `feature.clusterCIDR.autoDetect.podCidrMode` | cni cluster used, it can be specified as `k8s`, `calico` or `""` | `k8s` | +| `feature.clusterCIDR.autoDetect.clusterIP` | if ignore service ip | `true` | +| `feature.clusterCIDR.autoDetect.nodeIP` | if ignore node ip | `true` | +| `feature.clusterCIDR.extraCidr` | CIDRs provided manually | `[]` | +| `feature.maxNumberEndpointPerSlice` | max number of endpoints per slice | `100` | +| `feature.announcedInterfacesToExclude` | The list of network interface excluded for announcing Egress IP. | `["^cali.*","br-*"]` | ### Egressgateway agent parameters diff --git a/charts/crds/egressgateway.spidernet.io_egressclusterinfos.yaml b/charts/crds/egressgateway.spidernet.io_egressclusterinfos.yaml index 32ff72c83..1cd834c26 100644 --- a/charts/crds/egressgateway.spidernet.io_egressclusterinfos.yaml +++ b/charts/crds/egressgateway.spidernet.io_egressclusterinfos.yaml @@ -36,44 +36,66 @@ spec: metadata: type: object spec: - type: object - status: properties: - egressIgnoreCIDR: + autoDetect: properties: clusterIP: - properties: - ipv4: - items: - type: string - type: array - ipv6: - items: - type: string - type: array - type: object + default: true + type: boolean nodeIP: - properties: - ipv4: - items: - type: string - type: array - ipv6: - items: - type: string - type: array - type: object - podCIDR: - properties: - ipv4: - items: - type: string - type: array - ipv6: - items: - type: string - type: array - type: object + default: true + type: boolean + podCidrMode: + default: k8s + type: string + type: object + extraCidr: + items: + type: string + type: array + type: object + status: + properties: + clusterIP: + properties: + ipv4: + items: + type: string + type: array + ipv6: + items: + type: string + type: array + type: object + extraCidr: + items: + type: string + type: array + nodeIP: + additionalProperties: + properties: + ipv4: + items: + type: string + type: array + ipv6: + items: + type: string + type: array + type: object + type: object + podCIDR: + additionalProperties: + properties: + ipv4: + items: + type: string + type: array + ipv6: + items: + type: string + type: array + type: object type: object type: object required: diff --git a/charts/templates/egressClusterInfo.yaml b/charts/templates/egressClusterInfo.yaml new file mode 100644 index 000000000..3f3aa6818 --- /dev/null +++ b/charts/templates/egressClusterInfo.yaml @@ -0,0 +1,10 @@ +apiVersion: egressgateway.spidernet.io/v1beta1 +kind: EgressClusterInfo +metadata: + name: default +spec: + autoDetect: + clusterIP: {{ .Values.feature.clusterCIDR.autoDetect.clusterIP }} + podCidrMode: {{ .Values.feature.clusterCIDR.autoDetect.podCidrMode }} + nodeIP: {{ .Values.feature.clusterCIDR.autoDetect.nodeIP }} + extraCidr: {{ .Values.feature.clusterCIDR.extraCidr }} diff --git a/charts/values.yaml b/charts/values.yaml index 4f3799557..733e0316f 100644 --- a/charts/values.yaml +++ b/charts/values.yaml @@ -46,16 +46,16 @@ feature: id: 100 ## @param feature.vxlan.disableChecksumOffload Disable checksum offload disableChecksumOffload: true - egressIgnoreCIDR: + clusterCIDR: autoDetect: - ## @param feature.egressIgnoreCIDR.autoDetect.podCIDR cni cluster used - podCIDR: "calico" - ## @param feature.egressIgnoreCIDR.autoDetect.clusterIP if ignore service ip + ## @param feature.clusterCIDR.autoDetect.podCidrMode cni cluster used, it can be specified as `k8s`, `calico` or `""` + podCidrMode: "k8s" + ## @param feature.clusterCIDR.autoDetect.clusterIP if ignore service ip clusterIP: true - ## @param feature.egressIgnoreCIDR.autoDetect.nodeIP if ignore node ip + ## @param feature.clusterCIDR.autoDetect.nodeIP if ignore node ip nodeIP: true - ## @param feature.egressIgnoreCIDR.custom CIDRs provided manually - custom: [] + ## @param feature.clusterCIDR.extraCidr CIDRs provided manually + extraCidr: [] ## @param feature.maxNumberEndpointPerSlice max number of endpoints per slice maxNumberEndpointPerSlice: 100 ## @param feature.announcedInterfacesToExclude The list of network interface excluded for announcing Egress IP. diff --git a/pkg/agent/police.go b/pkg/agent/police.go index 6d29cfc56..79592fe73 100644 --- a/pkg/agent/police.go +++ b/pkg/agent/police.go @@ -565,8 +565,16 @@ func (r *policeReconciler) reconcileClusterInfo(ctx context.Context, req reconci } } - addIP(info.Status.EgressIgnoreCIDR.NodeIP.IPv4...) - addIP(info.Status.EgressIgnoreCIDR.NodeIP.IPv6...) + nodesIPv4 := make([]string, 0) + for _, pair := range info.Status.NodeIP { + nodesIPv4 = append(nodesIPv4, pair.IPv4...) + } + nodesIPv6 := make([]string, 0) + for _, pair := range info.Status.NodeIP { + nodesIPv6 = append(nodesIPv6, pair.IPv6...) + } + addIP(nodesIPv4...) + addIP(nodesIPv6...) addCIDR := func(items ...string) { for _, item := range items { @@ -582,13 +590,22 @@ func (r *policeReconciler) reconcileClusterInfo(ctx context.Context, req reconci } } - addCIDR(info.Status.EgressIgnoreCIDR.PodCIDR.IPv4...) - addCIDR(info.Status.EgressIgnoreCIDR.PodCIDR.IPv6...) + v4PodCidrs := make([]string, 0) + for _, pair := range info.Status.PodCIDR { + v4PodCidrs = append(v4PodCidrs, pair.IPv4...) + } + v6PodCidrs := make([]string, 0) + for _, pair := range info.Status.PodCIDR { + v6PodCidrs = append(v6PodCidrs, pair.IPv6...) + } + + addCIDR(v4PodCidrs...) + addCIDR(v6PodCidrs...) - addCIDR(info.Status.EgressIgnoreCIDR.ClusterIP.IPv4...) - addCIDR(info.Status.EgressIgnoreCIDR.ClusterIP.IPv6...) + addCIDR(info.Status.ClusterIP.IPv4...) + addCIDR(info.Status.ClusterIP.IPv6...) - addCIDR(r.cfg.FileConfig.EgressIgnoreCIDR.Custom...) + addCIDR(info.Status.ExtraCidr...) process := func(gotList []string, expList []string, toAdd, toDel func(item string) error) error { got := sets.NewString(gotList...) diff --git a/pkg/config/config.go b/pkg/config/config.go index 6b451a451..8c8d33819 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -66,21 +66,20 @@ type EnvConfig struct { } type FileConfig struct { - EnableIPv4 bool `yaml:"enableIPv4"` - EnableIPv6 bool `yaml:"enableIPv6"` - IPTables IPTables `yaml:"iptables"` - DatapathMode string `yaml:"datapathMode"` - TunnelIpv4Subnet string `yaml:"tunnelIpv4Subnet"` - TunnelIpv6Subnet string `yaml:"tunnelIpv6Subnet"` - TunnelIPv4Net *net.IPNet `json:"-"` - TunnelIPv6Net *net.IPNet `json:"-"` - TunnelDetectMethod string `yaml:"tunnelDetectMethod"` - VXLAN VXLAN `yaml:"vxlan"` - EgressIgnoreCIDR EgressIgnoreCIDR `yaml:"egressIgnoreCIDR"` - MaxNumberEndpointPerSlice int `yaml:"maxNumberEndpointPerSlice"` - Mark string `yaml:"mark"` - AnnouncedInterfacesToExclude []string `yaml:"announcedInterfacesToExclude"` - AnnounceExcludeRegexp *regexp.Regexp `json:"-"` + EnableIPv4 bool `yaml:"enableIPv4"` + EnableIPv6 bool `yaml:"enableIPv6"` + IPTables IPTables `yaml:"iptables"` + DatapathMode string `yaml:"datapathMode"` + TunnelIpv4Subnet string `yaml:"tunnelIpv4Subnet"` + TunnelIpv6Subnet string `yaml:"tunnelIpv6Subnet"` + TunnelIPv4Net *net.IPNet `json:"-"` + TunnelIPv6Net *net.IPNet `json:"-"` + TunnelDetectMethod string `yaml:"tunnelDetectMethod"` + VXLAN VXLAN `yaml:"vxlan"` + MaxNumberEndpointPerSlice int `yaml:"maxNumberEndpointPerSlice"` + Mark string `yaml:"mark"` + AnnouncedInterfacesToExclude []string `yaml:"announcedInterfacesToExclude"` + AnnounceExcludeRegexp *regexp.Regexp `json:"-"` } const TunnelInterfaceDefaultRoute = "defaultRouteInterface" @@ -104,11 +103,6 @@ type IPTables struct { LockFilePath string `yaml:"lockFilePath"` } -type EgressIgnoreCIDR struct { - AutoDetect `yaml:"autoDetect"` - Custom []string `yaml:"custom"` -} - type AutoDetect struct { PodCIDR string `yaml:"podCIDR"` ClusterIP bool `yaml:"clusterIP"` @@ -151,14 +145,6 @@ func LoadConfig(isAgent bool) (*Config, error) { LockFilePath: "/run/xtables.lock", RestoreSupportsLock: restoreSupportsLock, }, - EgressIgnoreCIDR: EgressIgnoreCIDR{ - AutoDetect: AutoDetect{ - PodCIDR: "", - ClusterIP: true, - NodeIP: true, - }, - Custom: []string{}, - }, Mark: "0x26000000", }, } diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 4a2f930f0..53b67e884 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -94,7 +94,7 @@ func New(cfg *config.Config) (types.Service, error) { if err != nil { return nil, fmt.Errorf("failed to create egress node controller: %w", err) } - err = newEgressClusterInfoController(mgr, log, cfg) + err = newEgressClusterInfoController(mgr, log) if err != nil { return nil, fmt.Errorf("failed to create egress cluster info controller: %w", err) } diff --git a/pkg/controller/egress_cluster_info.go b/pkg/controller/egress_cluster_info.go index f1c9dc6d1..822a1bec1 100644 --- a/pkg/controller/egress_cluster_info.go +++ b/pkg/controller/egress_cluster_info.go @@ -6,11 +6,11 @@ package controller import ( "context" "fmt" + "reflect" "strings" "sync" + "time" - "github.com/go-logr/logr" - calicov1 "github.com/tigera/operator/pkg/apis/crd.projectcalico.org/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" @@ -21,33 +21,66 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - "github.com/spidernet-io/egressgateway/pkg/config" + "github.com/go-logr/logr" + + calicov1 "github.com/tigera/operator/pkg/apis/crd.projectcalico.org/v1" + egressv1beta1 "github.com/spidernet-io/egressgateway/pkg/k8s/apis/v1beta1" "github.com/spidernet-io/egressgateway/pkg/utils" ) type eciReconciler struct { - eci *egressv1beta1.EgressClusterInfo - client client.Client - log logr.Logger - config *config.Config - doOnce sync.Once - nodeIPv4Map map[string]string - nodeIPv6Map map[string]string - calicoV4IPPoolMap map[string]string - calicoV6IPPoolMap map[string]string + mgr manager.Manager + c controller.Controller + ignoreCalico bool + k8sPodCidr map[string]egressv1beta1.IPListPair + v4ClusterCidr, v6ClusterCidr []string + eci *egressv1beta1.EgressClusterInfo + client client.Client + log logr.Logger + doOnce sync.Once + eciMutex sync.RWMutex } const ( defaultEgressClusterInfoName = "default" - calico = "calico" k8s = "k8s" serviceClusterIpRange = "service-cluster-ip-range" clusterCidr = "cluster-cidr" ) +const ( + kindNode = "Node" + kindCalicoIPPool = "CalicoIPPool" + kindEGCI = "EGCI" +) + var kubeControllerManagerPodLabel = map[string]string{"component": "kube-controller-manager"} +func newEgressClusterInfoController(mgr manager.Manager, log logr.Logger) error { + r := &eciReconciler{ + mgr: mgr, + eci: new(egressv1beta1.EgressClusterInfo), + client: mgr.GetClient(), + log: log, + doOnce: sync.Once{}, + k8sPodCidr: make(map[string]egressv1beta1.IPListPair), + v4ClusterCidr: make([]string, 0), + v6ClusterCidr: make([]string, 0), + } + + log.Info("new egressClusterInfo controller") + c, err := controller.New("egressClusterInfo", mgr, + controller.Options{Reconciler: r}) + if err != nil { + return err + } + r.c = c + + log.Info("egressClusterInfo controller watch EgressClusterInfo") + return watchSource(c, source.Kind(mgr.GetCache(), &egressv1beta1.EgressClusterInfo{}), kindEGCI) +} + func (r *eciReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { kind, newReq, err := utils.ParseKindWithReq(req) if err != nil { @@ -61,174 +94,123 @@ func (r *eciReconciler) Reconcile(ctx context.Context, req reconcile.Request) (r err := r.initEgressClusterInfo(ctx) if err != nil { r.log.Error(err, "first reconcile of egressClusterInfo controller, init egressClusterInfo") + time.Sleep(time.Second) goto redo } }) switch kind { - case "Node": + case kindNode: return r.reconcileNode(ctx, newReq, log) - case "IPPool": + case kindCalicoIPPool: return r.reconcileCalicoIPPool(ctx, newReq, log) + case kindEGCI: + return r.reconcileEgressClusterInfo(ctx, newReq, log) default: return reconcile.Result{}, nil } } -// reconcileCalicoIPPool reconcile calico IPPool -func (r *eciReconciler) reconcileCalicoIPPool(ctx context.Context, req reconcile.Request, log logr.Logger) (reconcile.Result, error) { +// reconcileEgressClusterInfo reconcile cr egressClusterInfo +func (r *eciReconciler) reconcileEgressClusterInfo(ctx context.Context, req reconcile.Request, log logr.Logger) (reconcile.Result, error) { log = log.WithValues("name", req.Name, "namespace", req.Namespace) log.Info("reconciling") - - // eci + // get egressClusterInfo err := r.getEgressClusterInfo(ctx) if err != nil { return reconcile.Result{Requeue: true}, err } - deleted := false - ippool := new(calicov1.IPPool) - err = r.client.Get(ctx, req.NamespacedName, ippool) - if err != nil { - if !errors.IsNotFound(err) { - log.V(2).Error(err, "failed to get calico ippool") - return reconcile.Result{Requeue: true}, err - } - deleted = true - } - deleted = deleted || !ippool.GetDeletionTimestamp().IsZero() + egciStatus := r.eci.Status.DeepCopy() - // delete event - if deleted { - log.Info("delete event", - "calicoV4IPPoolMap", r.calicoV4IPPoolMap, - "calicoV6IPPoolMap", r.calicoV6IPPoolMap) - // check calicoV4IPPoolMap and calicoV6IPPoolMap - cidr, v4ok := r.calicoV4IPPoolMap[req.Name] - if v4ok { - // need to delete cidr from calicoV4IPPoolMap - log.V(1).Info("remove IPPool from calicoV4IPPoolMap") - delete(r.calicoV4IPPoolMap, req.Name) - // update eci status - cidrs := r.getCalicoV4IPPoolsCidrs() - r.eci.Status.EgressIgnoreCIDR.PodCIDR.IPv4 = cidrs - log.V(1).Info("update EgressClusterInfo", "context", cidrs) - err := r.updateEgressClusterInfo(ctx) - if err != nil { - log.Error(err, "failed to update EgressClusterInfo") - r.calicoV4IPPoolMap[req.Name] = cidr - return reconcile.Result{Requeue: true}, err - } + // ignore nodeIP + if r.eci.Spec.AutoDetect.NodeIP { + // need watch node + if err = watchSource(r.c, source.Kind(r.mgr.GetCache(), &corev1.Node{}), kindNode); err != nil { + return reconcile.Result{Requeue: true}, err } - - cidr, v6ok := r.calicoV6IPPoolMap[req.Name] - if v6ok { - // need to delete cidr from calicoV6IPPoolMap - log.V(1).Info("remove IPPool from calicoV6IPPoolMap") - delete(r.calicoV6IPPoolMap, req.Name) - // update eci status - cidrs := r.getCalicoV6IPPoolsCidrs() - r.eci.Status.EgressIgnoreCIDR.PodCIDR.IPv6 = cidrs - log.V(1).Info("update egress cluster info", "context", r.eci) - err := r.updateEgressClusterInfo(ctx) - if err != nil { - r.calicoV6IPPoolMap[req.Name] = cidr - return reconcile.Result{Requeue: true}, err - } + // need to list all node + nodesIP, err := r.listNodeIPs(ctx) + if err != nil { + return reconcile.Result{Requeue: true}, err } - // need not update calicoIPPoolMap - return reconcile.Result{}, nil + r.eci.Status.NodeIP = nodesIP } - - // not delete event - log.Info("update event") - - // check if cidr about ippools changed - isv4Cidr, err := utils.IsIPv4Cidr(ippool.Spec.CIDR) - if err != nil { - return reconcile.Result{Requeue: true}, err - } - isv6Cidr, err := utils.IsIPv6Cidr(ippool.Spec.CIDR) - if err != nil { - return reconcile.Result{Requeue: true}, err + if !r.eci.Spec.AutoDetect.NodeIP { + r.eci.Status.NodeIP = nil } - cidr, ok := r.calicoV4IPPoolMap[req.Name] - if ok { - // v4PoolName but v6Cidr, delete it from calicoV4IPPoolMap - if isv6Cidr { - // update calicoV4IPPoolMap - delete(r.calicoV4IPPoolMap, req.Name) - r.eci.Status.EgressIgnoreCIDR.PodCIDR.IPv4 = r.getCalicoV4IPPoolsCidrs() - err := r.updateEgressClusterInfo(ctx) - if err != nil { - r.calicoV4IPPoolMap[req.Name] = cidr + // ignore podCidr + switch r.eci.Spec.AutoDetect.PodCidrMode { + case egressv1beta1.CniTypeCalico: + if !r.ignoreCalico { + // need watch calico + if err = watchSource(r.c, source.Kind(r.mgr.GetCache(), &calicov1.IPPool{}), kindCalicoIPPool); err != nil { return reconcile.Result{Requeue: true}, err } - } else if ippool.Spec.CIDR != cidr { - // need to update calicoV4IPPoolMap - r.calicoV4IPPoolMap[req.Name] = ippool.Spec.CIDR - r.eci.Status.EgressIgnoreCIDR.PodCIDR.IPv4 = r.getCalicoV4IPPoolsCidrs() - err := r.updateEgressClusterInfo(ctx) + r.ignoreCalico = true + // need to list ippools + pools, err := r.listCalicoIPPools(ctx) if err != nil { - r.calicoV4IPPoolMap[req.Name] = cidr return reconcile.Result{Requeue: true}, err } + r.eci.Status.PodCIDR = pools } - } else { - if isv4Cidr { - // need to update calicoV4IPPoolMap - r.calicoV4IPPoolMap[req.Name] = ippool.Spec.CIDR - r.eci.Status.EgressIgnoreCIDR.PodCIDR.IPv4 = r.getCalicoV4IPPoolsCidrs() - err := r.updateEgressClusterInfo(ctx) + case egressv1beta1.CniTypeK8s: + r.ignoreCalico = false + if _, ok := r.k8sPodCidr[k8s]; !ok { + cidr, err := r.getK8sPodCidr() if err != nil { - delete(r.calicoV4IPPoolMap, req.Name) return reconcile.Result{Requeue: true}, err } + r.k8sPodCidr = cidr } + r.eci.Status.PodCIDR = r.k8sPodCidr + default: } - cidr, ok = r.calicoV6IPPoolMap[req.Name] - if ok { - // v6PoolName but v4Cidr, delete it from calicoV6IPPoolMap - if isv4Cidr { - // update calicoV6IPPoolMap - delete(r.calicoV6IPPoolMap, req.Name) - r.eci.Status.EgressIgnoreCIDR.PodCIDR.IPv6 = r.getCalicoV6IPPoolsCidrs() - err := r.updateEgressClusterInfo(ctx) + // ignore clusterIP + if r.eci.Spec.AutoDetect.ClusterIP { + if len(r.v4ClusterCidr) == 0 { + v4Cidr, v6Cidr, err := r.getServiceClusterIPRange() if err != nil { - r.calicoV6IPPoolMap[req.Name] = cidr - return reconcile.Result{Requeue: true}, err - } - } else if ippool.Spec.CIDR != cidr { - // need to update calicoV6IPPoolMap - r.calicoV6IPPoolMap[req.Name] = ippool.Spec.CIDR - r.eci.Status.EgressIgnoreCIDR.PodCIDR.IPv6 = r.getCalicoV6IPPoolsCidrs() - err := r.updateEgressClusterInfo(ctx) - if err != nil { - r.calicoV6IPPoolMap[req.Name] = cidr return reconcile.Result{Requeue: true}, err } + r.v4ClusterCidr = v4Cidr + r.v6ClusterCidr = v6Cidr } + r.eci.Status.ClusterIP.IPv4 = r.v4ClusterCidr + r.eci.Status.ClusterIP.IPv6 = r.v6ClusterCidr + } + if !r.eci.Spec.AutoDetect.ClusterIP { + r.eci.Status.ClusterIP.IPv4 = nil + r.eci.Status.ClusterIP.IPv6 = nil + } + + // extraCidr + if r.eci.Spec.ExtraCidr != nil { + r.eci.Status.ExtraCidr = r.eci.Spec.ExtraCidr } else { - if isv6Cidr { - // need to update calicoV6IPPoolMap - r.calicoV6IPPoolMap[req.Name] = ippool.Spec.CIDR - r.eci.Status.EgressIgnoreCIDR.PodCIDR.IPv6 = r.getCalicoV6IPPoolsCidrs() - err := r.updateEgressClusterInfo(ctx) - if err != nil { - delete(r.calicoV6IPPoolMap, req.Name) - return reconcile.Result{Requeue: true}, err - } + r.eci.Status.ExtraCidr = nil + } + + r.log.V(1).Info("deepEqual", "egciStatus", egciStatus) + r.log.V(1).Info("deepEqual", "r.eci.Status", r.eci.Status) + + if !reflect.DeepEqual(egciStatus, r.eci.Status) { + // need to update + err = r.patchEgressClusterInfo(ctx) + if err != nil { + return reconcile.Result{Requeue: true}, err } } return reconcile.Result{}, nil } -// reconcileNode reconcile node -func (r *eciReconciler) reconcileNode(ctx context.Context, req reconcile.Request, log logr.Logger) (reconcile.Result, error) { - log = log.WithValues("name", req.Name) +// reconcileCalicoIPPool reconcile calico IPPool +func (r *eciReconciler) reconcileCalicoIPPool(ctx context.Context, req reconcile.Request, log logr.Logger) (reconcile.Result, error) { + log = log.WithValues("name", req.Name, "namespace", req.Namespace) log.Info("reconciling") // eci @@ -237,214 +219,274 @@ func (r *eciReconciler) reconcileNode(ctx context.Context, req reconcile.Request return reconcile.Result{Requeue: true}, err } + egciStatus := r.eci.Status.DeepCopy() + deleted := false - node := new(corev1.Node) - err = r.client.Get(ctx, req.NamespacedName, node) + ippool := new(calicov1.IPPool) + err = r.client.Get(ctx, req.NamespacedName, ippool) if err != nil { if !errors.IsNotFound(err) { return reconcile.Result{Requeue: true}, err } deleted = true } - deleted = deleted || !node.GetDeletionTimestamp().IsZero() + deleted = deleted || !ippool.GetDeletionTimestamp().IsZero() // delete event if deleted { - log.Info("delete event") - // check map - nodeIPv4, v4Ok := r.nodeIPv4Map[req.Name] - nodeIPv6, v6Ok := r.nodeIPv6Map[req.Name] - if v4Ok { - // update map - delete(r.nodeIPv4Map, req.Name) - // update eci - r.eci.Status.EgressIgnoreCIDR.NodeIP.IPv4 = r.getNodesIPs(r.nodeIPv4Map) - } - if v6Ok { - // update map - delete(r.nodeIPv6Map, req.Name) - // update eci - r.eci.Status.EgressIgnoreCIDR.NodeIP.IPv6 = r.getNodesIPs(r.nodeIPv6Map) - } + log.Info("delete event of calico ippool", "delete", req.Name) - // eci need to update - if v4Ok && v6Ok { - err := r.updateEgressClusterInfo(ctx) - if err != nil { - r.nodeIPv4Map[req.Name] = nodeIPv4 - r.nodeIPv6Map[req.Name] = nodeIPv6 - return reconcile.Result{Requeue: true}, err - } + delete(r.eci.Status.PodCIDR, req.Name) + } else { + // not delete event + log.Info("update event of calico ippool", "update", req.Name) + poolsMap, err := r.getCalicoIPPools(ctx, req.Name) + if err != nil { + return reconcile.Result{Requeue: true}, err } - if v4Ok && !v6Ok { - err := r.updateEgressClusterInfo(ctx) - if err != nil { - r.nodeIPv4Map[req.Name] = nodeIPv4 - return reconcile.Result{Requeue: true}, err - } + if r.eci.Status.PodCIDR == nil { + r.eci.Status.PodCIDR = make(map[string]egressv1beta1.IPListPair) } - if !v4Ok && v6Ok { - err := r.updateEgressClusterInfo(ctx) - if err != nil { - r.nodeIPv6Map[req.Name] = nodeIPv6 - return reconcile.Result{Requeue: true}, err - } - } - return reconcile.Result{}, nil + r.eci.Status.PodCIDR[req.Name] = poolsMap[req.Name] } - // not delete event - log.Info("update event") - - // get nodeIP, check if its changed - nodeIPv4, nodeIPv6 := utils.GetNodeIP(node) + r.log.V(1).Info("now egciStatus.PodCIDR", "podCIDR", egciStatus.PodCIDR) + r.log.V(1).Info("need update egciStatus.PodCIDR", "podCIDR", r.eci.Status.PodCIDR) - _, v4Ok := r.nodeIPv4Map[req.Name] - _, v6Ok := r.nodeIPv6Map[req.Name] + if !reflect.DeepEqual(egciStatus, r.eci.Status) { + // need to update + if err = r.patchEgressClusterInfo(ctx); err != nil { + return reconcile.Result{Requeue: true}, nil + } + } - needUpdateECI := false - if (!v4Ok || r.nodeIPv4Map[req.Name] != nodeIPv4) && len(nodeIPv4) != 0 { - needUpdateECI = true - // update map - r.nodeIPv4Map[req.Name] = nodeIPv4 + return reconcile.Result{}, nil +} - // need to update node ip from eci status - r.eci.Status.EgressIgnoreCIDR.NodeIP.IPv4 = r.getNodesIPs(r.nodeIPv4Map) +// reconcileNode reconcile node +func (r *eciReconciler) reconcileNode(ctx context.Context, req reconcile.Request, log logr.Logger) (reconcile.Result, error) { + log = log.WithValues("name", req.Name) + log.Info("reconciling") + // eci + err := r.getEgressClusterInfo(ctx) + if err != nil { + return reconcile.Result{Requeue: true}, err } - if (!v6Ok || r.nodeIPv6Map[req.Name] != nodeIPv6) && len(nodeIPv6) != 0 { - needUpdateECI = true - // update map - r.nodeIPv6Map[req.Name] = nodeIPv6 + egciStatus := r.eci.Status.DeepCopy() - // need to update node ip from eci status - r.eci.Status.EgressIgnoreCIDR.NodeIP.IPv6 = r.getNodesIPs(r.nodeIPv6Map) + deleted := false + node := new(corev1.Node) + err = r.client.Get(ctx, req.NamespacedName, node) + if err != nil { + if !errors.IsNotFound(err) { + return reconcile.Result{Requeue: true}, err + } + deleted = true } + deleted = deleted || !node.GetDeletionTimestamp().IsZero() - if needUpdateECI { - err = r.updateEgressClusterInfo(ctx) + // delete event + if deleted { + log.Info("delete event of node", "delete", req.Name) + delete(r.eci.Status.NodeIP, req.Name) + } else { + // not delete event + log.Info("update event of node", "update", req.Name) + nodeIPMap, err := r.getNodeIPs(ctx, req.Name) if err != nil { - delete(r.nodeIPv4Map, req.Name) - delete(r.nodeIPv6Map, req.Name) - return reconcile.Result{Requeue: true}, err } + if r.eci.Status.NodeIP == nil { + r.eci.Status.NodeIP = make(map[string]egressv1beta1.IPListPair) + } + r.eci.Status.NodeIP[req.Name] = nodeIPMap[req.Name] } - return reconcile.Result{}, nil -} + r.log.V(1).Info("now egciStatus.NodeIP", "nodeIP", egciStatus.NodeIP) + r.log.V(1).Info("need to update egciStatus.NodeIP", "nodeIP", r.eci.Status.NodeIP) -func newEgressClusterInfoController(mgr manager.Manager, log logr.Logger, cfg *config.Config) error { - if cfg == nil { - return fmt.Errorf("cfg can not be nil") + if !reflect.DeepEqual(egciStatus, r.eci.Status) { + // need to update + if err = r.patchEgressClusterInfo(ctx); err != nil { + return reconcile.Result{Requeue: true}, nil + } } - r := &eciReconciler{ - eci: new(egressv1beta1.EgressClusterInfo), - client: mgr.GetClient(), - log: log, - config: cfg, - doOnce: sync.Once{}, - nodeIPv4Map: make(map[string]string), - nodeIPv6Map: make(map[string]string), - calicoV4IPPoolMap: make(map[string]string), - calicoV6IPPoolMap: make(map[string]string), - } + return reconcile.Result{}, nil +} - log.Info("new egressClusterInfo controller") - c, err := controller.New("egressClusterInfo", mgr, - controller.Options{Reconciler: r}) +// initEgressClusterInfo create EgressClusterInfo cr when first reconcile +func (r *eciReconciler) initEgressClusterInfo(ctx context.Context) error { + r.log.Info("start init EgressClusterInfo", "name", defaultEgressClusterInfoName) + err := r.getEgressClusterInfo(ctx) if err != nil { - return err + if !errors.IsNotFound(err) { + return err + } + err = r.createEgressClusterInfo(ctx) + if err != nil { + return err + } } - podCidr, _, ignoreNodeIP := r.getEgressIgnoreCIDRConfig() + egci := r.eci.DeepCopy() + + ignoreClusterIP := r.eci.Spec.AutoDetect.ClusterIP + ignoreNodeIP := r.eci.Spec.AutoDetect.NodeIP + cniType := r.eci.Spec.AutoDetect.PodCidrMode + + if ignoreClusterIP { + // get service-cluster-ip-range + ipv4Range, ipv6Range, err := r.getServiceClusterIPRange() + if err != nil { + return err + } + r.v4ClusterCidr = ipv4Range + r.v6ClusterCidr = ipv6Range + r.eci.Status.ClusterIP.IPv4 = ipv4Range + r.eci.Status.ClusterIP.IPv6 = ipv6Range + } if ignoreNodeIP { - log.Info("egressClusterInfo controller watch Node") - if err := watchSource(c, source.Kind(mgr.GetCache(), &corev1.Node{}), "Node"); err != nil { + nodesIP, err := r.listNodeIPs(ctx) + if err != nil { return err } + r.eci.Status.NodeIP = nodesIP } - switch podCidr { - case calico: - log.Info("egressClusterInfo controller watch calico") - if err := watchSource(c, source.Kind(mgr.GetCache(), &calicov1.IPPool{}), "IPPool"); err != nil { + switch cniType { + case egressv1beta1.CniTypeK8s: + // get cluster-cidr + k8sCidr, err := r.getK8sPodCidr() + if err != nil { + return err + } + r.k8sPodCidr = k8sCidr + r.eci.Status.PodCIDR = k8sCidr + case egressv1beta1.CniTypeCalico: + // get calico ippool + pools, err := r.listCalicoIPPools(ctx) + if err != nil { return err } + r.eci.Status.PodCIDR = pools default: + err = fmt.Errorf("invalid cniTyp") + return err } - return nil -} - -// initEgressClusterInfo create EgressClusterInfo cr when first reconcile -func (r *eciReconciler) initEgressClusterInfo(ctx context.Context) error { - r.log.Info("start init EgressClusterInfo", "name", defaultEgressClusterInfoName) - err := r.getOrCreateEgressClusterInfo(ctx) - if err != nil { - return err + if r.eci.Spec.ExtraCidr != nil { + r.eci.Status.ExtraCidr = r.eci.Spec.ExtraCidr } - ignorePod, ignoreClusterCidr, _ := r.getEgressIgnoreCIDRConfig() - if !ignoreClusterCidr && (ignorePod == k8s || ignorePod == "") { - return nil + r.log.V(1).Info("now egci.Status", "Status", egci.Status) + r.log.V(1).Info("need to update egci.Status", "Status", r.eci.Status) + + if !reflect.DeepEqual(egci.Status, r.eci.Status) { + r.log.Info("first init egressClusterInfo, need update") + return r.client.Status().Update(ctx, r.eci) } + return nil +} - // get service-cluster-ip-range from kube-controller-manager pod - pod, err := getPod(r.client, kubeControllerManagerPodLabel) +// listCalicoIPPools list all calico ippools +func (r *eciReconciler) listCalicoIPPools(ctx context.Context) (map[string]egressv1beta1.IPListPair, error) { + ippoolList := new(calicov1.IPPoolList) + calicoIPPoolMap := make(map[string]egressv1beta1.IPListPair, 0) + + err := r.client.List(ctx, ippoolList) if err != nil { - return err + return nil, err } + for _, item := range ippoolList.Items { + cidr := item.Spec.CIDR + ipListPair := egressv1beta1.IPListPair{} - if ignoreClusterCidr { - // get service-cluster-ip-range - ipv4Range, ipv6Range, err := r.getServiceClusterIPRange(pod) + isV4Cidr, err := utils.IsIPv4Cidr(cidr) if err != nil { - return err + return nil, err } - r.eci.Status.EgressIgnoreCIDR.ClusterIP.IPv4 = ipv4Range - r.eci.Status.EgressIgnoreCIDR.ClusterIP.IPv6 = ipv6Range - } + if isV4Cidr { - if ignorePod == k8s || ignorePod == "" { - // get cluster-cidr - ipv4Range, ipv6Range, err := r.getClusterCidr(pod) + ipListPair.IPv4 = append(ipListPair.IPv4, cidr) + calicoIPPoolMap[item.Name] = ipListPair + } + isV6Cidr, err := utils.IsIPv6Cidr(cidr) if err != nil { - return err + return nil, err + } + if isV6Cidr { + ipListPair.IPv6 = append(ipListPair.IPv6, cidr) + calicoIPPoolMap[item.Name] = ipListPair } - r.eci.Status.EgressIgnoreCIDR.PodCIDR.IPv4 = ipv4Range - r.eci.Status.EgressIgnoreCIDR.PodCIDR.IPv6 = ipv6Range } + return calicoIPPoolMap, nil +} + +// getCalicoIPPools get calico ippool by name +func (r *eciReconciler) getCalicoIPPools(ctx context.Context, poolName string) (map[string]egressv1beta1.IPListPair, error) { + ippool := new(calicov1.IPPool) + calicoIPPoolMap := make(map[string]egressv1beta1.IPListPair, 0) + + err := r.client.Get(ctx, types.NamespacedName{Name: poolName}, ippool) + if err != nil { + return nil, err + } + cidr := ippool.Spec.CIDR + ipListPair := egressv1beta1.IPListPair{} - r.log.V(1).Info("update EgressClusterInfo", "context", r.eci) - err = r.updateEgressClusterInfo(ctx) + isV4Cidr, err := utils.IsIPv4Cidr(cidr) if err != nil { - return err + return nil, err } - return nil + if isV4Cidr { + ipListPair.IPv4 = append(ipListPair.IPv4, cidr) + calicoIPPoolMap[ippool.Name] = ipListPair + } + isV6Cidr, err := utils.IsIPv6Cidr(cidr) + if err != nil { + return nil, err + } + if isV6Cidr { + ipListPair.IPv6 = append(ipListPair.IPv6, cidr) + calicoIPPoolMap[ippool.Name] = ipListPair + } + return calicoIPPoolMap, nil } -func watchSource(c controller.Controller, source source.Source, kind string) error { - if err := c.Watch(source, handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat(kind))); err != nil { - return fmt.Errorf("failed to watch %s: %w", kind, err) +// listNodeIPs list all node ips +func (r *eciReconciler) listNodeIPs(ctx context.Context) (map[string]egressv1beta1.IPListPair, error) { + nodeList := new(corev1.NodeList) + nodesIPMap := make(map[string]egressv1beta1.IPListPair, 0) + + err := r.client.List(ctx, nodeList) + if err != nil { + return nil, err } - return nil + + for _, item := range nodeList.Items { + nodeIPv4, nodeIPv6 := utils.GetNodeIP(&item) + nodesIPMap[item.Name] = egressv1beta1.IPListPair{IPv4: []string{nodeIPv4}, IPv6: []string{nodeIPv6}} + } + return nodesIPMap, nil } -// getOrCreateEgressClusterInfo get EgressClusterInfo, if not found create -func (r *eciReconciler) getOrCreateEgressClusterInfo(ctx context.Context) error { - err := r.getEgressClusterInfo(ctx) +// getNodeIPs get node ip by name +func (r *eciReconciler) getNodeIPs(ctx context.Context, nodeName string) (map[string]egressv1beta1.IPListPair, error) { + node := new(corev1.Node) + nodesIPMap := make(map[string]egressv1beta1.IPListPair, 0) + + err := r.client.Get(ctx, types.NamespacedName{Name: nodeName}, node) if err != nil { - err = r.createEgressClusterInfo(ctx) - if err != nil { - return err - } + return nil, err } - r.eci.Status = egressv1beta1.EgressClusterStatus{} - return nil + + nodeIPv4, nodeIPv6 := utils.GetNodeIP(node) + nodesIPMap[nodeName] = egressv1beta1.IPListPair{IPv4: []string{nodeIPv4}, IPv6: []string{nodeIPv6}} + return nodesIPMap, nil } // createEgressClusterInfo create EgressClusterInfo @@ -458,44 +500,33 @@ func (r *eciReconciler) createEgressClusterInfo(ctx context.Context) error { return nil } -// getCalicoV4IPPoolsCidrs get calico all ipv4 ippools cidrs from calicoV4IPPoolMap -func (r *eciReconciler) getCalicoV4IPPoolsCidrs() []string { - cidrs := make([]string, 0) - for _, cidr := range r.calicoV4IPPoolMap { - cidrs = append(cidrs, cidr) - } - return cidrs +// getEgressClusterInfo get EgressClusterInfo cr +func (r *eciReconciler) getEgressClusterInfo(ctx context.Context) error { + return r.client.Get(ctx, types.NamespacedName{Name: defaultEgressClusterInfoName}, r.eci) } -// getCalicoV6IPPoolsCidrs get calico all ipv6 ippools cidrs from calicoV6IPPoolMap -func (r *eciReconciler) getCalicoV6IPPoolsCidrs() []string { - cidrs := make([]string, 0) - for _, cidr := range r.calicoV6IPPoolMap { - cidrs = append(cidrs, cidr) +// getServiceClusterIPRange get service-cluster-ip-range from kube controller manager +func (r *eciReconciler) getServiceClusterIPRange() (ipv4Range, ipv6Range []string, err error) { + pod, err := getPodByLabel(r.client, kubeControllerManagerPodLabel) + if err != nil { + return nil, nil, err } - return cidrs + return parseCidrFromControllerManager(pod, serviceClusterIpRange) } -// getNodesIPs get all node IP from nodeMap -func (r *eciReconciler) getNodesIPs(nodeMap map[string]string) []string { - nodeIPs := make([]string, 0) - nodeIPMap := make(map[string]struct{}) - for _, v := range nodeMap { - nodeIPMap[v] = struct{}{} - } - for ip := range nodeIPMap { - nodeIPs = append(nodeIPs, ip) +// getK8sPodCidr get k8s default podCidr +func (r *eciReconciler) getK8sPodCidr() (map[string]egressv1beta1.IPListPair, error) { + v4Cidr, v6Cidr, err := getClusterCidr(r.client) + if err != nil { + return nil, err } - return nodeIPs -} - -// getEgressClusterInfo get EgressClusterInfo cr -func (r *eciReconciler) getEgressClusterInfo(ctx context.Context) error { - return r.client.Get(ctx, types.NamespacedName{Name: defaultEgressClusterInfoName}, r.eci) + k8sPodCIDR := make(map[string]egressv1beta1.IPListPair) + k8sPodCIDR[k8s] = egressv1beta1.IPListPair{IPv4: v4Cidr, IPv6: v6Cidr} + return k8sPodCIDR, nil } -// updateEgressClusterInfo update EgressClusterInfo cr -func (r *eciReconciler) updateEgressClusterInfo(ctx context.Context) error { +// patchEgressClusterInfo patch EgressClusterInfo cr +func (r *eciReconciler) patchEgressClusterInfo(ctx context.Context) error { eci := new(egressv1beta1.EgressClusterInfo) err := r.client.Get(ctx, types.NamespacedName{Name: defaultEgressClusterInfoName}, eci) if err != nil { @@ -508,27 +539,33 @@ func (r *eciReconciler) updateEgressClusterInfo(ctx context.Context) error { return err } r.log.V(1).Info("update", "patch.Data", string(data)) - return r.client.Status().Patch(ctx, r.eci, patch) -} - -// getEgressIgnoreCIDRConfig get config about EgressIgnoreCIDR from egressgateway configmap -func (r *eciReconciler) getEgressIgnoreCIDRConfig() (string, bool, bool) { - i := r.config.FileConfig.EgressIgnoreCIDR - return i.PodCIDR, i.ClusterIP, i.NodeIP -} - -// getServiceClusterIPRange get service-cluster-ip-range from kube controller manager -func (r *eciReconciler) getServiceClusterIPRange(pod *corev1.Pod) (ipv4Range, ipv6Range []string, err error) { - return getCidr(pod, serviceClusterIpRange) -} - -// getClusterCidr get cluster-cidr from kube controller manager -func (r *eciReconciler) getClusterCidr(pod *corev1.Pod) (ipv4Range, ipv6Range []string, err error) { - return getCidr(pod, clusterCidr) + err = r.client.Status().Patch(ctx, r.eci, patch) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.TODO(), time.Second*3) + defer cancel() + for { + select { + case <-ctx.Done(): + return fmt.Errorf("timeout to wait egci update") + default: + err = r.client.Get(ctx, types.NamespacedName{Name: defaultEgressClusterInfoName}, eci) + if err != nil { + return err + } + r.log.V(1).Info("status", "eci.Status", eci.Status) + r.log.V(1).Info("status", "r.eci.Status", r.eci.Status) + if reflect.DeepEqual(eci.Status, r.eci.Status) { + return nil + } + time.Sleep(time.Millisecond * 100) + } + } } -// getCidr get cidr value from kube controller manager -func getCidr(pod *corev1.Pod, param string) (ipv4Range, ipv6Range []string, err error) { +// parseCidrFromControllerManager get cidr value from kube controller manager +func parseCidrFromControllerManager(pod *corev1.Pod, param string) (ipv4Range, ipv6Range []string, err error) { containers := pod.Spec.Containers if len(containers) == 0 { return nil, nil, fmt.Errorf("failed to found containers") @@ -563,8 +600,8 @@ func getCidr(pod *corev1.Pod, param string) (ipv4Range, ipv6Range []string, err return } -// getPod get pod by label -func getPod(c client.Client, label map[string]string) (*corev1.Pod, error) { +// getPodByLabel get pod by label +func getPodByLabel(c client.Client, label map[string]string) (*corev1.Pod, error) { podList := corev1.PodList{} opts := client.MatchingLabels(label) err := c.List(context.Background(), &podList, opts) @@ -577,3 +614,20 @@ func getPod(c client.Client, label map[string]string) (*corev1.Pod, error) { } return &pods[0], nil } + +// getClusterCidr get k8s default podCidr +func getClusterCidr(c client.Client) (ipv4Range, ipv6Range []string, err error) { + pod, err := getPodByLabel(c, kubeControllerManagerPodLabel) + if err != nil { + return nil, nil, err + } + return parseCidrFromControllerManager(pod, clusterCidr) +} + +// watchSource controller watch given resource +func watchSource(c controller.Controller, source source.Source, kind string) error { + if err := c.Watch(source, handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat(kind))); err != nil { + return fmt.Errorf("failed to watch %s: %w", kind, err) + } + return nil +} diff --git a/pkg/k8s/apis/v1beta1/egressclusterinfo_types.go b/pkg/k8s/apis/v1beta1/egressclusterinfo_types.go index eac1421ac..3192ba7c6 100644 --- a/pkg/k8s/apis/v1beta1/egressclusterinfo_types.go +++ b/pkg/k8s/apis/v1beta1/egressclusterinfo_types.go @@ -5,7 +5,7 @@ package v1beta1 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// EgressClusterInfoList contains a list of EgressClusterStatus +// EgressClusterInfoList contains a list of EgressClusterInfo // +kubebuilder:object:root=true type EgressClusterInfoList struct { metav1.TypeMeta `json:",inline"` @@ -24,25 +24,39 @@ type EgressClusterInfo struct { metav1.ObjectMeta `json:"metadata"` // +kubebuilder:validation:Optional - Spec EgressClusterStatusSpec `json:"spec,omitempty"` + Spec EgressClusterInfoSpec `json:"spec,omitempty"` // +kubebuilder:validation:Optional - Status EgressClusterStatus `json:"status,omitempty"` + Status EgressClusterInfoStatus `json:"status,omitempty"` } -type EgressClusterStatusSpec struct{} - -type EgressClusterStatus struct { +type EgressClusterInfoSpec struct { + // +kubebuilder:validation:Optional + AutoDetect AutoDetect `json:"autoDetect,omitempty"` // +kubebuilder:validation:Optional - EgressIgnoreCIDR EgressIgnoreCIDR `json:"egressIgnoreCIDR,omitempty"` + ExtraCidr []string `json:"extraCidr,omitempty"` } -type EgressIgnoreCIDR struct { +type EgressClusterInfoStatus struct { // +kubebuilder:validation:Optional - NodeIP IPListPair `json:"nodeIP,omitempty"` + NodeIP map[string]IPListPair `json:"nodeIP,omitempty"` // +kubebuilder:validation:Optional ClusterIP IPListPair `json:"clusterIP,omitempty"` // +kubebuilder:validation:Optional - PodCIDR IPListPair `json:"podCIDR,omitempty"` + PodCIDR map[string]IPListPair `json:"podCIDR,omitempty"` + // +kubebuilder:validation:Optional + ExtraCidr []string `json:"extraCidr,omitempty"` +} + +type AutoDetect struct { + // +kubebuilder:validation:Optional + // +kubebuilder:default:="k8s" + PodCidrMode PodCidrMode `json:"podCidrMode,omitempty"` + // +kubebuilder:validation:Optional + // +kubebuilder:default:=true + ClusterIP bool `json:"clusterIP,omitempty"` + // +kubebuilder:validation:Optional + // +kubebuilder:default:=true + NodeIP bool `json:"nodeIP,omitempty"` } type IPListPair struct { @@ -52,6 +66,13 @@ type IPListPair struct { IPv6 []string `json:"ipv6,omitempty"` } +type PodCidrMode string + +const ( + CniTypeK8s PodCidrMode = "k8s" + CniTypeCalico PodCidrMode = "calico" +) + func init() { SchemeBuilder.Register(&EgressClusterInfo{}, &EgressClusterInfoList{}) } diff --git a/pkg/k8s/apis/v1beta1/zz_generated.deepcopy.go b/pkg/k8s/apis/v1beta1/zz_generated.deepcopy.go index 8b5439aaa..063a919d9 100644 --- a/pkg/k8s/apis/v1beta1/zz_generated.deepcopy.go +++ b/pkg/k8s/apis/v1beta1/zz_generated.deepcopy.go @@ -35,6 +35,21 @@ func (in *AppliedTo) DeepCopy() *AppliedTo { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AutoDetect) DeepCopyInto(out *AutoDetect) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutoDetect. +func (in *AutoDetect) DeepCopy() *AutoDetect { + if in == nil { + return nil + } + out := new(AutoDetect) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterAppliedTo) DeepCopyInto(out *ClusterAppliedTo) { *out = *in @@ -138,7 +153,7 @@ func (in *EgressClusterInfo) DeepCopyInto(out *EgressClusterInfo) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } @@ -192,6 +207,62 @@ func (in *EgressClusterInfoList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressClusterInfoSpec) DeepCopyInto(out *EgressClusterInfoSpec) { + *out = *in + out.AutoDetect = in.AutoDetect + if in.ExtraCidr != nil { + in, out := &in.ExtraCidr, &out.ExtraCidr + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressClusterInfoSpec. +func (in *EgressClusterInfoSpec) DeepCopy() *EgressClusterInfoSpec { + if in == nil { + return nil + } + out := new(EgressClusterInfoSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EgressClusterInfoStatus) DeepCopyInto(out *EgressClusterInfoStatus) { + *out = *in + if in.NodeIP != nil { + in, out := &in.NodeIP, &out.NodeIP + *out = make(map[string]IPListPair, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } + in.ClusterIP.DeepCopyInto(&out.ClusterIP) + if in.PodCIDR != nil { + in, out := &in.PodCIDR, &out.PodCIDR + *out = make(map[string]IPListPair, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } + if in.ExtraCidr != nil { + in, out := &in.ExtraCidr, &out.ExtraCidr + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressClusterInfoStatus. +func (in *EgressClusterInfoStatus) DeepCopy() *EgressClusterInfoStatus { + if in == nil { + return nil + } + out := new(EgressClusterInfoStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EgressClusterPolicy) DeepCopyInto(out *EgressClusterPolicy) { *out = *in @@ -273,37 +344,6 @@ func (in *EgressClusterPolicySpec) DeepCopy() *EgressClusterPolicySpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *EgressClusterStatus) DeepCopyInto(out *EgressClusterStatus) { - *out = *in - in.EgressIgnoreCIDR.DeepCopyInto(&out.EgressIgnoreCIDR) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressClusterStatus. -func (in *EgressClusterStatus) DeepCopy() *EgressClusterStatus { - if in == nil { - return nil - } - out := new(EgressClusterStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *EgressClusterStatusSpec) DeepCopyInto(out *EgressClusterStatusSpec) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressClusterStatusSpec. -func (in *EgressClusterStatusSpec) DeepCopy() *EgressClusterStatusSpec { - if in == nil { - return nil - } - out := new(EgressClusterStatusSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EgressEndpoint) DeepCopyInto(out *EgressEndpoint) { *out = *in @@ -528,24 +568,6 @@ func (in *EgressIPStatus) DeepCopy() *EgressIPStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *EgressIgnoreCIDR) DeepCopyInto(out *EgressIgnoreCIDR) { - *out = *in - in.NodeIP.DeepCopyInto(&out.NodeIP) - in.ClusterIP.DeepCopyInto(&out.ClusterIP) - in.PodCIDR.DeepCopyInto(&out.PodCIDR) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EgressIgnoreCIDR. -func (in *EgressIgnoreCIDR) DeepCopy() *EgressIgnoreCIDR { - if in == nil { - return nil - } - out := new(EgressIgnoreCIDR) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EgressNodeSpec) DeepCopyInto(out *EgressNodeSpec) { *out = *in diff --git a/test/e2e/common/egressconfigmap.go b/test/e2e/common/egressconfigmap.go index 934034306..80dd75012 100644 --- a/test/e2e/common/egressconfigmap.go +++ b/test/e2e/common/egressconfigmap.go @@ -50,10 +50,10 @@ func GetIPVersion(f *framework.Framework) (enableV4, enableV6 bool, e error) { return c.EnableIPv4, c.EnableIPv6, nil } -func GetEgressIgnoreCIDR(f *framework.Framework) (*config.EgressIgnoreCIDR, error) { - c, e := GetEgressConfigmap(f) - if e != nil { - return nil, e - } - return &c.EgressIgnoreCIDR, nil -} +//func GetEgressIgnoreCIDR(f *framework.Framework) (*config.EgressIgnoreCIDR, error) { +// c, e := GetEgressConfigmap(f) +// if e != nil { +// return nil, e +// } +// return &c.EgressIgnoreCIDR, nil +//}