diff --git a/pkg/agent/eip.go b/pkg/agent/eip.go index 5c1fb47b8..047d74396 100644 --- a/pkg/agent/eip.go +++ b/pkg/agent/eip.go @@ -16,7 +16,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" "github.com/spidernet-io/egressgateway/pkg/config" egressv1 "github.com/spidernet-io/egressgateway/pkg/k8s/apis/v1beta1" @@ -149,14 +148,23 @@ func newEipCtrl(mgr manager.Manager, log logr.Logger, cfg *config.Config) error return err } - if err := c.Watch(source.Kind(mgr.GetCache(), &egressv1.EgressPolicy{}), - handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressPolicy"))); err != nil { + sourceEgressPolicy := utils.SourceKind( + mgr.GetCache(), + &egressv1.EgressPolicy{}, + handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressPolicy")), + ) + if err := c.Watch(sourceEgressPolicy); err != nil { return fmt.Errorf("failed to watch EgressPolicy: %w", err) } - if err := c.Watch(source.Kind(mgr.GetCache(), &egressv1.EgressClusterPolicy{}), - handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressClusterPolicy"))); err != nil { + sourceEgressPolicy = utils.SourceKind( + mgr.GetCache(), + &egressv1.EgressClusterPolicy{}, + handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressClusterPolicy")), + ) + if err := c.Watch(sourceEgressPolicy); err != nil { return fmt.Errorf("failed to watch EgressClusterPolicy: %w", err) + } return nil diff --git a/pkg/agent/police.go b/pkg/agent/police.go index c6e0cf5e9..7f5c73d64 100644 --- a/pkg/agent/police.go +++ b/pkg/agent/police.go @@ -16,11 +16,6 @@ import ( "time" "github.com/go-logr/logr" - "github.com/spidernet-io/egressgateway/pkg/config" - "github.com/spidernet-io/egressgateway/pkg/ipset" - "github.com/spidernet-io/egressgateway/pkg/iptables" - egressv1 "github.com/spidernet-io/egressgateway/pkg/k8s/apis/v1beta1" - "github.com/spidernet-io/egressgateway/pkg/utils" apierr "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -32,7 +27,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" + + "github.com/spidernet-io/egressgateway/pkg/config" + "github.com/spidernet-io/egressgateway/pkg/ipset" + "github.com/spidernet-io/egressgateway/pkg/iptables" + egressv1 "github.com/spidernet-io/egressgateway/pkg/k8s/apis/v1beta1" + "github.com/spidernet-io/egressgateway/pkg/utils" ) const ( @@ -1147,39 +1147,49 @@ func newPolicyController(mgr manager.Manager, log logr.Logger, cfg *config.Confi return err } - if err := c.Watch(source.Kind(mgr.GetCache(), &egressv1.EgressGateway{}), - handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressGateway"))); err != nil { + sourceEgressGateway := utils.SourceKind(mgr.GetCache(), + &egressv1.EgressGateway{}, + handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressGateway"))) + if err := c.Watch(sourceEgressGateway); err != nil { return fmt.Errorf("failed to watch EgressGateway: %w", err) } - if err := c.Watch(source.Kind(mgr.GetCache(), &egressv1.EgressPolicy{}), - handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressPolicy")), policyPredicate{}); err != nil { + sourceEgressPolicy := utils.SourceKind(mgr.GetCache(), + &egressv1.EgressPolicy{}, + handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressPolicy")), + policyPredicate{}) + if err := c.Watch(sourceEgressPolicy); err != nil { return fmt.Errorf("failed to watch EgressPolicy: %w", err) } - if err := c.Watch(source.Kind(mgr.GetCache(), &egressv1.EgressClusterPolicy{}), - handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressClusterPolicy")), policyPredicate{}); err != nil { + sourceEgressClusterPolicy := utils.SourceKind(mgr.GetCache(), + &egressv1.EgressClusterPolicy{}, + handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressClusterPolicy")), + policyPredicate{}) + if err := c.Watch(sourceEgressClusterPolicy); err != nil { return fmt.Errorf("failed to watch EgressClusterPolicy: %w", err) } - if err := c.Watch( - source.Kind(mgr.GetCache(), &egressv1.EgressEndpointSlice{}), - handler.EnqueueRequestsFromMapFunc(enqueueEndpointSlice()), - epSlicePredicate{}, - ); err != nil { + sourceEgressEndpointSlice := utils.SourceKind(mgr.GetCache(), + &egressv1.EgressEndpointSlice{}, + handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressEndpointSlice")), + epSlicePredicate{}) + if err := c.Watch(sourceEgressEndpointSlice); err != nil { return fmt.Errorf("failed to watch EgressEndpointSlice: %w", err) } - if err := c.Watch( - source.Kind(mgr.GetCache(), &egressv1.EgressClusterEndpointSlice{}), - handler.EnqueueRequestsFromMapFunc(enqueueEndpointSlice()), - epSlicePredicate{}, - ); err != nil { + sourceEgressClusterEndpointSlice := utils.SourceKind(mgr.GetCache(), + &egressv1.EgressClusterEndpointSlice{}, + handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressClusterEndpointSlice")), + epSlicePredicate{}) + if err := c.Watch(sourceEgressClusterEndpointSlice); err != nil { return fmt.Errorf("failed to watch EgressClusterEndpointSlice: %w", err) } - if err := c.Watch(source.Kind(mgr.GetCache(), &egressv1.EgressClusterInfo{}), - handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressClusterInfo"))); err != nil { + sourceEgressClusterInfo := utils.SourceKind(mgr.GetCache(), + &egressv1.EgressClusterInfo{}, + handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressClusterInfo"))) + if err := c.Watch(sourceEgressClusterInfo); err != nil { return fmt.Errorf("failed to watch EgressClusterInfo: %w", err) } diff --git a/pkg/agent/vxlan.go b/pkg/agent/vxlan.go index bee8b6b7a..61dd1779d 100644 --- a/pkg/agent/vxlan.go +++ b/pkg/agent/vxlan.go @@ -25,7 +25,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" "github.com/spidernet-io/egressgateway/pkg/agent/route" "github.com/spidernet-io/egressgateway/pkg/agent/vxlan" @@ -918,31 +917,34 @@ func newEgressTunnelController(mgr manager.Manager, cfg *config.Config, log logr return err } - if err := c.Watch(source.Kind(mgr.GetCache(), &egressv1.EgressTunnel{}), + sourceEgressTunnel := utils.SourceKind(mgr.GetCache(), + &egressv1.EgressTunnel{}, handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressTunnel")), - egressTunnelPredicate{}, - ); err != nil { + egressTunnelPredicate{}) + if err := c.Watch(sourceEgressTunnel); err != nil { return fmt.Errorf("failed to watch EgressTunnel: %w", err) } - if err := c.Watch(source.Kind(mgr.GetCache(), &egressv1.EgressGateway{}), - handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressGateway"))); err != nil { + sourceEgressGateway := utils.SourceKind(mgr.GetCache(), + &egressv1.EgressGateway{}, + handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressGateway"))) + if err := c.Watch(sourceEgressGateway); err != nil { return fmt.Errorf("failed to watch EgressGateway: %w", err) } - if err := c.Watch( - source.Kind(mgr.GetCache(), &egressv1.EgressEndpointSlice{}), + sourceEgressEndpointSlice := utils.SourceKind(mgr.GetCache(), + &egressv1.EgressEndpointSlice{}, handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressEndpointSlice")), - epSlicePredicate{}, - ); err != nil { + epSlicePredicate{}) + if err := c.Watch(sourceEgressEndpointSlice); err != nil { return fmt.Errorf("failed to watch EgressEndpointSlice: %w", err) } - if err := c.Watch( - source.Kind(mgr.GetCache(), &egressv1.EgressClusterEndpointSlice{}), + sourceEgressClusterEndpointSlice := utils.SourceKind(mgr.GetCache(), + &egressv1.EgressClusterEndpointSlice{}, handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressClusterEndpointSlice")), - epSlicePredicate{}, - ); err != nil { + epSlicePredicate{}) + if err := c.Watch(sourceEgressClusterEndpointSlice); err != nil { return fmt.Errorf("failed to watch EgressClusterEndpointSlice: %w", err) } diff --git a/pkg/controller/clusterinfo/clusterinfo.go b/pkg/controller/clusterinfo/clusterinfo.go new file mode 100644 index 000000000..12422e712 --- /dev/null +++ b/pkg/controller/clusterinfo/clusterinfo.go @@ -0,0 +1,375 @@ +// Copyright 2022 Authors of spidernet-io +// SPDX-License-Identifier: Apache-2.0 + +package clusterinfo + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/go-logr/logr" + calicov1 "github.com/tigera/operator/pkg/apis/crd.projectcalico.org/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/spidernet-io/egressgateway/pkg/config" + egressv1 "github.com/spidernet-io/egressgateway/pkg/k8s/apis/v1beta1" + "github.com/spidernet-io/egressgateway/pkg/utils" +) + +var defaultName = "default" + +func NewController(mgr manager.Manager, log logr.Logger, cfg *config.Config) error { + if cfg == nil { + return fmt.Errorf("cfg can not be nil") + } + r := &clusterInfo{ + mgr: mgr, + log: log, + config: cfg, + cli: mgr.GetClient(), + } + c, err := controller.New("cluster-info", mgr, + controller.Options{Reconciler: r}) + if err != nil { + return err + } + + sourceNode := utils.SourceKind(r.mgr.GetCache(), + &corev1.Node{}, + handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("Node")), + nodePredicate{}) + err = c.Watch(sourceNode) + if err != nil { + return fmt.Errorf("failed to watch Node: %w", err) + } + + sourceInfo := utils.SourceKind(r.mgr.GetCache(), + &egressv1.EgressClusterInfo{}, + handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressClusterInfo")), + infoPredicate{}) + err = c.Watch(sourceInfo) + if err != nil { + return fmt.Errorf("failed to watch EgressClusterInfo: %w", err) + } + + r.watchCalico = func() { + for { + err := r.cli.List(context.Background(), &calicov1.IPPoolList{}) + if err == nil { + sourceCalicoIPPool := utils.SourceKind(r.mgr.GetCache(), + &calicov1.IPPool{}, + handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("CalicoIPPool"))) + err = c.Watch(sourceCalicoIPPool) + if err != nil { + log.Error(err, "failed to watch CalicoIPPool", "error", err) + } + break + } + time.Sleep(time.Second * 3) + } + } + + return nil +} + +type clusterInfo struct { + mgr manager.Manager + cli client.Client + log logr.Logger + config *config.Config + doOnce sync.Once + watchCalico func() +} + +func (r *clusterInfo) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + r.doOnce.Do(func() { + for { + err := r.updateK8sServiceCIDR(ctx) + if err == nil { + break + } + r.log.Error(err, "failed to update k8s Service CIDR") + time.Sleep(time.Second * 3) + } + r.watchCalico() + }) + + kind, newReq, err := utils.ParseKindWithReq(req) + if err != nil { + return reconcile.Result{}, err + } + log := r.log.WithValues("kind", kind) + var res reconcile.Result + switch kind { + case "Node": + err = r.reconcileNode(ctx, newReq, log) + case "CalicoIPPool": + err = r.reconcileCalicoIPPool(ctx, newReq, log) + case "EgressClusterInfo": + err = r.reconcileInfo(ctx, newReq, log) + default: + return reconcile.Result{}, nil + } + if err != nil { + return reconcile.Result{}, err + } + return res, nil +} + +func (r *clusterInfo) reconcileNode(ctx context.Context, req reconcile.Request, log logr.Logger) error { + log = log.WithValues("name", req.Name) + log.Info("reconciling") + + info := new(egressv1.EgressClusterInfo) + err := r.cli.Get(ctx, client.ObjectKey{Name: defaultName}, info) + if err != nil { + if !errors.IsNotFound(err) { + return err + } + log.Info("not found default EgressClusterInfo") + return nil + } + // skip if not enable detect node + if !info.Spec.AutoDetect.NodeIP { + return nil + } + + deleted := false + node := new(corev1.Node) + err = r.cli.Get(ctx, req.NamespacedName, node) + if err != nil { + if !errors.IsNotFound(err) { + return err + } + deleted = true + } + deleted = deleted || !node.GetDeletionTimestamp().IsZero() + + if deleted { + log.Info("delete event of node", "delete", req.Name) + if _, ok := info.Status.NodeIP[req.Name]; ok { + delete(info.Status.NodeIP, req.Name) + err := r.cli.Update(ctx, info) + if err != nil { + return fmt.Errorf("failed to update EgressClusterInfo when delete node(%s) event: %w", node.Name, err) + } + } + } else { + log.Info("update event of node", "update", req.Name) + ipv4, ipv6 := getNodeIPList(node) + if val, ok := info.Status.NodeIP[req.Name]; ok { + // diff info.Status.NodeIP[req.Name].IPv4 == ipv4 + // diff info.Status.NodeIP[req.Name].IPv6 == ipv6 + if !utils.EqualStringSlice(val.IPv4, ipv4) || + !utils.EqualStringSlice(val.IPv6, ipv6) { + // then update info.Status.NodeIP[req.Name] + info.Status.NodeIP[req.Name] = egressv1.IPListPair{ + IPv4: ipv4, + IPv6: ipv6, + } + err := r.cli.Update(ctx, info) + if err != nil { + return fmt.Errorf("failed to update EgressClusterInfo when update node(%s) event: %w", node.Name, err) + } + } + } else { + info.Status.NodeIP[req.Name] = egressv1.IPListPair{ + IPv4: ipv4, + IPv6: ipv6, + } + err := r.cli.Update(ctx, info) + if err != nil { + return fmt.Errorf("failed to update EgressClusterInfo when update node(%s) event: %w", node.Name, err) + } + } + } + return nil +} + +func (r *clusterInfo) reconcileCalicoIPPool(ctx context.Context, req reconcile.Request, log logr.Logger) error { + log = log.WithValues("name", req.Name) + log.Info("reconciling") + + info := new(egressv1.EgressClusterInfo) + err := r.cli.Get(ctx, client.ObjectKey{Name: defaultName}, info) + if err != nil { + if !errors.IsNotFound(err) { + return err + } + log.Info("not found default EgressClusterInfo") + return nil + } + // skip if not enable detect pod cidr + if info.Spec.AutoDetect.PodCidrMode != egressv1.CniTypeCalico && + info.Spec.AutoDetect.PodCidrMode != egressv1.CniTypeAuto { + return nil + } + + deleted := false + pool := new(calicov1.IPPool) + err = r.cli.Get(ctx, req.NamespacedName, pool) + if err != nil { + if !errors.IsNotFound(err) { + return err + } + deleted = true + } + deleted = deleted || !pool.GetDeletionTimestamp().IsZero() + + if deleted { + log.Info("delete event of CalicoIPPool", "delete", req.Name) + if _, ok := info.Status.PodCIDR[req.Name]; ok { + delete(info.Status.PodCIDR, req.Name) + err := r.cli.Update(ctx, info) + if err != nil { + return fmt.Errorf("failed to update EgressClusterInfo when delete CalicoIPPool(%s) event: %w", pool.Name, err) + } + } + } else { + log.Info("update event of CalicoIPPool", "update", req.Name) + ipv4, ipv6 := getCalicoIPPoolList(pool) + if val, ok := info.Status.PodCIDR[req.Name]; ok { + // diff info.Status.PodCIDR[req.Name].IPv4 == ipv4 + // diff info.Status.PodCIDR[req.Name].IPv6 == ipv6 + if !utils.EqualStringSlice(val.IPv4, ipv4) || + !utils.EqualStringSlice(val.IPv6, ipv6) { + // then update info.Status.PodCIDR[req.Name] + info.Status.PodCIDR[req.Name] = egressv1.IPListPair{ + IPv4: ipv4, + IPv6: ipv6, + } + err := r.cli.Update(ctx, info) + if err != nil { + return fmt.Errorf("failed to update EgressClusterInfo when update CalicoIPPool(%s) event: %w", pool.Name, err) + } + } + } else { + info.Status.PodCIDR[req.Name] = egressv1.IPListPair{ + IPv4: ipv4, + IPv6: ipv6, + } + err := r.cli.Update(ctx, info) + if err != nil { + return fmt.Errorf("failed to update EgressClusterInfo when update CalicoIPPool(%s) event: %w", pool.Name, err) + } + } + } + + return nil +} + +func (r *clusterInfo) reconcileInfo(ctx context.Context, req reconcile.Request, log logr.Logger) error { + log = log.WithValues("name", req.Name) + log.Info("reconciling") + + deleted := false + info := new(egressv1.EgressClusterInfo) + err := r.cli.Get(ctx, req.NamespacedName, info) + if err != nil { + if !errors.IsNotFound(err) { + return err + } + deleted = true + } + deleted = deleted || !info.GetDeletionTimestamp().IsZero() + + if deleted { + log.Info("delete event of EgressClusterInfo", "delete", req.Name) + return nil + } else { + log.Info("update event of EgressClusterInfo", "update", req.Name) + if !utils.EqualStringSlice(info.Spec.ExtraCidr, info.Status.ExtraCidr) { + info.Status.ExtraCidr = info.Spec.ExtraCidr + err := r.cli.Update(ctx, info) + if err != nil { + return fmt.Errorf("failed to update EgressClusterInfo when update EgressClusterInfo(%s) event: %w", info.Name, err) + } + } + } + return nil +} + +func (r *clusterInfo) updateK8sServiceCIDR(ctx context.Context) error { + info := new(egressv1.EgressClusterInfo) + err := r.cli.Get(ctx, client.ObjectKey{Name: defaultName}, info) + if err != nil { + if !errors.IsNotFound(err) { + return err + } + return nil + } + // skip if not enable detect service cidr + if !info.Spec.AutoDetect.ClusterIP { + return nil + } + v4, v6, err := getClusterCIDR(ctx, r.cli) + if err != nil { + r.log.Error(err, "failed to get cluster cidr") + } + if !utils.EqualStringSlice(info.Status.ClusterIP.IPv4, v4) || + !utils.EqualStringSlice(info.Status.ClusterIP.IPv6, v6) { + + info.Status.ClusterIP = &egressv1.IPListPair{IPv4: v4, IPv6: v6} + err := r.cli.Update(ctx, info) + if err != nil { + return fmt.Errorf("failed to update EgressClusterInfo when update k8s Service CIDR: %w", err) + } + } + return nil +} + +type nodePredicate struct{} + +func (p nodePredicate) Create(_ event.CreateEvent) bool { return true } +func (p nodePredicate) Delete(_ event.DeleteEvent) bool { return true } +func (p nodePredicate) Update(updateEvent event.UpdateEvent) bool { + oldObj, ok := updateEvent.ObjectOld.(*corev1.Node) + if !ok { + return false + } + newObj, ok := updateEvent.ObjectNew.(*corev1.Node) + if !ok { + return false + } + + oldV4, oldV6 := getNodeIPList(oldObj) + newV4, newV6 := getNodeIPList(newObj) + if utils.EqualStringSlice(oldV4, newV4) && utils.EqualStringSlice(oldV6, newV6) { + return false + } + return true + +} +func (p nodePredicate) Generic(_ event.GenericEvent) bool { return false } + +type infoPredicate struct{} + +func (p infoPredicate) Create(_ event.CreateEvent) bool { return true } +func (p infoPredicate) Delete(_ event.DeleteEvent) bool { return false } +func (p infoPredicate) Update(updateEvent event.UpdateEvent) bool { + oldObj, ok := updateEvent.ObjectOld.(*egressv1.EgressClusterInfo) + if !ok { + return false + } + newObj, ok := updateEvent.ObjectNew.(*egressv1.EgressClusterInfo) + if !ok { + return false + } + if oldObj.Spec.AutoDetect != newObj.Spec.AutoDetect { + return true + } + if !utils.EqualStringSlice(oldObj.Spec.ExtraCidr, newObj.Spec.ExtraCidr) { + return true + } + return false +} +func (p infoPredicate) Generic(_ event.GenericEvent) bool { return false } diff --git a/pkg/controller/clusterinfo/cnicalico.go b/pkg/controller/clusterinfo/cnicalico.go new file mode 100644 index 000000000..65643b44b --- /dev/null +++ b/pkg/controller/clusterinfo/cnicalico.go @@ -0,0 +1,22 @@ +// Copyright 2022 Authors of spidernet-io +// SPDX-License-Identifier: Apache-2.0 + +package clusterinfo + +import ( + "github.com/spidernet-io/egressgateway/pkg/utils/ip" + calicov1 "github.com/tigera/operator/pkg/apis/crd.projectcalico.org/v1" +) + +func getCalicoIPPoolList(pool *calicov1.IPPool) (ipv4, ipv6 []string) { + if pool == nil { + return + } + if isV4, err := ip.IsIPv4Cidr(pool.Spec.CIDR); err == nil && isV4 { + ipv4 = append(ipv4, pool.Spec.CIDR) + } + if isV6, err := ip.IsIPv6Cidr(pool.Spec.CIDR); err == nil && isV6 { + ipv6 = append(ipv6, pool.Spec.CIDR) + } + return +} diff --git a/pkg/controller/clusterinfo/k8snode.go b/pkg/controller/clusterinfo/k8snode.go new file mode 100644 index 000000000..54588d19e --- /dev/null +++ b/pkg/controller/clusterinfo/k8snode.go @@ -0,0 +1,28 @@ +// Copyright 2022 Authors of spidernet-io +// SPDX-License-Identifier: Apache-2.0 + +package clusterinfo + +import ( + v1 "k8s.io/api/core/v1" + + "github.com/spidernet-io/egressgateway/pkg/utils/ip" +) + +// GetNodeIPList get node ip list +func getNodeIPList(node *v1.Node) (nodeIPv4, nodeIPv6 []string) { + if node == nil { + return + } + for _, addresses := range node.Status.Addresses { + if addresses.Type == v1.NodeInternalIP { + if isV4, _ := ip.IsIPv4(addresses.Address); isV4 { + nodeIPv4 = append(nodeIPv4, addresses.Address) + } + if isV6, _ := ip.IsIPv6(addresses.Address); isV6 { + nodeIPv6 = append(nodeIPv6, addresses.Address) + } + } + } + return +} diff --git a/pkg/controller/egress_cluster_info/tools.go b/pkg/controller/clusterinfo/k8sservice.go similarity index 55% rename from pkg/controller/egress_cluster_info/tools.go rename to pkg/controller/clusterinfo/k8sservice.go index d95328ed8..383ed7e7b 100644 --- a/pkg/controller/egress_cluster_info/tools.go +++ b/pkg/controller/clusterinfo/k8sservice.go @@ -1,21 +1,43 @@ // Copyright 2022 Authors of spidernet-io // SPDX-License-Identifier: Apache-2.0 -package egressclusterinfo +package clusterinfo import ( "context" "fmt" - "strings" - + "github.com/spidernet-io/egressgateway/pkg/utils/ip" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/spidernet-io/egressgateway/pkg/utils/ip" + "strings" ) -// ParseCidrFromControllerManager get cidr value from kube controller manager -func ParseCidrFromControllerManager(pod *corev1.Pod, param string) (ipv4Range, ipv6Range []string, err error) { +var kubeControllerManagerPodLabel = map[string]string{"component": "kube-controller-manager"} + +func getClusterCIDR(ctx context.Context, cli client.Client) (ipv4, ipv6 []string, err error) { + pods, err := listPodByLabel(ctx, cli, kubeControllerManagerPodLabel) + if err != nil { + return nil, nil, err + } + return parseCIDRFromControllerManager(&pods[0], "cluster-cidr") +} + +func listPodByLabel(ctx context.Context, cli client.Client, + label map[string]string) ([]corev1.Pod, error) { + podList := new(corev1.PodList) + opts := client.MatchingLabels(label) + err := cli.List(ctx, podList, opts) + if err != nil { + return nil, err + } + pods := podList.Items + if len(pods) == 0 { + return nil, fmt.Errorf("failed to get pod") + } + return pods, nil +} + +func parseCIDRFromControllerManager(pod *corev1.Pod, param string) (ipv4, ipv6 []string, err error) { containers := pod.Spec.Containers if len(containers) == 0 { return nil, nil, fmt.Errorf("failed to found containers") @@ -31,45 +53,19 @@ func ParseCidrFromControllerManager(pod *corev1.Pod, param string) (ipv4Range, i if len(ipRange) == 0 { return nil, nil, fmt.Errorf("failed to found %s", param) } - // get cidr ipRanges := strings.Split(ipRange, ",") if len(ipRanges) == 1 { if isV4, _ := ip.IsIPv4Cidr(ipRanges[0]); isV4 { - ipv4Range = ipRanges - ipv6Range = []string{} + ipv4 = ipRanges + ipv6 = []string{} } if isV6, _ := ip.IsIPv6Cidr(ipRanges[0]); isV6 { - ipv6Range = ipRanges - ipv4Range = []string{} - + ipv6 = ipRanges + ipv4 = []string{} } } if len(ipRanges) == 2 { - ipv4Range, ipv6Range = ipRanges[:1], ipRanges[1:] + ipv4, ipv6 = ipRanges[:1], ipRanges[1:] } return } - -// GetPodByLabel get pod by label -func GetPodByLabel(c client.Client, label map[string]string) (*corev1.Pod, error) { - podList := corev1.PodList{} - opts := client.MatchingLabels(label) - err := c.List(context.Background(), &podList, opts) - if err != nil { - return nil, err - } - pods := podList.Items - if len(pods) == 0 { - return nil, fmt.Errorf("failed to get pod") - } - return &pods[0], nil -} - -// GetClusterCidr get k8s default podCidr -func GetClusterCidr(c client.Client) (ipv4Range, ipv6Range []string, err error) { - pod, err := GetPodByLabel(c, kubeControllerManagerPodLabel) - if err != nil { - return nil, nil, err - } - return ParseCidrFromControllerManager(pod, clusterCidr) -} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 1850f844b..2d3b12aea 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -15,7 +15,7 @@ import ( runtimeWebhook "sigs.k8s.io/controller-runtime/pkg/webhook" "github.com/spidernet-io/egressgateway/pkg/config" - egressclusterinfo "github.com/spidernet-io/egressgateway/pkg/controller/egress_cluster_info" + "github.com/spidernet-io/egressgateway/pkg/controller/clusterinfo" "github.com/spidernet-io/egressgateway/pkg/controller/endpoint" "github.com/spidernet-io/egressgateway/pkg/controller/metrics" "github.com/spidernet-io/egressgateway/pkg/controller/tunnel" @@ -80,9 +80,10 @@ func New(cfg *config.Config) (types.Service, error) { if err != nil { return nil, fmt.Errorf("failed to create egress tunnel controller: %w", err) } - err = egressclusterinfo.NewEgressClusterInfoController(mgr, log) + + err = clusterinfo.NewController(mgr, log, cfg) if err != nil { - return nil, fmt.Errorf("failed to create egress cluster info controller: %w", err) + return nil, fmt.Errorf("failed to create cluster info controller: %w", err) } err = endpoint.NewEgressEndpointSliceController(mgr, log, cfg) diff --git a/pkg/controller/egress_cluster_info/egress_cluster_info.go b/pkg/controller/egress_cluster_info/egress_cluster_info.go deleted file mode 100644 index 85faff17b..000000000 --- a/pkg/controller/egress_cluster_info/egress_cluster_info.go +++ /dev/null @@ -1,525 +0,0 @@ -// Copyright 2022 Authors of spidernet-io -// SPDX-License-Identifier: Apache-2.0 - -package egressclusterinfo - -import ( - "context" - "fmt" - "reflect" - "sync/atomic" - "time" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - - "github.com/go-logr/logr" - - calicov1 "github.com/tigera/operator/pkg/apis/crd.projectcalico.org/v1" - - egressv1beta1 "github.com/spidernet-io/egressgateway/pkg/k8s/apis/v1beta1" - "github.com/spidernet-io/egressgateway/pkg/lock" - "github.com/spidernet-io/egressgateway/pkg/utils" - "github.com/spidernet-io/egressgateway/pkg/utils/ip" -) - -type eciReconciler struct { - mgr manager.Manager - c controller.Controller - isWatchCalico atomic.Bool // whether controller need to watch calico - isWatchingCalico atomic.Bool // whether controller is watching calico - isWatchingNode atomic.Bool // whether controller need to watch node - k8sPodCidr map[string]egressv1beta1.IPListPair - v4ClusterCidr, v6ClusterCidr []string - eci *egressv1beta1.EgressClusterInfo - client client.Client - log logr.Logger - eciMutex lock.RWMutex -} - -const ( - defaultEgressClusterInfoName = "default" - k8s = "k8s" - serviceClusterIpRange = "service-cluster-ip-range" - clusterCidr = "cluster-cidr" -) - -const ( - kindNode = "Node" - kindCalicoIPPool = "CalicoIPPool" - kindEGCI = "EGCI" -) - -var kubeControllerManagerPodLabel = map[string]string{"component": "kube-controller-manager"} - -func NewEgressClusterInfoController(mgr manager.Manager, log logr.Logger) error { - r := &eciReconciler{ - mgr: mgr, - eci: new(egressv1beta1.EgressClusterInfo), - client: mgr.GetClient(), - log: log, - k8sPodCidr: make(map[string]egressv1beta1.IPListPair), - v4ClusterCidr: make([]string, 0), - v6ClusterCidr: make([]string, 0), - } - - log.Info("new egressClusterInfo controller") - c, err := controller.New("egressClusterInfo", mgr, - controller.Options{Reconciler: r}) - if err != nil { - return err - } - r.c = c - - log.Info("egressClusterInfo controller watch EgressClusterInfo") - return watchSource(c, source.Kind(mgr.GetCache(), &egressv1beta1.EgressClusterInfo{}), kindEGCI) -} - -// Reconcile support to reconcile of nodes, calicoIPPool and egressClusterInfo -func (r *eciReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - kind, newReq, err := utils.ParseKindWithReq(req) - if err != nil { - return reconcile.Result{}, err - } - log := r.log.WithValues("kind", kind) - - r.eciMutex.Lock() - defer r.eciMutex.Unlock() - - eciStatusCopy := r.eci.Status.DeepCopy() - - // get egressClusterInfo - err = r.getEgressClusterInfo(ctx) - if err != nil { - return reconcile.Result{Requeue: true}, err - } - - switch kind { - case kindNode: - err = r.reconcileNode(ctx, newReq, log) - case kindCalicoIPPool: - if r.eci.Spec.AutoDetect.PodCidrMode != egressv1beta1.CniTypeCalico { - return reconcile.Result{}, nil - } - err = r.reconcileCalicoIPPool(ctx, newReq, log) - case kindEGCI: - err = r.reconcileEgressClusterInfo(ctx, newReq, log) - default: - return reconcile.Result{}, nil - } - - if err != nil { - return reconcile.Result{Requeue: true}, err - } - - if !reflect.DeepEqual(eciStatusCopy, r.eci.Status) { - err = r.client.Status().Update(ctx, r.eci) - if err != nil { - //r.eci = eciCopy - if errors.IsConflict(err) { - return reconcile.Result{Requeue: true}, nil - } - return reconcile.Result{Requeue: true}, err - } - } - - return reconcile.Result{}, nil -} - -// reconcileEgressClusterInfo reconcile cr egressClusterInfo -func (r *eciReconciler) reconcileEgressClusterInfo(ctx context.Context, req reconcile.Request, log logr.Logger) error { - log = log.WithValues("name", req.Name, "namespace", req.Namespace) - log.Info("reconciling") - - // ignore nodeIP - if r.eci.Spec.AutoDetect.NodeIP { - if !r.isWatchingNode.Load() { - // need watch node - if err := watchSource(r.c, source.Kind(r.mgr.GetCache(), &corev1.Node{}), kindNode); err != nil { - return err - } - r.isWatchingNode.Store(true) - // need to list all node - nodesIP, err := r.listNodeIPs(ctx) - if err != nil { - return err - } - r.eci.Status.NodeIP = nodesIP - } - } else { - r.eci.Status.NodeIP = nil - } - - // ignore podCidr - // if r.eci.Spec.AutoDetect.PodCidrMode is not Calico, stop checking for the existence of Calico - if r.eci.Spec.AutoDetect.PodCidrMode != egressv1beta1.CniTypeCalico && r.isWatchCalico.Load() { - r.stopCheckCalico() - } - - switch r.eci.Spec.AutoDetect.PodCidrMode { - case egressv1beta1.CniTypeAuto: - err := r.checkSomeCniExists() - if err != nil { - return err - } - case egressv1beta1.CniTypeCalico: - if !r.isWatchingCalico.Load() { - // set empty before check calico - r.eci.Status.PodCIDR = nil - r.eci.Status.PodCidrMode = egressv1beta1.CniTypeEmpty - - // check if calico is exists - r.startCheckCalico() - } else { - pools, err := r.listCalicoIPPools(ctx) - if err != nil { - return err - } - r.eci.Status.PodCIDR = pools - r.eci.Status.PodCidrMode = egressv1beta1.CniTypeCalico - } - case egressv1beta1.CniTypeK8s: - if _, ok := r.k8sPodCidr[k8s]; !ok { - cidr, err := r.getK8sPodCidr() - if err != nil { - return err - } - r.k8sPodCidr = cidr - } - r.eci.Status.PodCIDR = r.k8sPodCidr - r.eci.Status.PodCidrMode = egressv1beta1.CniTypeK8s - case egressv1beta1.CniTypeEmpty: - r.eci.Status.PodCIDR = nil - r.eci.Status.PodCidrMode = "" - default: - r.log.Error(fmt.Errorf("invalid podCidrMode"), "invalid podCidrMode", "Spec.AutoDetect.PodCidrMode", r.eci.Spec.AutoDetect.PodCidrMode) - } - - // ignore clusterIP - if r.eci.Spec.AutoDetect.ClusterIP { - if len(r.v4ClusterCidr) == 0 { - v4Cidr, v6Cidr, err := r.getServiceClusterIPRange() - if err != nil { - return err - } - r.v4ClusterCidr = v4Cidr - r.v6ClusterCidr = v6Cidr - } - if r.eci.Status.ClusterIP == nil { - r.eci.Status.ClusterIP = new(egressv1beta1.IPListPair) - } - r.eci.Status.ClusterIP.IPv4 = r.v4ClusterCidr - r.eci.Status.ClusterIP.IPv6 = r.v6ClusterCidr - } else { - r.eci.Status.ClusterIP = nil - } - - // extraCidr - if r.eci.Spec.ExtraCidr != nil { - r.eci.Status.ExtraCidr = r.eci.Spec.ExtraCidr - } else { - r.eci.Status.ExtraCidr = nil - } - return nil -} - -// reconcileCalicoIPPool reconcile calico IPPool -func (r *eciReconciler) reconcileCalicoIPPool(ctx context.Context, req reconcile.Request, log logr.Logger) error { - log = log.WithValues("name", req.Name, "namespace", req.Namespace) - log.Info("reconciling") - - deleted := false - ippool := new(calicov1.IPPool) - err := r.client.Get(ctx, req.NamespacedName, ippool) - if err != nil { - if !errors.IsNotFound(err) { - return err - } - deleted = true - } - deleted = deleted || !ippool.GetDeletionTimestamp().IsZero() - - // delete event - if deleted { - log.Info("delete event of calico ippool", "delete", req.Name) - delete(r.eci.Status.PodCIDR, req.Name) - } else { - // not delete event - log.Info("update event of calico ippool", "update", req.Name) - poolsMap, err := r.getCalicoIPPools(ctx, req.Name) - if err != nil { - return err - } - if r.eci.Status.PodCIDR == nil { - r.eci.Status.PodCIDR = make(map[string]egressv1beta1.IPListPair) - } - r.eci.Status.PodCIDR[req.Name] = poolsMap[req.Name] - } - - return nil -} - -// reconcileNode reconcile node -func (r *eciReconciler) reconcileNode(ctx context.Context, req reconcile.Request, log logr.Logger) error { - log = log.WithValues("name", req.Name) - log.Info("reconciling") - - deleted := false - node := new(corev1.Node) - err := r.client.Get(ctx, req.NamespacedName, node) - if err != nil { - if !errors.IsNotFound(err) { - return err - } - deleted = true - } - deleted = deleted || !node.GetDeletionTimestamp().IsZero() - - // delete event - if deleted { - log.Info("delete event of node", "delete", req.Name) - delete(r.eci.Status.NodeIP, req.Name) - } else { - // not delete event - log.Info("update event of node", "update", req.Name) - nodeIPMap, err := r.getNodeIPs(ctx, req.Name) - if err != nil { - return err - } - if r.eci.Status.NodeIP == nil { - r.eci.Status.NodeIP = make(map[string]egressv1beta1.IPListPair) - } - r.eci.Status.NodeIP[req.Name] = nodeIPMap[req.Name] - } - - return nil -} - -// listCalicoIPPools list all calico ippools -func (r *eciReconciler) listCalicoIPPools(ctx context.Context) (map[string]egressv1beta1.IPListPair, error) { - ippoolList := new(calicov1.IPPoolList) - calicoIPPoolMap := make(map[string]egressv1beta1.IPListPair, 0) - - err := r.client.List(ctx, ippoolList) - if err != nil { - return nil, err - } - for _, item := range ippoolList.Items { - cidr := item.Spec.CIDR - ipListPair := egressv1beta1.IPListPair{} - - isV4Cidr, err := ip.IsIPv4Cidr(cidr) - if err != nil { - return nil, err - } - if isV4Cidr { - ipListPair.IPv4 = append(ipListPair.IPv4, cidr) - calicoIPPoolMap[item.Name] = ipListPair - } - isV6Cidr, err := ip.IsIPv6Cidr(cidr) - if err != nil { - return nil, err - } - if isV6Cidr { - ipListPair.IPv6 = append(ipListPair.IPv6, cidr) - calicoIPPoolMap[item.Name] = ipListPair - } - } - return calicoIPPoolMap, nil -} - -// getCalicoIPPools get calico ippool by name -func (r *eciReconciler) getCalicoIPPools(ctx context.Context, poolName string) (map[string]egressv1beta1.IPListPair, error) { - ippool := new(calicov1.IPPool) - calicoIPPoolMap := make(map[string]egressv1beta1.IPListPair, 0) - - err := r.client.Get(ctx, types.NamespacedName{Name: poolName}, ippool) - if err != nil { - return nil, err - } - cidr := ippool.Spec.CIDR - ipListPair := egressv1beta1.IPListPair{} - - isV4Cidr, err := ip.IsIPv4Cidr(cidr) - if err != nil { - return nil, err - } - if isV4Cidr { - ipListPair.IPv4 = append(ipListPair.IPv4, cidr) - calicoIPPoolMap[ippool.Name] = ipListPair - } - isV6Cidr, err := ip.IsIPv6Cidr(cidr) - if err != nil { - return nil, err - } - if isV6Cidr { - ipListPair.IPv6 = append(ipListPair.IPv6, cidr) - calicoIPPoolMap[ippool.Name] = ipListPair - } - return calicoIPPoolMap, nil -} - -// listNodeIPs list all node ips -func (r *eciReconciler) listNodeIPs(ctx context.Context) (map[string]egressv1beta1.IPListPair, error) { - nodeList := new(corev1.NodeList) - nodesIPMap := make(map[string]egressv1beta1.IPListPair, 0) - - err := r.client.List(ctx, nodeList) - if err != nil { - return nil, err - } - - for _, item := range nodeList.Items { - var ipv4s, ipv6s []string - nodeIPv4, nodeIPv6 := utils.GetNodeIP(&item) - if len(nodeIPv4) != 0 { - ipv4s = []string{nodeIPv4} - } - if len(nodeIPv6) != 0 { - ipv6s = []string{nodeIPv6} - } - nodesIPMap[item.Name] = egressv1beta1.IPListPair{IPv4: ipv4s, IPv6: ipv6s} - } - return nodesIPMap, nil -} - -// getNodeIPs get node ip by name -func (r *eciReconciler) getNodeIPs(ctx context.Context, nodeName string) (map[string]egressv1beta1.IPListPair, error) { - node := new(corev1.Node) - nodesIPMap := make(map[string]egressv1beta1.IPListPair, 0) - - err := r.client.Get(ctx, types.NamespacedName{Name: nodeName}, node) - if err != nil { - return nil, err - } - - nodeIPv4, nodeIPv6 := utils.GetNodeIP(node) - var ipv4s, ipv6s []string - if len(nodeIPv4) != 0 { - ipv4s = []string{nodeIPv4} - } - if len(nodeIPv6) != 0 { - ipv6s = []string{nodeIPv6} - } - nodesIPMap[nodeName] = egressv1beta1.IPListPair{IPv4: ipv4s, IPv6: ipv6s} - return nodesIPMap, nil -} - -// getEgressClusterInfo get EgressClusterInfo cr -func (r *eciReconciler) getEgressClusterInfo(ctx context.Context) error { - return r.client.Get(ctx, types.NamespacedName{Name: defaultEgressClusterInfoName}, r.eci) -} - -// getServiceClusterIPRange get service-cluster-ip-range from kube controller manager -func (r *eciReconciler) getServiceClusterIPRange() (ipv4Range, ipv6Range []string, err error) { - pod, err := GetPodByLabel(r.client, kubeControllerManagerPodLabel) - if err != nil { - return nil, nil, err - } - return ParseCidrFromControllerManager(pod, serviceClusterIpRange) -} - -// getK8sPodCidr get k8s default podCidr -func (r *eciReconciler) getK8sPodCidr() (map[string]egressv1beta1.IPListPair, error) { - v4Cidr, v6Cidr, err := GetClusterCidr(r.client) - if err != nil { - return nil, err - } - k8sPodCIDR := make(map[string]egressv1beta1.IPListPair) - k8sPodCIDR[k8s] = egressv1beta1.IPListPair{IPv4: v4Cidr, IPv6: v6Cidr} - return k8sPodCIDR, nil -} - -// checkCalicoExists once calico is detected, start watch -func (r *eciReconciler) checkCalicoExists() { - r.log.V(1).Info("checkCalicoExists...") - for { - if !r.isWatchCalico.Load() { - r.log.V(1).Info("the podCidrMode changed, need not to watch calico") - return - } - pools, err := r.listCalicoIPPools(context.Background()) - if err != nil { - r.log.V(1).Info("failed listCalicoIPPools when checkCalicoExists, try again", "details", err) - time.Sleep(time.Second * 3) - continue - } - - r.log.V(1).Info("find calico ippool, egressClusterInfo controller begin to watch calico ippool") - if err = watchSource(r.c, source.Kind(r.mgr.GetCache(), &calicov1.IPPool{}), kindCalicoIPPool); err != nil { - r.log.V(1).Info("failed watch calico ippool, try again", "details", err) - time.Sleep(time.Second) - continue - } - r.log.V(1).Info("egressClusterInfo controller succeeded to watch calico ippool") - r.isWatchingCalico.Store(true) - - // find calico update the egci - r.eciMutex.Lock() - r.eci.Status.PodCIDR = pools - r.eci.Status.PodCidrMode = egressv1beta1.CniTypeCalico - _ = r.client.Status().Update(context.Background(), r.eci) - r.eciMutex.Unlock() - return - } -} - -// startCheckCalico start a goroutine to check calico exists -func (r *eciReconciler) startCheckCalico() { - r.log.V(1).Info("startCheckCalico...") - r.isWatchCalico.Store(true) - go r.checkCalicoExists() -} - -// stopCheckCalico close the goroutine that check calico exists -func (r *eciReconciler) stopCheckCalico() { - r.log.V(1).Info("stopCheckCalico...") - r.isWatchCalico.Store(false) -} - -// checkSomeCniExists -func (r *eciReconciler) checkSomeCniExists() error { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - // check calico exists - pools, err := r.listCalicoIPPools(ctx) - if err != nil { - r.log.V(1).Info("Not find calico ippool") - } else { - // controller to watch calico ippool - r.log.V(1).Info("begin to watch calico ippool when checkSomeCniExists") - if err = watchSource(r.c, source.Kind(r.mgr.GetCache(), &calicov1.IPPool{}), kindCalicoIPPool); err != nil { - r.log.V(1).Info("failed watch calico ippool when checkSomeCniExists", "details", err) - return err - } - - r.eci.Status.PodCIDR = pools - r.eci.Status.PodCidrMode = egressv1beta1.CniTypeCalico - return nil - } - - // if all cni not found, default is k8s podCidr - k8sCidr, err := r.getK8sPodCidr() - if err != nil { - return err - } - r.k8sPodCidr = k8sCidr - r.eci.Status.PodCIDR = k8sCidr - r.eci.Status.PodCidrMode = egressv1beta1.CniTypeK8s - return nil -} - -// watchSource controller watch given resource -func watchSource(c controller.Controller, source source.Source, kind string) error { - if err := c.Watch(source, handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat(kind))); err != nil { - return fmt.Errorf("failed to watch %s: %w", kind, err) - } - return nil -} diff --git a/pkg/controller/egress_cluster_info/egress_cluster_info_internal_test.go b/pkg/controller/egress_cluster_info/egress_cluster_info_internal_test.go deleted file mode 100644 index e0bf7f6a2..000000000 --- a/pkg/controller/egress_cluster_info/egress_cluster_info_internal_test.go +++ /dev/null @@ -1,533 +0,0 @@ -// Copyright 2022 Authors of spidernet-io -// SPDX-License-Identifier: Apache-2.0 - -package egressclusterinfo - -import ( - "context" - "testing" - - "github.com/go-logr/logr" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/rest" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - "github.com/spidernet-io/egressgateway/pkg/config" - egressv1 "github.com/spidernet-io/egressgateway/pkg/k8s/apis/v1beta1" - "github.com/spidernet-io/egressgateway/pkg/logger" - "github.com/spidernet-io/egressgateway/pkg/schema" - calicov1 "github.com/tigera/operator/pkg/apis/crd.projectcalico.org/v1" -) - -var _ = Describe("EgressClusterInfo", Serial, Label("EgressClusterInfo UT"), func() { - var ctx context.Context - - var ( - // err error - builder *fake.ClientBuilder - r *eciReconciler - egci *egressv1.EgressClusterInfo - objs []client.Object - // cli client.WithWatch - controllerManagerPod, controllerManagerPodV4, controllerManagerPodV6, controllerManagerPodNoCommand, controllerManagerPodNoContainer *corev1.Pod - calicoIPPoolV4, calicoIPPoolV6 *calicov1.IPPool - testNode *corev1.Node - ) - - BeforeEach(func() { - ctx = context.TODO() - - // builder - builder = fake.NewClientBuilder() - builder.WithScheme(schema.GetScheme()) - - // objs - objs = []client.Object{} - - // eciReconciler - r = &eciReconciler{ - // mgr: mgr, - eci: new(egressv1.EgressClusterInfo), - log: logr.Logger{}, - k8sPodCidr: make(map[string]egressv1.IPListPair), - v4ClusterCidr: make([]string, 0), - v6ClusterCidr: make([]string, 0), - } - - // egci - egci = &egressv1.EgressClusterInfo{ - TypeMeta: metav1.TypeMeta{ - Kind: "egressclusterinfos", - APIVersion: "egressgateway.spidernet.io/v1beta1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: egciName, - }, - // Spec: egressv1.EgressClusterInfoSpec{ - // ExtraCidr: []string{"10.10.0.0/16"}, - // }, - } - - // kube-controller-manager pod - controllerManagerPod = &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-controller-manager-test", - Namespace: "kube-system", - Labels: kubeControllerManagerPodLabel, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Command: []string{ - "--cluster-cidr=172.40.0.0/16,fd40::/48", - "--service-cluster-ip-range=172.41.0.0/16,fd41::/108", - }, - }, - }, - }, - } - - controllerManagerPodV4 = &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-controller-manager-test", - Namespace: "kube-system", - Labels: kubeControllerManagerPodLabel, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Command: []string{ - "--cluster-cidr=172.40.0.0/16", - "--service-cluster-ip-range=172.41.0.0/16", - }, - }, - }, - }, - } - - controllerManagerPodV6 = &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-controller-manager-test", - Namespace: "kube-system", - Labels: kubeControllerManagerPodLabel, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Command: []string{ - "--cluster-cidr=fd40::/48", - "--service-cluster-ip-range=fd41::/108", - }, - }, - }, - }, - } - - controllerManagerPodNoCommand = &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-controller-manager-test", - Namespace: "kube-system", - Labels: kubeControllerManagerPodLabel, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Command: []string{}, - }, - }, - }, - } - - controllerManagerPodNoContainer = &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-controller-manager-test", - Namespace: "kube-system", - Labels: kubeControllerManagerPodLabel, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{}, - }, - } - - // test calico ippools - calicoIPPoolV4 = &calicov1.IPPool{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-ippool-v4", - }, - Spec: calicov1.IPPoolSpec{ - CIDR: "10.10.0.0/18", - }, - } - calicoIPPoolV6 = &calicov1.IPPool{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-ippool-v6", - }, - Spec: calicov1.IPPoolSpec{ - CIDR: "fdee:120::/120", - }, - } - - // test node - testNode = &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node", - }, - Status: corev1.NodeStatus{ - Addresses: []corev1.NodeAddress{ - { - Type: corev1.NodeInternalIP, - Address: "10.10.10.1", - }, - { - Type: corev1.NodeInternalIP, - Address: "fddd:10::1", - }, - }, - }, - } - - DeferCleanup(func() { - }) - }) - - // reconcileEgressClusterInfo - Context("reconcileEgressClusterInfo", func() { - It("when failed ParseKindWithReq", func() { - res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "badNamespace", Name: egciName}}) - Expect(err).To(HaveOccurred()) - Expect(res).To(Equal(reconcile.Result{})) - }) - - It("when failed getEgressClusterInfo", func() { - // set client - r.client = builder.Build() - - res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: kindEGCI + "/", Name: egciName}}) - Expect(err).To(HaveOccurred()) - Expect(res).To(Equal(reconcile.Result{Requeue: true})) - }) - - It("when failed update egci", func() { - objs = append(objs, egci) - - // set client without subresource - builder.WithObjects(objs...) - r.client = builder.Build() - - res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: kindEGCI + "/", Name: egciName}}) - Expect(err).To(HaveOccurred()) - Expect(res).To(Equal(reconcile.Result{Requeue: true})) - }) - - It("will success when both egci.Spec.AutoDetect.NodeIP and isWatchingNode are true", func() { - // when egci.Spec.AutoDetect.NodeIP is true - egci.Spec.AutoDetect.NodeIP = true - // when isWatchingNode is true - r.isWatchingNode.Store(true) - - // set client - objs = append(objs, egci) - builder.WithObjects(objs...) - builder.WithStatusSubresource(objs...) - r.client = builder.Build() - - res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: kindEGCI + "/", Name: egciName}}) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(reconcile.Result{})) - }) - - DescribeTable("when egci.Spec.AutoDetec.PodCidrMode", func(isOK bool, prepare func(egci *egressv1.EgressClusterInfo, r *eciReconciler)) { - prepare(egci, r) - res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: kindEGCI + "/", Name: egciName}}) - if isOK { - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(reconcile.Result{})) - } else { - Expect(err).To(HaveOccurred()) - Expect(res).To(Equal(reconcile.Result{Requeue: true})) - } - }, - // when egci.Spec.AutoDetect.PodCidrMode is CniTypeCalico - Entry("CniTypeCalico success", true, func(egci *egressv1.EgressClusterInfo, r *eciReconciler) { - // set egci - egci.Spec.AutoDetect.PodCidrMode = egressv1.CniTypeCalico - - // set eciReconciler - r.isWatchingCalico.Store(true) - objs = append(objs, egci) - builder.WithObjects(objs...) - builder.WithStatusSubresource(objs...) - r.client = builder.Build() - }), - - // when egci.Spec.AutoDetect.PodCidrMode is CniTypeK8s - Entry("CniTypeK8s fail with no commands", false, func(egci *egressv1.EgressClusterInfo, r *eciReconciler) { - // set egci - egci.Spec.AutoDetect.PodCidrMode = egressv1.CniTypeK8s - - // set eciReconciler - objs = []client.Object{controllerManagerPodNoCommand, egci} - builder.WithObjects(objs...) - builder.WithStatusSubresource(objs...) - r.client = builder.Build() - }), - - // when egci.Spec.AutoDetect.PodCidrMode is CniTypeK8s - Entry("CniTypeK8s fail with no containers", false, func(egci *egressv1.EgressClusterInfo, r *eciReconciler) { - // set egci - egci.Spec.AutoDetect.PodCidrMode = egressv1.CniTypeK8s - - // set eciReconciler - objs = []client.Object{controllerManagerPodNoContainer, egci} - builder.WithObjects(objs...) - builder.WithStatusSubresource(objs...) - r.client = builder.Build() - }), - - // when egci.Spec.AutoDetect.PodCidrMode is CniTypeK8s - Entry("CniTypeK8s success dual", true, func(egci *egressv1.EgressClusterInfo, r *eciReconciler) { - // set egci - egci.Spec.AutoDetect.PodCidrMode = egressv1.CniTypeK8s - - // set eciReconciler - objs = []client.Object{controllerManagerPod, egci} - builder.WithObjects(objs...) - builder.WithStatusSubresource(objs...) - r.client = builder.Build() - }), - - // when egci.Spec.AutoDetect.PodCidrMode is CniTypeK8s - Entry("CniTypeK8s success v4", true, func(egci *egressv1.EgressClusterInfo, r *eciReconciler) { - // set egci - egci.Spec.AutoDetect.PodCidrMode = egressv1.CniTypeK8s - - // set eciReconciler - objs = []client.Object{controllerManagerPodV4, egci} - builder.WithObjects(objs...) - builder.WithStatusSubresource(objs...) - r.client = builder.Build() - }), - - // when egci.Spec.AutoDetect.PodCidrMode is CniTypeK8s - Entry("CniTypeK8s success v6", true, func(egci *egressv1.EgressClusterInfo, r *eciReconciler) { - // set egci - egci.Spec.AutoDetect.PodCidrMode = egressv1.CniTypeK8s - - // set eciReconciler - objs = []client.Object{controllerManagerPodV6, egci} - builder.WithObjects(objs...) - builder.WithStatusSubresource(objs...) - r.client = builder.Build() - }), - - // when egci.Spec.AutoDetect.PodCidrMode is CniTypeEmpty - Entry("CniTypeEmpty success", true, func(egci *egressv1.EgressClusterInfo, r *eciReconciler) { - // set egci - egci.Spec.AutoDetect.PodCidrMode = egressv1.CniTypeEmpty - - // set eciReconciler - objs = append(objs, egci) - builder.WithObjects(objs...) - builder.WithStatusSubresource(objs...) - r.client = builder.Build() - }), - - // when egci.Spec.AutoDetect.PodCidrMode is unkownType - Entry("unkownType fail", true, func(egci *egressv1.EgressClusterInfo, r *eciReconciler) { - // set egci - egci.Spec.AutoDetect.PodCidrMode = "unkownType" - - // set eciReconciler - objs = append(objs, egci) - builder.WithObjects(objs...) - builder.WithStatusSubresource(objs...) - r.client = builder.Build() - }), - ) - - It("will success when egci.Spec.AutoDetect.ClusterIP is true", func() { - // when egci.Spec.AutoDetect.NodeIP is true - egci.Spec.AutoDetect.ClusterIP = true - - // set client - objs = []client.Object{controllerManagerPod, egci} - builder.WithObjects(objs...) - builder.WithStatusSubresource(objs...) - r.client = builder.Build() - - res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: kindEGCI + "/", Name: egciName}}) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(reconcile.Result{})) - }) - - It("will success when egci.Spec.ExtraCidr is not nil", func() { - // when egci.Spec.ExtraCidr is not nil - egci.Spec.ExtraCidr = []string{"10.10.0.0/16"} - - // set client - objs = append(objs, egci) - builder.WithObjects(objs...) - builder.WithStatusSubresource(objs...) - r.client = builder.Build() - - res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: kindEGCI + "/", Name: egciName}}) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(reconcile.Result{})) - }) - - }) - - // reconcileCalicoIPPool - Context("reconcileCalicoIPPool", func() { - It("will success when delete event", func() { - // when egci.Spec.AutoDetect.PodCidrMode is CniTypeCalico - egci.Spec.AutoDetect.PodCidrMode = egressv1.CniTypeCalico - - // set client - objs = append(objs, egci) - builder.WithObjects(objs...) - builder.WithStatusSubresource(objs...) - r.client = builder.Build() - - res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: kindCalicoIPPool + "/", Name: "xxx"}}) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(reconcile.Result{})) - }) - - It("will success when update event", func() { - egci.Spec.AutoDetect.PodCidrMode = egressv1.CniTypeCalico - - // set client - objs = []client.Object{egci, calicoIPPoolV4, calicoIPPoolV6} - builder.WithObjects(objs...) - builder.WithStatusSubresource(objs...) - r.client = builder.Build() - - // reconcile calico ippools v4 - res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: kindCalicoIPPool + "/", Name: calicoIPPoolV4.Name}}) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(reconcile.Result{})) - - // reconcile calico ippools v6 - res, err = r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: kindCalicoIPPool + "/", Name: calicoIPPoolV6.Name}}) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(reconcile.Result{})) - - // test method listCalicoIPPools - _, _ = r.listCalicoIPPools(ctx) - - // test method stopCheckCalico - r.stopCheckCalico() - - }) - - }) - - // reconcileNode - Context("reconcileNode", func() { - It("will success when delete event", func() { - // when egci.Spec.AutoDetect.NodeIP is true - egci.Spec.AutoDetect.NodeIP = true - - // set client - objs = append(objs, egci) - builder.WithObjects(objs...) - builder.WithStatusSubresource(objs...) - r.client = builder.Build() - - res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: kindNode + "/", Name: "xxx"}}) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(reconcile.Result{})) - }) - - It("will success when update event", func() { - // when egci.Spec.AutoDetect.NodeIP is true - egci.Spec.AutoDetect.NodeIP = true - - // set client - objs = []client.Object{egci, testNode} - builder.WithObjects(objs...) - builder.WithStatusSubresource(objs...) - r.client = builder.Build() - - // reconcile node - res, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: kindNode + "/", Name: testNode.Name}}) - Expect(err).NotTo(HaveOccurred()) - Expect(res).To(Equal(reconcile.Result{})) - - // test method listNodeIPs - _, _ = r.listNodeIPs(ctx) - }) - }) -}) - -func TestNewEgressClusterInfoController(t *testing.T) { - labels := map[string]string{"app": "nginx1"} - initialObjects := []client.Object{ - &egressv1.EgressClusterPolicy{ - ObjectMeta: metav1.ObjectMeta{ - Name: "p1", - }, - Spec: egressv1.EgressClusterPolicySpec{ - AppliedTo: egressv1.ClusterAppliedTo{ - PodSelector: &metav1.LabelSelector{MatchLabels: labels}, - }, - }, - }, - } - - builder := fake.NewClientBuilder() - builder.WithScheme(schema.GetScheme()) - builder.WithObjects(initialObjects...) - cli := builder.Build() - - mgrOpts := manager.Options{ - Scheme: schema.GetScheme(), - NewClient: func(config *rest.Config, options client.Options) (client.Client, error) { - return cli, nil - }, - } - - cfg := &config.Config{ - KubeConfig: &rest.Config{}, - FileConfig: config.FileConfig{ - TunnelIpv4Subnet: "10.6.1.21/24", - TunnelIpv6Subnet: "fd00::/126", - EnableIPv4: true, - EnableIPv6: true, - MaxNumberEndpointPerSlice: 100, - IPTables: config.IPTables{ - RefreshIntervalSecond: 90, - PostWriteIntervalSecond: 1, - LockTimeoutSecond: 0, - LockProbeIntervalMillis: 50, - LockFilePath: "/run/xtables.lock", - RestoreSupportsLock: true, - }, - Mark: "0x26000000", - GatewayFailover: config.GatewayFailover{ - Enable: true, - TunnelMonitorPeriod: 5, - TunnelUpdatePeriod: 5, - EipEvictionTimeout: 15, - }, - }, - } - log := logger.NewLogger(cfg.EnvConfig.Logger) - mgr, err := ctrl.NewManager(cfg.KubeConfig, mgrOpts) - if err != nil { - t.Fatal(err) - } - err = NewEgressClusterInfoController(mgr, log) - if err != nil { - t.Fatal(err) - } -} diff --git a/pkg/controller/egress_cluster_info/egress_cluster_info_suite_test.go b/pkg/controller/egress_cluster_info/egress_cluster_info_suite_test.go deleted file mode 100644 index 7b445bc0b..000000000 --- a/pkg/controller/egress_cluster_info/egress_cluster_info_suite_test.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2022 Authors of spidernet-io -// SPDX-License-Identifier: Apache-2.0 - -package egressclusterinfo - -import ( - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestEgressClusterInfo(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "EgressClusterInfo Suite UT") -} - -var egciName = "default" - -var _ = BeforeSuite(func() { - - DeferCleanup(func() { - - }) -}) diff --git a/pkg/controller/egress_cluster_info/egress_cluster_info_test.go b/pkg/controller/egress_cluster_info/egress_cluster_info_test.go deleted file mode 100644 index 603b0365a..000000000 --- a/pkg/controller/egress_cluster_info/egress_cluster_info_test.go +++ /dev/null @@ -1,1216 +0,0 @@ -// Copyright 2022 Authors of spidernet-io -// SPDX-License-Identifier: Apache-2.0 - -package egressclusterinfo - -import ( - "context" - "errors" - "reflect" - "testing" - "time" - - "github.com/agiledragon/gomonkey/v2" - "github.com/go-logr/logr" - egressv1 "github.com/spidernet-io/egressgateway/pkg/k8s/apis/v1beta1" - "github.com/spidernet-io/egressgateway/pkg/schema" - "github.com/spidernet-io/egressgateway/pkg/utils/ip" - "github.com/stretchr/testify/assert" - calicov1 "github.com/tigera/operator/pkg/apis/crd.projectcalico.org/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/rest" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" -) - -var ErrForMock = errors.New("mock err") - -func Test_NewEgressClusterInfoController(t *testing.T) { - kubeConfig := &rest.Config{} - mgr, _ := ctrl.NewManager(kubeConfig, manager.Options{}) - log := logr.Logger{} - - patch := gomonkey.NewPatches() - patch.ApplyFuncReturn(controller.New, nil, ErrForMock) - defer patch.Reset() - - err := NewEgressClusterInfoController(mgr, log) - assert.Error(t, err) -} - -func Test_eciReconciler_Reconcile(t *testing.T) { - cases := map[string]struct { - getReqFunc func() reconcile.Request - setReconciler func(*eciReconciler) - patchFunc func(*eciReconciler) []gomonkey.Patches - expErr bool - expRequeue bool - }{ - "reconcile calico, AutoDetect.PodCidrMode not calico": { - getReqFunc: mock_request_calico, - setReconciler: mock_eciReconciler_info_AutoDetect_PodCidrMode_not_calico, - patchFunc: mock_eciReconciler_getEgressClusterInfo_not_err, - }, - "reconcile no matched kind": { - getReqFunc: mock_request_no_match, - patchFunc: mock_eciReconciler_getEgressClusterInfo_not_err, - }, - "failed status Update IsConflict": { - getReqFunc: mock_request_calico, - setReconciler: mock_eciReconciler_info_AutoDetect_PodCidrMode_calico, - patchFunc: mock_Reconciler_Reconcile_failed_Update, - expErr: false, - expRequeue: true, - }, - } - - builder := fake.NewClientBuilder() - builder.WithScheme(schema.GetScheme()) - - r := &eciReconciler{ - eci: new(egressv1.EgressClusterInfo), - log: logr.Logger{}, - k8sPodCidr: make(map[string]egressv1.IPListPair), - v4ClusterCidr: make([]string, 0), - v6ClusterCidr: make([]string, 0), - client: builder.Build(), - } - - for name, tc := range cases { - t.Run(name, func(t *testing.T) { - if tc.setReconciler != nil { - tc.setReconciler(r) - } - - patches := make([]gomonkey.Patches, 0) - if tc.patchFunc != nil { - patches = tc.patchFunc(r) - } - - req := tc.getReqFunc() - ctx := context.TODO() - - res, err := r.Reconcile(ctx, req) - if tc.expErr { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } - if tc.expRequeue { - assert.True(t, res.Requeue) - } - - for _, p := range patches { - p.Reset() - } - }) - } -} - -func Test_eciReconciler_reconcileEgressClusterInfo(t *testing.T) { - cases := map[string]struct { - setReconciler func(*eciReconciler) - patchFunc func(*eciReconciler) []gomonkey.Patches - expErr bool - }{ - "not watch node, failed watch node": { - setReconciler: mock_eciReconciler_info_isWatchingNode_false, - patchFunc: mock_Reconciler_reconcileEgressClusterInfo_watchSource_err, - expErr: true, - }, - - "not watnch node, failed listNodeIPs": { - setReconciler: mock_eciReconciler_info_isWatchingNode_false, - patchFunc: mock_Reconciler_reconcileEgressClusterInfo_listNodeIPs_err, - expErr: true, - }, - - "not watch node, succeeded listNodeIPs": { - setReconciler: mock_eciReconciler_info_isWatchingNode_false, - patchFunc: mock_Reconciler_reconcileEgressClusterInfo_listNodeIPs_succ, - }, - - "need stopCheckCalico": { - setReconciler: mock_eciReconciler_info_need_stopCheckCalico, - }, - - "failed checkSomeCniExists": { - setReconciler: mock_eciReconciler_info_AutoDetect_PodCidrMode_auto, - patchFunc: mock_Reconciler_reconcileEgressClusterInfo_checkSomeCniExists_err, - expErr: true, - }, - - "autoDetect calico, need watch calico, startCheckCalico": { - setReconciler: mock_eciReconciler_info_AutoDetect_calico_isWatchingCalico_false, - patchFunc: mock_Reconciler_reconcileEgressClusterInfo_checkSomeCniExists_err, - }, - - "autoDetect calico, watching calico, failed listCalicoIPPools": { - setReconciler: mock_eciReconciler_info_AutoDetect_calico_isWatchingCalico_true, - patchFunc: mock_Reconciler_reconcileEgressClusterInfo_listCalicoIPPools_err, - expErr: true, - }, - - // "autoDetect ClusterIP, failed getServiceClusterIPRange": { - // setReconciler: mock_eciReconciler_info_AutoDetect_ClusterIP, - // patchFunc: mock_Reconciler_reconcileEgressClusterInfo_getServiceClusterIPRange_err, - // expErr: true, - // }, - } - - builder := fake.NewClientBuilder() - builder.WithScheme(schema.GetScheme()) - cli := builder.Build() - - // objs = append(objs, egci) - // builder.WithObjects(objs...) - // builder.WithStatusSubresource(objs...) - - // mgrOpts := manager.Options{ - // Scheme: schema.GetScheme(), - // NewClient: func(config *rest.Config, options client.Options) (client.Client, error) { - // return cli, nil - // }, - // } - - mgr, _ := ctrl.NewManager(&rest.Config{}, manager.Options{}) - - r := &eciReconciler{ - mgr: mgr, - eci: new(egressv1.EgressClusterInfo), - log: logr.Logger{}, - k8sPodCidr: make(map[string]egressv1.IPListPair), - v4ClusterCidr: make([]string, 0), - v6ClusterCidr: make([]string, 0), - client: cli, - } - c, _ := controller.New("egressClusterInfo", mgr, - controller.Options{Reconciler: r}) - r.c = c - - req := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: kindEGCI + "/", Name: egciName}} - - for name, tc := range cases { - t.Run(name, func(t *testing.T) { - if tc.setReconciler != nil { - tc.setReconciler(r) - } - - patches := make([]gomonkey.Patches, 0) - if tc.patchFunc != nil { - patches = tc.patchFunc(r) - } - - ctx := context.TODO() - - err := r.reconcileEgressClusterInfo(ctx, req, r.log) - if tc.expErr { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } - - for _, p := range patches { - p.Reset() - } - }) - } - -} - -func Test_eciReconciler_reconcileCalicoIPPool(t *testing.T) { - cases := map[string]struct { - setReconciler func(*eciReconciler) - patchFunc func(*eciReconciler) []gomonkey.Patches - expErr bool - }{ - "failed get calicoIPPool": { - patchFunc: mock_Reconciler_reconcileCalicoIPPool_Get_err, - expErr: true, - }, - - "failed getCalicoIPPools": { - patchFunc: mock_Reconciler_reconcileCalicoIPPool_getCalicoIPPools_err, - expErr: true, - }, - } - - builder := fake.NewClientBuilder() - builder.WithScheme(schema.GetScheme()) - cli := builder.Build() - - // objs = append(objs, egci) - // builder.WithObjects(objs...) - // builder.WithStatusSubresource(objs...) - - // mgrOpts := manager.Options{ - // Scheme: schema.GetScheme(), - // NewClient: func(config *rest.Config, options client.Options) (client.Client, error) { - // return cli, nil - // }, - // } - - mgr, _ := ctrl.NewManager(&rest.Config{}, manager.Options{}) - - r := &eciReconciler{ - mgr: mgr, - eci: new(egressv1.EgressClusterInfo), - log: logr.Logger{}, - k8sPodCidr: make(map[string]egressv1.IPListPair), - v4ClusterCidr: make([]string, 0), - v6ClusterCidr: make([]string, 0), - client: cli, - } - c, _ := controller.New("egressClusterInfo", mgr, - controller.Options{Reconciler: r}) - r.c = c - - req := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: kindCalicoIPPool + "/", Name: "xxx"}} - - for name, tc := range cases { - t.Run(name, func(t *testing.T) { - if tc.setReconciler != nil { - tc.setReconciler(r) - } - - patches := make([]gomonkey.Patches, 0) - if tc.patchFunc != nil { - patches = tc.patchFunc(r) - } - - ctx := context.TODO() - - err := r.reconcileCalicoIPPool(ctx, req, r.log) - if tc.expErr { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } - for _, p := range patches { - p.Reset() - } - }) - } -} - -func Test_eciReconciler_reconcileNode(t *testing.T) { - cases := map[string]struct { - setReconciler func(*eciReconciler) - patchFunc func(*eciReconciler) []gomonkey.Patches - expErr bool - }{ - "failed get node": { - patchFunc: mock_Reconciler_reconcileNode_Get_err, - expErr: true, - }, - - "failed getNodeIPs": { - patchFunc: mock_Reconciler_reconcileNode_getNodeIPs_err, - expErr: true, - }, - } - - builder := fake.NewClientBuilder() - builder.WithScheme(schema.GetScheme()) - cli := builder.Build() - - // objs = append(objs, egci) - // builder.WithObjects(objs...) - // builder.WithStatusSubresource(objs...) - - // mgrOpts := manager.Options{ - // Scheme: schema.GetScheme(), - // NewClient: func(config *rest.Config, options client.Options) (client.Client, error) { - // return cli, nil - // }, - // } - - mgr, _ := ctrl.NewManager(&rest.Config{}, manager.Options{}) - - r := &eciReconciler{ - mgr: mgr, - eci: new(egressv1.EgressClusterInfo), - log: logr.Logger{}, - k8sPodCidr: make(map[string]egressv1.IPListPair), - v4ClusterCidr: make([]string, 0), - v6ClusterCidr: make([]string, 0), - client: cli, - } - c, _ := controller.New("egressClusterInfo", mgr, - controller.Options{Reconciler: r}) - r.c = c - - req := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: kindCalicoIPPool + "/", Name: "xxx"}} - - for name, tc := range cases { - t.Run(name, func(t *testing.T) { - if tc.setReconciler != nil { - tc.setReconciler(r) - } - - patches := make([]gomonkey.Patches, 0) - if tc.patchFunc != nil { - patches = tc.patchFunc(r) - } - - ctx := context.TODO() - - err := r.reconcileNode(ctx, req, r.log) - if tc.expErr { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } - for _, p := range patches { - p.Reset() - } - }) - } -} - -func Test_eciReconciler_listCalicoIPPools(t *testing.T) { - cases := map[string]struct { - setReconciler func(*eciReconciler) - patchFunc func(*eciReconciler) []gomonkey.Patches - expErr bool - }{ - "failed List": { - patchFunc: mock_Reconciler_listCalicoIPPools_List_err, - expErr: true, - }, - - "failed IsIPv4Cidr": { - patchFunc: mock_Reconciler_listCalicoIPPools_IsIPv4Cidr_err, - expErr: true, - }, - "failed IsIPv6Cidr": { - patchFunc: mock_Reconciler_listCalicoIPPools_IsIPv6Cidr_err, - expErr: true, - }, - } - - calicoIPPoolV4 := &calicov1.IPPool{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-ippool-v4", - }, - Spec: calicov1.IPPoolSpec{ - // CIDR: "xxx", - CIDR: "10.10.0.0/18", - }, - } - calicoIPPoolV6 := &calicov1.IPPool{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-ippool-v6", - }, - Spec: calicov1.IPPoolSpec{ - CIDR: "fdee:120::/120", - }, - } - - var objs []client.Object - - builder := fake.NewClientBuilder() - builder.WithScheme(schema.GetScheme()) - objs = append(objs, calicoIPPoolV4, calicoIPPoolV6) - builder.WithObjects(objs...) - builder.WithStatusSubresource(objs...) - - cli := builder.Build() - - mgrOpts := manager.Options{ - Scheme: schema.GetScheme(), - NewClient: func(config *rest.Config, options client.Options) (client.Client, error) { - return cli, nil - }, - } - - mgr, _ := ctrl.NewManager(&rest.Config{}, mgrOpts) - - r := &eciReconciler{ - mgr: mgr, - eci: new(egressv1.EgressClusterInfo), - log: logr.Logger{}, - k8sPodCidr: make(map[string]egressv1.IPListPair), - v4ClusterCidr: make([]string, 0), - v6ClusterCidr: make([]string, 0), - client: cli, - } - c, _ := controller.New("egressClusterInfo", mgr, - controller.Options{Reconciler: r}) - r.c = c - - for name, tc := range cases { - t.Run(name, func(t *testing.T) { - if tc.setReconciler != nil { - tc.setReconciler(r) - } - - patches := make([]gomonkey.Patches, 0) - if tc.patchFunc != nil { - patches = tc.patchFunc(r) - } - - ctx := context.TODO() - - _, err := r.listCalicoIPPools(ctx) - if tc.expErr { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } - - for _, p := range patches { - p.Reset() - } - }) - } -} - -func Test_eciReconciler_getCalicoIPPools(t *testing.T) { - cases := map[string]struct { - setReconciler func(*eciReconciler) - objs []client.Object - patchFunc func(*eciReconciler) []gomonkey.Patches - expErr bool - }{ - "failed Get": { - objs: mock_IPPoolList(), - patchFunc: mock_Reconciler_reconcileNode_getCalicoIPPools_err, - expErr: true, - }, - - "failed IsIPv4Cidr": { - objs: mock_calicoIPPoolV4(), - patchFunc: mock_Reconciler_listCalicoIPPools_IsIPv4Cidr_err, - expErr: true, - }, - "failed IsIPv6Cidr": { - objs: mock_calicoIPPoolV6(), - patchFunc: mock_Reconciler_listCalicoIPPools_IsIPv6Cidr_err, - expErr: true, - }, - } - - for name, tc := range cases { - t.Run(name, func(t *testing.T) { - builder := fake.NewClientBuilder() - builder.WithScheme(schema.GetScheme()) - builder.WithObjects(tc.objs...) - builder.WithStatusSubresource(tc.objs...) - - cli := builder.Build() - - mgrOpts := manager.Options{ - Scheme: schema.GetScheme(), - NewClient: func(config *rest.Config, options client.Options) (client.Client, error) { - return cli, nil - }, - } - - mgr, _ := ctrl.NewManager(&rest.Config{}, mgrOpts) - - r := &eciReconciler{ - mgr: mgr, - eci: new(egressv1.EgressClusterInfo), - log: logr.Logger{}, - k8sPodCidr: make(map[string]egressv1.IPListPair), - v4ClusterCidr: make([]string, 0), - v6ClusterCidr: make([]string, 0), - client: cli, - } - c, _ := controller.New("egressClusterInfo", mgr, - controller.Options{Reconciler: r}) - r.c = c - - if tc.setReconciler != nil { - tc.setReconciler(r) - } - - patches := make([]gomonkey.Patches, 0) - if tc.patchFunc != nil { - patches = tc.patchFunc(r) - } - - ctx := context.TODO() - - _, err := r.getCalicoIPPools(ctx, tc.objs[0].GetName()) - if tc.expErr { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } - for _, p := range patches { - p.Reset() - } - }) - } -} - -func Test_eciReconciler_listNodeIPs(t *testing.T) { - cases := map[string]struct { - setReconciler func(*eciReconciler) - patchFunc func(*eciReconciler) []gomonkey.Patches - expErr bool - }{ - "failed List": { - patchFunc: mock_Reconciler_listNodeIPs_List_err, - expErr: true, - }, - } - - builder := fake.NewClientBuilder() - builder.WithScheme(schema.GetScheme()) - - cli := builder.Build() - - mgrOpts := manager.Options{ - Scheme: schema.GetScheme(), - NewClient: func(config *rest.Config, options client.Options) (client.Client, error) { - return cli, nil - }, - } - - mgr, _ := ctrl.NewManager(&rest.Config{}, mgrOpts) - - r := &eciReconciler{ - mgr: mgr, - eci: new(egressv1.EgressClusterInfo), - log: logr.Logger{}, - k8sPodCidr: make(map[string]egressv1.IPListPair), - v4ClusterCidr: make([]string, 0), - v6ClusterCidr: make([]string, 0), - client: cli, - } - c, _ := controller.New("egressClusterInfo", mgr, - controller.Options{Reconciler: r}) - r.c = c - - for name, tc := range cases { - t.Run(name, func(t *testing.T) { - if tc.setReconciler != nil { - tc.setReconciler(r) - } - - patches := make([]gomonkey.Patches, 0) - if tc.patchFunc != nil { - patches = tc.patchFunc(r) - } - - ctx := context.TODO() - - _, err := r.listNodeIPs(ctx) - if tc.expErr { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } - - for _, p := range patches { - p.Reset() - } - }) - } -} - -func Test_eciReconciler_getNodeIPs(t *testing.T) { - cases := map[string]struct { - setReconciler func(*eciReconciler) - patchFunc func(*eciReconciler) []gomonkey.Patches - expErr bool - }{ - "failed Get": { - patchFunc: mock_Reconciler_reconcileNode_getNodeIPs_Get_err, - expErr: true, - }, - } - - builder := fake.NewClientBuilder() - builder.WithScheme(schema.GetScheme()) - - cli := builder.Build() - - mgrOpts := manager.Options{ - Scheme: schema.GetScheme(), - NewClient: func(config *rest.Config, options client.Options) (client.Client, error) { - return cli, nil - }, - } - - mgr, _ := ctrl.NewManager(&rest.Config{}, mgrOpts) - - r := &eciReconciler{ - mgr: mgr, - eci: new(egressv1.EgressClusterInfo), - log: logr.Logger{}, - k8sPodCidr: make(map[string]egressv1.IPListPair), - v4ClusterCidr: make([]string, 0), - v6ClusterCidr: make([]string, 0), - client: cli, - } - c, _ := controller.New("egressClusterInfo", mgr, - controller.Options{Reconciler: r}) - r.c = c - - for name, tc := range cases { - t.Run(name, func(t *testing.T) { - if tc.setReconciler != nil { - tc.setReconciler(r) - } - - patches := make([]gomonkey.Patches, 0) - if tc.patchFunc != nil { - patches = tc.patchFunc(r) - } - - ctx := context.TODO() - - _, err := r.getNodeIPs(ctx, "fakeName") - if tc.expErr { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } - for _, p := range patches { - p.Reset() - } - }) - } -} - -func Test_eciReconciler_checkCalicoExists(t *testing.T) { - - builder := fake.NewClientBuilder() - builder.WithScheme(schema.GetScheme()) - - cli := builder.Build() - - mgrOpts := manager.Options{ - Scheme: schema.GetScheme(), - NewClient: func(config *rest.Config, options client.Options) (client.Client, error) { - return cli, nil - }, - } - - mgr, _ := ctrl.NewManager(&rest.Config{}, mgrOpts) - - r := &eciReconciler{ - mgr: mgr, - eci: new(egressv1.EgressClusterInfo), - log: logr.Logger{}, - k8sPodCidr: make(map[string]egressv1.IPListPair), - v4ClusterCidr: make([]string, 0), - v6ClusterCidr: make([]string, 0), - client: cli, - } - c, _ := controller.New("egressClusterInfo", mgr, - controller.Options{Reconciler: r}) - r.c = c - - t.Run("isWatchCalico is false", func(t *testing.T) { - r.isWatchCalico.Store(false) - r.checkCalicoExists() - }) - t.Run("failed listCalicoIPPools", func(t *testing.T) { - r.isWatchCalico.Store(true) - - var patches []gomonkey.Patches - defer func() { - for _, p := range patches { - p.Reset() - } - }() - - go func() { - patch1 := gomonkey.ApplyPrivateMethod(r, "listCalicoIPPools", func(_ *eciReconciler) (map[string]egressv1.IPListPair, error) { - return nil, ErrForMock - }) - - time.Sleep(time.Second * 3) - patch1.Reset() - - patch2 := gomonkey.ApplyPrivateMethod(r, "listCalicoIPPools", func(_ *eciReconciler) (map[string]egressv1.IPListPair, error) { - return nil, nil - }) - patches = append(patches, *patch2) - - patch3 := gomonkey.ApplyFuncReturn(watchSource, nil) - patches = append(patches, *patch3) - - }() - - time.Sleep(time.Second) - - r.checkCalicoExists() - - }) - - t.Run("failed watchSource", func(t *testing.T) { - calicoIPPoolV4 := &calicov1.IPPool{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-ippool-v4", - }, - Spec: calicov1.IPPoolSpec{ - // CIDR: "xxx", - CIDR: "10.10.0.0/18", - }, - } - calicoIPPoolV6 := &calicov1.IPPool{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-ippool-v6", - }, - Spec: calicov1.IPPoolSpec{ - CIDR: "fdee:120::/120", - }, - } - - var objs []client.Object - - builder := fake.NewClientBuilder() - builder.WithScheme(schema.GetScheme()) - objs = append(objs, calicoIPPoolV4, calicoIPPoolV6) - builder.WithObjects(objs...) - builder.WithStatusSubresource(objs...) - cli := builder.Build() - - mgrOpts := manager.Options{ - Scheme: schema.GetScheme(), - NewClient: func(config *rest.Config, options client.Options) (client.Client, error) { - return cli, nil - }, - } - - mgr, _ := ctrl.NewManager(&rest.Config{}, mgrOpts) - - r := &eciReconciler{ - mgr: mgr, - eci: new(egressv1.EgressClusterInfo), - log: logr.Logger{}, - k8sPodCidr: make(map[string]egressv1.IPListPair), - v4ClusterCidr: make([]string, 0), - v6ClusterCidr: make([]string, 0), - client: cli, - } - c, _ := controller.New("egressClusterInfo", mgr, - controller.Options{Reconciler: r}) - r.c = c - - r.isWatchCalico.Store(true) - - var patches []gomonkey.Patches - - patch2 := gomonkey.ApplyFuncSeq(watchSource, []gomonkey.OutputCell{ - {Values: gomonkey.Params{ErrForMock}, Times: 3}, - {Values: gomonkey.Params{nil}, Times: 3}, - }) - patches = append(patches, *patch2) - - r.checkCalicoExists() - - for _, p := range patches { - p.Reset() - } - }) -} - -func Test_eciReconciler_getServiceClusterIPRange(t *testing.T) { - - builder := fake.NewClientBuilder() - builder.WithScheme(schema.GetScheme()) - - cli := builder.Build() - - mgrOpts := manager.Options{ - Scheme: schema.GetScheme(), - NewClient: func(config *rest.Config, options client.Options) (client.Client, error) { - return cli, nil - }, - } - - mgr, _ := ctrl.NewManager(&rest.Config{}, mgrOpts) - - r := &eciReconciler{ - mgr: mgr, - eci: new(egressv1.EgressClusterInfo), - log: logr.Logger{}, - k8sPodCidr: make(map[string]egressv1.IPListPair), - v4ClusterCidr: make([]string, 0), - v6ClusterCidr: make([]string, 0), - client: cli, - } - c, _ := controller.New("egressClusterInfo", mgr, - controller.Options{Reconciler: r}) - r.c = c - - t.Run("failed GetPodByLabel", func(t *testing.T) { - patch := gomonkey.ApplyFuncReturn(GetPodByLabel, nil, ErrForMock) - defer patch.Reset() - _, _, err := r.getServiceClusterIPRange() - assert.Error(t, err) - }) -} - -func Test_eciReconciler_checkSomeCniExists(t *testing.T) { - cases := map[string]struct { - setReconciler func(*eciReconciler) - patchFunc func(*eciReconciler) []gomonkey.Patches - expErr bool - }{ - "failed listCalicoIPPools": { - patchFunc: mock_Reconciler_checkSomeCniExists_listCalicoIPPools_err, - expErr: true, - }, - "failed watchSource": { - patchFunc: mock_Reconciler_checkSomeCniExists_watchSource_err, - expErr: true, - }, - "succeeded watchSource": { - patchFunc: mock_Reconciler_checkSomeCniExists_watchSource_succ, - }, - - "failed getK8sPodCidr": { - patchFunc: mock_Reconciler_checkSomeCniExists_getK8sPodCidr_err, - expErr: true, - }, - - "succeeded getK8sPodCidr": { - patchFunc: mock_Reconciler_checkSomeCniExists_getK8sPodCidr_succ, - }, - } - - builder := fake.NewClientBuilder() - builder.WithScheme(schema.GetScheme()) - - cli := builder.Build() - - mgrOpts := manager.Options{ - Scheme: schema.GetScheme(), - NewClient: func(config *rest.Config, options client.Options) (client.Client, error) { - return cli, nil - }, - } - - mgr, _ := ctrl.NewManager(&rest.Config{}, mgrOpts) - - r := &eciReconciler{ - mgr: mgr, - eci: new(egressv1.EgressClusterInfo), - log: logr.Logger{}, - k8sPodCidr: make(map[string]egressv1.IPListPair), - v4ClusterCidr: make([]string, 0), - v6ClusterCidr: make([]string, 0), - client: cli, - } - c, _ := controller.New("egressClusterInfo", mgr, - controller.Options{Reconciler: r}) - r.c = c - - for name, tc := range cases { - t.Run(name, func(t *testing.T) { - if tc.setReconciler != nil { - tc.setReconciler(r) - } - - patches := make([]gomonkey.Patches, 0) - if tc.patchFunc != nil { - patches = tc.patchFunc(r) - } - - err := r.checkSomeCniExists() - if tc.expErr { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } - for _, p := range patches { - p.Reset() - } - }) - } -} -func Test_watchSource(t *testing.T) { - - builder := fake.NewClientBuilder() - builder.WithScheme(schema.GetScheme()) - - cli := builder.Build() - - mgr, _ := ctrl.NewManager(&rest.Config{}, manager.Options{}) - - r := &eciReconciler{ - mgr: mgr, - eci: new(egressv1.EgressClusterInfo), - log: logr.Logger{}, - k8sPodCidr: make(map[string]egressv1.IPListPair), - v4ClusterCidr: make([]string, 0), - v6ClusterCidr: make([]string, 0), - client: cli, - } - c, _ := controller.New("egressClusterInfo", mgr, - controller.Options{Reconciler: r}) - r.c = c - - t.Run("failed Watch", func(t *testing.T) { - patch := gomonkey.ApplyMethodReturn(c, "Watch", ErrForMock) - defer patch.Reset() - err := watchSource(c, source.Kind(mgr.GetCache(), &egressv1.EgressClusterInfo{}), kindEGCI) - assert.Error(t, err) - }) -} - -func mock_eciReconciler_info_AutoDetect_PodCidrMode_not_calico(r *eciReconciler) { - r.eci.Spec.AutoDetect.PodCidrMode = egressv1.CniTypeK8s -} - -func mock_eciReconciler_info_AutoDetect_PodCidrMode_calico(r *eciReconciler) { - r.eci.Spec.AutoDetect.PodCidrMode = egressv1.CniTypeCalico -} - -func mock_request_calico() reconcile.Request { - return reconcile.Request{NamespacedName: types.NamespacedName{Namespace: kindCalicoIPPool + "/", Name: "xxx"}} -} - -func mock_request_no_match() reconcile.Request { - return reconcile.Request{NamespacedName: types.NamespacedName{Namespace: "notMatch" + "/", Name: "xxx"}} -} - -func mock_eciReconciler_getEgressClusterInfo_not_err(r *eciReconciler) []gomonkey.Patches { - patch := gomonkey.ApplyPrivateMethod(r, "getEgressClusterInfo", func(_ *eciReconciler) error { - return nil - }) - return []gomonkey.Patches{*patch} -} - -func mock_Reconciler_Reconcile_failed_Update(r *eciReconciler) []gomonkey.Patches { - patch1 := gomonkey.ApplyPrivateMethod(r, "getEgressClusterInfo", func(_ *eciReconciler) error { - return nil - }) - patch2 := gomonkey.ApplyFuncReturn(reflect.DeepEqual, false) - patch3 := gomonkey.ApplyPrivateMethod(r, "reconcileCalicoIPPool", func(_ *eciReconciler) error { - return nil - }) - patch4 := gomonkey.ApplyFuncReturn(apierrors.IsConflict, true) - return []gomonkey.Patches{*patch1, *patch2, *patch3, *patch4} -} - -func mock_Reconciler_reconcileEgressClusterInfo_watchSource_err(r *eciReconciler) []gomonkey.Patches { - patch1 := gomonkey.ApplyFuncReturn(watchSource, ErrForMock) - return []gomonkey.Patches{*patch1} -} - -func mock_eciReconciler_info_isWatchingNode_false(r *eciReconciler) { - r.eci.Spec.AutoDetect.NodeIP = true - r.isWatchingNode.Store(false) -} - -func mock_Reconciler_reconcileEgressClusterInfo_listNodeIPs_err(r *eciReconciler) []gomonkey.Patches { - patch1 := gomonkey.ApplyFuncReturn(watchSource, nil) - patch2 := gomonkey.ApplyPrivateMethod(r, "listNodeIPs", func(_ *eciReconciler) (map[string]egressv1.IPListPair, error) { - return nil, ErrForMock - }) - return []gomonkey.Patches{*patch1, *patch2} -} - -func mock_Reconciler_reconcileEgressClusterInfo_listNodeIPs_succ(r *eciReconciler) []gomonkey.Patches { - patch1 := gomonkey.ApplyFuncReturn(watchSource, nil) - patch2 := gomonkey.ApplyPrivateMethod(r, "listNodeIPs", func(_ *eciReconciler) (map[string]egressv1.IPListPair, error) { - return nil, nil - }) - return []gomonkey.Patches{*patch1, *patch2} -} - -func mock_eciReconciler_info_need_stopCheckCalico(r *eciReconciler) { - r.eci.Spec.AutoDetect.PodCidrMode = egressv1.CniTypeEmpty - r.isWatchCalico.Store(true) -} - -func mock_eciReconciler_info_AutoDetect_PodCidrMode_auto(r *eciReconciler) { - r.eci.Spec.AutoDetect.PodCidrMode = egressv1.CniTypeAuto -} - -func mock_Reconciler_reconcileEgressClusterInfo_checkSomeCniExists_err(r *eciReconciler) []gomonkey.Patches { - patch1 := gomonkey.ApplyPrivateMethod(r, "checkSomeCniExists", func(_ *eciReconciler) error { - return ErrForMock - }) - return []gomonkey.Patches{*patch1} -} - -func mock_eciReconciler_info_AutoDetect_calico_isWatchingCalico_false(r *eciReconciler) { - r.isWatchingCalico.Store(false) - r.eci.Spec.AutoDetect.PodCidrMode = egressv1.CniTypeCalico -} - -func mock_eciReconciler_info_AutoDetect_calico_isWatchingCalico_true(r *eciReconciler) { - r.isWatchingCalico.Store(true) - r.eci.Spec.AutoDetect.PodCidrMode = egressv1.CniTypeCalico -} - -func mock_Reconciler_reconcileEgressClusterInfo_listCalicoIPPools_err(r *eciReconciler) []gomonkey.Patches { - patch1 := gomonkey.ApplyPrivateMethod(r, "listCalicoIPPools", func(_ *eciReconciler) (map[string]egressv1.IPListPair, error) { - return nil, ErrForMock - }) - return []gomonkey.Patches{*patch1} -} - -func mock_eciReconciler_info_AutoDetect_ClusterIP(r *eciReconciler) { - r.eci.Spec.AutoDetect.ClusterIP = true -} - -func mock_Reconciler_reconcileEgressClusterInfo_getServiceClusterIPRange_err(r *eciReconciler) []gomonkey.Patches { - patch1 := gomonkey.ApplyPrivateMethod(r, "getServiceClusterIPRange", func(_ *eciReconciler) (ipv4Range, ipv6Range []string, err error) { - return nil, nil, ErrForMock - }) - return []gomonkey.Patches{*patch1} -} - -func mock_Reconciler_reconcileCalicoIPPool_Get_err(r *eciReconciler) []gomonkey.Patches { - patch1 := gomonkey.ApplyMethodReturn(r.client, "Get", ErrForMock) - return []gomonkey.Patches{*patch1} -} - -func mock_Reconciler_reconcileCalicoIPPool_getCalicoIPPools_err(r *eciReconciler) []gomonkey.Patches { - patch1 := gomonkey.ApplyMethodReturn(r.client, "Get", nil) - patch2 := gomonkey.ApplyPrivateMethod(r, "getCalicoIPPools", func(_ *eciReconciler) (map[string]egressv1.IPListPair, error) { - return nil, ErrForMock - }) - return []gomonkey.Patches{*patch1, *patch2} -} - -func mock_Reconciler_reconcileNode_Get_err(r *eciReconciler) []gomonkey.Patches { - patch1 := gomonkey.ApplyMethodReturn(r.client, "Get", ErrForMock) - return []gomonkey.Patches{*patch1} -} - -func mock_Reconciler_reconcileNode_getNodeIPs_err(r *eciReconciler) []gomonkey.Patches { - patch1 := gomonkey.ApplyMethodReturn(r.client, "Get", nil) - patch2 := gomonkey.ApplyPrivateMethod(r, "getNodeIPs", func(_ *eciReconciler) (map[string]egressv1.IPListPair, error) { - return nil, ErrForMock - }) - return []gomonkey.Patches{*patch1, *patch2} -} - -func mock_Reconciler_listCalicoIPPools_List_err(r *eciReconciler) []gomonkey.Patches { - patch1 := gomonkey.ApplyMethodReturn(r.client, "List", ErrForMock) - return []gomonkey.Patches{*patch1} -} - -func mock_Reconciler_listCalicoIPPools_IsIPv4Cidr_err(r *eciReconciler) []gomonkey.Patches { - patch1 := gomonkey.ApplyFuncReturn(ip.IsIPv4Cidr, false, ErrForMock) - return []gomonkey.Patches{*patch1} -} - -func mock_Reconciler_listCalicoIPPools_IsIPv6Cidr_err(r *eciReconciler) []gomonkey.Patches { - patch1 := gomonkey.ApplyFuncReturn(ip.IsIPv6Cidr, false, ErrForMock) - return []gomonkey.Patches{*patch1} -} - -func mock_Reconciler_reconcileNode_getCalicoIPPools_err(r *eciReconciler) []gomonkey.Patches { - patch1 := gomonkey.ApplyMethodReturn(r.client, "Get", ErrForMock) - return []gomonkey.Patches{*patch1} -} - -func mock_calicoIPPoolV4() []client.Object { - pool := &calicov1.IPPool{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-ippool-v4", - }, - Spec: calicov1.IPPoolSpec{ - // CIDR: "xxx", - CIDR: "10.10.0.0/18", - }, - } - return []client.Object{pool} -} - -func mock_calicoIPPoolV6() []client.Object { - pool := &calicov1.IPPool{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-ippool-v6", - }, - Spec: calicov1.IPPoolSpec{ - CIDR: "fdee:120::/120", - }, - } - return []client.Object{pool} - -} - -func mock_IPPoolList() []client.Object { - calicoIPPoolV4 := &calicov1.IPPool{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-ippool-v4", - }, - Spec: calicov1.IPPoolSpec{ - // CIDR: "xxx", - CIDR: "10.10.0.0/18", - }, - } - calicoIPPoolV6 := &calicov1.IPPool{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-ippool-v6", - }, - Spec: calicov1.IPPoolSpec{ - CIDR: "fdee:120::/120", - }, - } - return []client.Object{calicoIPPoolV4, calicoIPPoolV6} -} - -func mock_Reconciler_listNodeIPs_List_err(r *eciReconciler) []gomonkey.Patches { - patch1 := gomonkey.ApplyMethodReturn(r.client, "List", ErrForMock) - return []gomonkey.Patches{*patch1} -} - -func mock_Reconciler_reconcileNode_getNodeIPs_Get_err(r *eciReconciler) []gomonkey.Patches { - patch1 := gomonkey.ApplyMethodReturn(r.client, "Get", ErrForMock) - return []gomonkey.Patches{*patch1} -} - -func mock_Reconciler_checkSomeCniExists_listCalicoIPPools_err(r *eciReconciler) []gomonkey.Patches { - patch1 := gomonkey.ApplyPrivateMethod(r, "listCalicoIPPools", func(_ *eciReconciler) (map[string]egressv1.IPListPair, error) { - return nil, ErrForMock - }) - return []gomonkey.Patches{*patch1} -} - -func mock_Reconciler_checkSomeCniExists_watchSource_err(r *eciReconciler) []gomonkey.Patches { - patch1 := gomonkey.ApplyPrivateMethod(r, "listCalicoIPPools", func(_ *eciReconciler) (map[string]egressv1.IPListPair, error) { - return nil, nil - }) - patch2 := gomonkey.ApplyFuncReturn(watchSource, ErrForMock) - return []gomonkey.Patches{*patch1, *patch2} -} - -func mock_Reconciler_checkSomeCniExists_watchSource_succ(r *eciReconciler) []gomonkey.Patches { - patch1 := gomonkey.ApplyPrivateMethod(r, "listCalicoIPPools", func(_ *eciReconciler) (map[string]egressv1.IPListPair, error) { - return nil, nil - }) - patch2 := gomonkey.ApplyFuncReturn(watchSource, nil) - return []gomonkey.Patches{*patch1, *patch2} -} - -func mock_Reconciler_checkSomeCniExists_getK8sPodCidr_err(r *eciReconciler) []gomonkey.Patches { - patch1 := gomonkey.ApplyPrivateMethod(r, "listCalicoIPPools", func(_ *eciReconciler) (map[string]egressv1.IPListPair, error) { - return nil, ErrForMock - }) - patch2 := gomonkey.ApplyPrivateMethod(r, "getK8sPodCidr", func(_ *eciReconciler) (map[string]egressv1.IPListPair, error) { - return nil, ErrForMock - }) - return []gomonkey.Patches{*patch1, *patch2} -} - -func mock_Reconciler_checkSomeCniExists_getK8sPodCidr_succ(r *eciReconciler) []gomonkey.Patches { - patch1 := gomonkey.ApplyPrivateMethod(r, "listCalicoIPPools", func(_ *eciReconciler) (map[string]egressv1.IPListPair, error) { - return nil, ErrForMock - }) - patch2 := gomonkey.ApplyPrivateMethod(r, "getK8sPodCidr", func(_ *eciReconciler) (map[string]egressv1.IPListPair, error) { - return nil, nil - }) - return []gomonkey.Patches{*patch1, *patch2} -} diff --git a/pkg/controller/egress_cluster_info/tools_test.go b/pkg/controller/egress_cluster_info/tools_test.go deleted file mode 100644 index f96675c1c..000000000 --- a/pkg/controller/egress_cluster_info/tools_test.go +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2022 Authors of spidernet-io -// SPDX-License-Identifier: Apache-2.0 - -package egressclusterinfo - -import ( - "testing" - - "github.com/agiledragon/gomonkey/v2" - "github.com/stretchr/testify/assert" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -func Test_GetPodByLabel(t *testing.T) { - t.Run("failed List", func(t *testing.T) { - c := fake.NewFakeClient() - patch := gomonkey.ApplyMethodReturn(c, "List", ErrForMock) - defer patch.Reset() - - _, err := GetPodByLabel(c, map[string]string{"test": "GetPodByLabel"}) - assert.Error(t, err) - }) -} diff --git a/pkg/controller/endpoint/cluster_endpoint_slice.go b/pkg/controller/endpoint/cluster_endpoint_slice.go index eee894e41..f71a47489 100644 --- a/pkg/controller/endpoint/cluster_endpoint_slice.go +++ b/pkg/controller/endpoint/cluster_endpoint_slice.go @@ -308,27 +308,32 @@ func NewEgressClusterEpSliceController(mgr manager.Manager, log logr.Logger, cfg return err } - if err = c.Watch(source.Kind(mgr.GetCache(), &corev1.Pod{}), - handler.EnqueueRequestsFromMapFunc(enqueueEGCP(r.client)), podPredicate{}); err != nil { - return fmt.Errorf("failed to watch pod: %v", err) + podHandler := handler.TypedEnqueueRequestsFromMapFunc[*corev1.Pod] + sourcePod := source.Kind(mgr.GetCache(), &corev1.Pod{}, podHandler(enqueuePod(r.client)), podPredicate{}) + if err = c.Watch(sourcePod); err != nil { + return fmt.Errorf("failed to watch Pod: %w", err) } - if err = c.Watch(source.Kind(mgr.GetCache(), &corev1.Namespace{}), - handler.EnqueueRequestsFromMapFunc(enqueueNS(r.client)), nsPredicate{}); err != nil { - return fmt.Errorf("failed to watch namespace: %v", err) + nsHandler := handler.TypedEnqueueRequestsFromMapFunc[*corev1.Namespace] + sourceNamespace := source.Kind(mgr.GetCache(), &corev1.Namespace{}, nsHandler(enqueueNS(r.client)), nsPredicate{}) + if err = c.Watch(sourceNamespace); err != nil { + return fmt.Errorf("failed to watch Namespace: %v", err) } - if err = c.Watch(source.Kind(mgr.GetCache(), &v1beta1.EgressClusterPolicy{}), - &handler.EnqueueRequestForObject{}); err != nil { + sourceEgressClusterPolicy := source.Kind( + mgr.GetCache(), &v1beta1.EgressClusterPolicy{}, + &handler.TypedEnqueueRequestForObject[*v1beta1.EgressClusterPolicy]{}) + if err = c.Watch(sourceEgressClusterPolicy); err != nil { return fmt.Errorf("failed to watch EgressClusterPolicy: %v", err) } - opt := handler.OnlyControllerOwner() - eventHandler := handler.EnqueueRequestForOwner( - mgr.GetScheme(), mgr.GetRESTMapper(), &v1beta1.EgressClusterPolicy{}, opt, - ) - if err = c.Watch(source.Kind(mgr.GetCache(), &v1beta1.EgressClusterEndpointSlice{}), - eventHandler); err != nil { + sourceEgressClusterEndpointSlice := source.Kind(mgr.GetCache(), + &v1beta1.EgressClusterEndpointSlice{}, + handler.TypedEnqueueRequestForOwner[*v1beta1.EgressClusterEndpointSlice]( + mgr.GetScheme(), mgr.GetRESTMapper(), + &v1beta1.EgressClusterEndpointSlice{}, + handler.OnlyControllerOwner())) + if err = c.Watch(sourceEgressClusterEndpointSlice); err != nil { return fmt.Errorf("failed to watch EgressClusterEndpointSlice: %v", err) } @@ -338,43 +343,28 @@ func NewEgressClusterEpSliceController(mgr manager.Manager, log logr.Logger, cfg type nsPredicate struct { } -func (p nsPredicate) Create(_ event.CreateEvent) bool { +func (p nsPredicate) Create(_ event.TypedCreateEvent[*corev1.Namespace]) bool { return true } -func (p nsPredicate) Delete(_ event.DeleteEvent) bool { +func (p nsPredicate) Delete(_ event.TypedDeleteEvent[*corev1.Namespace]) bool { return true } -func (p nsPredicate) Update(updateEvent event.UpdateEvent) bool { - oldNS, ok := updateEvent.ObjectOld.(*corev1.Namespace) - if !ok { - return false - } - newNS, ok := updateEvent.ObjectNew.(*corev1.Namespace) - if !ok { - return false - } - +func (p nsPredicate) Update(updateEvent event.TypedUpdateEvent[*corev1.Namespace]) bool { // case by pods labels are changed - if reflect.DeepEqual(oldNS.Labels, newNS.Labels) { - return false + if updateEvent.ObjectOld != nil && updateEvent.ObjectNew != nil { + return !reflect.DeepEqual(updateEvent.ObjectOld.Labels, updateEvent.ObjectNew.Labels) } - return true } -func (p nsPredicate) Generic(_ event.GenericEvent) bool { +func (p nsPredicate) Generic(_ event.TypedGenericEvent[*corev1.Namespace]) bool { return true } -func enqueueNS(cli client.Client) handler.MapFunc { - return func(ctx context.Context, obj client.Object) []reconcile.Request { - ns, ok := obj.(*corev1.Namespace) - if !ok { - return nil - } - +func enqueueNS(cli client.Client) handler.TypedMapFunc[*corev1.Namespace] { + return func(ctx context.Context, ns *corev1.Namespace) []reconcile.Request { policyList := new(v1beta1.EgressClusterPolicyList) err := cli.List(ctx, policyList) if err != nil { diff --git a/pkg/controller/endpoint/cluster_endpoint_slice_test.go b/pkg/controller/endpoint/cluster_endpoint_slice_test.go index 15d4bc0e9..3cb40c6e3 100644 --- a/pkg/controller/endpoint/cluster_endpoint_slice_test.go +++ b/pkg/controller/endpoint/cluster_endpoint_slice_test.go @@ -34,15 +34,15 @@ import ( func TestNamespacePredicate(t *testing.T) { p := nsPredicate{} - if !p.Create(event.CreateEvent{}) { + if !p.Create(event.TypedCreateEvent[*corev1.Namespace]{}) { t.Fatal("got false") } - if !p.Delete(event.DeleteEvent{}) { + if !p.Delete(event.TypedDeleteEvent[*corev1.Namespace]{}) { t.Fatal("got false") } - if !p.Update(event.UpdateEvent{ + if !p.Update(event.TypedUpdateEvent[*corev1.Namespace]{ ObjectOld: &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: "default", @@ -922,19 +922,19 @@ func Test_listClusterEndpointSlices(t *testing.T) { func Test_Update(t *testing.T) { cases := map[string]struct { - in event.UpdateEvent + in event.TypedUpdateEvent[*corev1.Namespace] res bool }{ "ObjectOld not Namespace": { - in: event.UpdateEvent{}, + in: event.TypedUpdateEvent[*corev1.Namespace]{}, }, "ObjectNew not Namespace": { - in: event.UpdateEvent{ + in: event.TypedUpdateEvent[*corev1.Namespace]{ ObjectOld: &corev1.Namespace{}, }, }, "ObjectNew Namespace label equal": { - in: event.UpdateEvent{ + in: event.TypedUpdateEvent[*corev1.Namespace]{ ObjectOld: &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"foo": "bar"}, @@ -966,7 +966,7 @@ func Test_Update(t *testing.T) { func Test_Generic(t *testing.T) { t.Run("test Generic", func(t *testing.T) { p := nsPredicate{} - e := event.GenericEvent{} + e := event.TypedGenericEvent[*corev1.Namespace]{} res := p.Generic(e) assert.True(t, res) }) @@ -974,16 +974,11 @@ func Test_Generic(t *testing.T) { func Test_enqueueNS(t *testing.T) { cases := map[string]struct { - in client.Object + in *corev1.Namespace objs []client.Object patchFun func(c client.Client) []gomonkey.Patches expErr bool }{ - "failed not namespace obj": { - in: &corev1.Pod{}, - patchFun: mock_enqueueNS_List_err, - expErr: true, - }, "failed List": { in: &corev1.Namespace{}, patchFun: mock_enqueueNS_List_err, diff --git a/pkg/controller/endpoint/endpoint_slice.go b/pkg/controller/endpoint/endpoint_slice.go index 214995dc7..48bbe77fe 100644 --- a/pkg/controller/endpoint/endpoint_slice.go +++ b/pkg/controller/endpoint/endpoint_slice.go @@ -353,19 +353,27 @@ func NewEgressEndpointSliceController(mgr manager.Manager, log logr.Logger, cfg return err } - if err = c.Watch(source.Kind(mgr.GetCache(), &corev1.Pod{}), - handler.EnqueueRequestsFromMapFunc(enqueuePod(r.client)), podPredicate{}); err != nil { + h := handler.TypedEnqueueRequestsFromMapFunc[*corev1.Pod] + kindPod := source.Kind(mgr.GetCache(), &corev1.Pod{}, h(enqueuePod(r.client)), podPredicate{}) + if err = c.Watch(kindPod); err != nil { return fmt.Errorf("failed to watch Pod: %v", err) } - if err = c.Watch(source.Kind(mgr.GetCache(), &v1beta1.EgressPolicy{}), - &handler.EnqueueRequestForObject{}); err != nil { + kindEgressPolicy := source.Kind( + mgr.GetCache(), &v1beta1.EgressPolicy{}, + &handler.TypedEnqueueRequestForObject[*v1beta1.EgressPolicy]{}, + ) + if err = c.Watch(kindEgressPolicy); err != nil { return fmt.Errorf("failed to watch EgressPolicy: %v", err) } opt := handler.OnlyControllerOwner() - h := handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &v1beta1.EgressPolicy{}, opt) - if err = c.Watch(source.Kind(mgr.GetCache(), &v1beta1.EgressEndpointSlice{}), h); err != nil { + egressEndpointSliceHandler := handler.TypedEnqueueRequestForOwner[*v1beta1.EgressEndpointSlice] + kindEgressEndpointSlice := source.Kind( + mgr.GetCache(), &v1beta1.EgressEndpointSlice{}, + egressEndpointSliceHandler(mgr.GetScheme(), mgr.GetRESTMapper(), &v1beta1.EgressPolicy{}, opt), + ) + if err = c.Watch(kindEgressEndpointSlice); err != nil { return fmt.Errorf("failed to watch EgressEndpointSlice: %v", err) } @@ -375,31 +383,17 @@ func NewEgressEndpointSliceController(mgr manager.Manager, log logr.Logger, cfg type podPredicate struct { } -func (p podPredicate) Create(createEvent event.CreateEvent) bool { - pod, ok := createEvent.Object.(*corev1.Pod) - if !ok { - return false - } - if len(pod.Status.PodIPs) == 0 { - return false - } - return true +func (p podPredicate) Create(createEvent event.TypedCreateEvent[*corev1.Pod]) bool { + return len(createEvent.Object.Status.PodIPs) != 0 } -func (p podPredicate) Delete(_ event.DeleteEvent) bool { +func (p podPredicate) Delete(_ event.TypedDeleteEvent[*corev1.Pod]) bool { return true } -func (p podPredicate) Update(updateEvent event.UpdateEvent) bool { - oldPod, ok := updateEvent.ObjectOld.(*corev1.Pod) - if !ok { - return false - } - newPod, ok := updateEvent.ObjectNew.(*corev1.Pod) - if !ok { - return false - } - +func (p podPredicate) Update(updateEvent event.TypedUpdateEvent[*corev1.Pod]) bool { + oldPod := updateEvent.ObjectOld + newPod := updateEvent.ObjectNew // case by pods labels are changed if reflect.DeepEqual(oldPod.Labels, newPod.Labels) && reflect.DeepEqual(oldPod.Status.PodIPs, newPod.Status.PodIPs) && @@ -410,17 +404,12 @@ func (p podPredicate) Update(updateEvent event.UpdateEvent) bool { return true } -func (p podPredicate) Generic(_ event.GenericEvent) bool { +func (p podPredicate) Generic(_ event.TypedGenericEvent[*corev1.Pod]) bool { return true } -func enqueuePod(cli client.Client) handler.MapFunc { - return func(ctx context.Context, obj client.Object) []reconcile.Request { - pod, ok := obj.(*corev1.Pod) - if !ok { - return nil - } - +func enqueuePod(cli client.Client) func(ctx context.Context, pod *corev1.Pod) []reconcile.Request { + return func(ctx context.Context, pod *corev1.Pod) []reconcile.Request { policyList := new(v1beta1.EgressPolicyList) err := cli.List(ctx, policyList) if err != nil { diff --git a/pkg/controller/endpoint/endpoint_slice_test.go b/pkg/controller/endpoint/endpoint_slice_test.go index 29ca1d4fa..b06bf9ad5 100644 --- a/pkg/controller/endpoint/endpoint_slice_test.go +++ b/pkg/controller/endpoint/endpoint_slice_test.go @@ -506,7 +506,7 @@ func caseDeletePod() TestCaseEPS { func TestPodPredicate(t *testing.T) { p := podPredicate{} - if !p.Create(event.CreateEvent{ + if !p.Create(event.TypedCreateEvent[*corev1.Pod]{ Object: &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "default", @@ -525,11 +525,11 @@ func TestPodPredicate(t *testing.T) { t.Fatal("got false") } - if !p.Delete(event.DeleteEvent{}) { + if !p.Delete(event.TypedDeleteEvent[*corev1.Pod]{}) { t.Fatal("got false") } - if !p.Update(event.UpdateEvent{ + if !p.Update(event.TypedUpdateEvent[*corev1.Pod]{ ObjectOld: &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: "default", @@ -1035,14 +1035,14 @@ func Test_listEndpointSlices(t *testing.T) { func Test_podPredicate_Create(t *testing.T) { cases := map[string]struct { - in event.CreateEvent + in event.TypedCreateEvent[*corev1.Pod] res bool }{ "createEvent not pod": { - in: event.CreateEvent{}, + in: event.TypedCreateEvent[*corev1.Pod]{}, }, "pod no ip": { - in: event.CreateEvent{ + in: event.TypedCreateEvent[*corev1.Pod]{ Object: &corev1.Pod{}, }, }, @@ -1064,19 +1064,19 @@ func Test_podPredicate_Create(t *testing.T) { func Test_podPredicate_Update(t *testing.T) { cases := map[string]struct { - in event.UpdateEvent + in event.TypedUpdateEvent[*corev1.Pod] res bool }{ "ObjectOld not pod": { - in: event.UpdateEvent{}, + in: event.TypedUpdateEvent[*corev1.Pod]{}, }, "ObjectNew not pod": { - in: event.UpdateEvent{ + in: event.TypedUpdateEvent[*corev1.Pod]{ ObjectOld: &corev1.Pod{}, }, }, "nodeName not equal": { - in: event.UpdateEvent{ + in: event.TypedUpdateEvent[*corev1.Pod]{ ObjectOld: &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"foo": "bar"}, @@ -1114,22 +1114,18 @@ func Test_podPredicate_Update(t *testing.T) { func Test_podPredicate_Generic(t *testing.T) { t.Run("test Generic", func(t *testing.T) { p := podPredicate{} - e := event.GenericEvent{} + e := event.TypedGenericEvent[*corev1.Pod]{} res := p.Generic(e) assert.True(t, res) }) } func Test_enqueuePod(t *testing.T) { cases := map[string]struct { - in client.Object + in *corev1.Pod objs []client.Object patchFun func(c client.Client) []gomonkey.Patches expErr bool }{ - "failed not pod obj": { - in: &corev1.Namespace{}, - expErr: true, - }, "failed List": { in: &corev1.Pod{}, patchFun: mock_enqueuePod_List_err, diff --git a/pkg/controller/tunnel/egress_tunnel.go b/pkg/controller/tunnel/egress_tunnel.go index 7bb575c91..8ec22afcd 100644 --- a/pkg/controller/tunnel/egress_tunnel.go +++ b/pkg/controller/tunnel/egress_tunnel.go @@ -25,7 +25,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" "github.com/spidernet-io/egressgateway/pkg/config" egressv1 "github.com/spidernet-io/egressgateway/pkg/k8s/apis/v1beta1" @@ -697,14 +696,20 @@ func NewEgressTunnelController(mgr manager.Manager, log logr.Logger, cfg *config } log.Info("egresstunnel controller watch EgressTunnel") - if err := c.Watch(source.Kind(mgr.GetCache(), &egressv1.EgressTunnel{}), - handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressTunnel"))); err != nil { + + sourceEgressTunnel := utils.SourceKind(mgr.GetCache(), + &egressv1.EgressTunnel{}, + handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressTunnel"))) + if err := c.Watch(sourceEgressTunnel); err != nil { return fmt.Errorf("failed to watch EgressTunnel: %w", err) } log.Info("egresstunnel controller watch Node") - if err := c.Watch(source.Kind(mgr.GetCache(), &corev1.Node{}), - handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("Node"))); err != nil { + + sourceNode := utils.SourceKind(mgr.GetCache(), + &corev1.Node{}, + handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("Node"))) + if err := c.Watch(sourceNode); err != nil { return fmt.Errorf("failed to watch Node: %w", err) } diff --git a/pkg/egressgateway/egress_gateway.go b/pkg/egressgateway/egress_gateway.go index a48327c7f..aa8575f7d 100644 --- a/pkg/egressgateway/egress_gateway.go +++ b/pkg/egressgateway/egress_gateway.go @@ -6,12 +6,18 @@ package egressgateway import ( "context" "fmt" + "math" "math/rand" "net" "reflect" "time" "github.com/go-logr/logr" + "github.com/spidernet-io/egressgateway/pkg/config" + "github.com/spidernet-io/egressgateway/pkg/constant" + egress "github.com/spidernet-io/egressgateway/pkg/k8s/apis/v1beta1" + "github.com/spidernet-io/egressgateway/pkg/utils" + "github.com/spidernet-io/egressgateway/pkg/utils/ip" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -23,13 +29,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - - "github.com/spidernet-io/egressgateway/pkg/config" - "github.com/spidernet-io/egressgateway/pkg/constant" - egress "github.com/spidernet-io/egressgateway/pkg/k8s/apis/v1beta1" - "github.com/spidernet-io/egressgateway/pkg/utils" - "github.com/spidernet-io/egressgateway/pkg/utils/ip" ) type egnReconciler struct { @@ -943,9 +942,44 @@ func assignIP(from *egress.EgressGateway, req reconcile.Request, specEgressIP eg } } } + // case3 not spec eip in egw status + assignedIP := &AssignedIP{ + Node: "", + IPv4: specEgressIP.IPv4, + IPv6: specEgressIP.IPv6, + UseNodeIP: false, + } + // + bestNodeIndex := -1 + eipNumMin := math.MaxInt8 + // find best node + for i, node := range from.Status.NodeList { + if node.Status != string(egress.EgressTunnelReady) { + continue + } + if len(node.Eips) < eipNumMin { + eipNumMin = len(node.Eips) + bestNodeIndex = i + } + } + if bestNodeIndex != -1 { + from.Status.NodeList[bestNodeIndex].Eips = append( + from.Status.NodeList[bestNodeIndex].Eips, + egress.Eips{ + IPv4: specEgressIP.IPv4, + IPv6: specEgressIP.IPv6, + Policies: []egress.Policy{{Name: req.Name, Namespace: req.Namespace}}, + }, + ) + assignedIP.Node = from.Status.NodeList[bestNodeIndex].Name + } + if assignedIP.Node == "" { + return nil, fmt.Errorf("EgressGateway %s does not have an available Node", from.Name) + } + return assignedIP, nil } - // case3 assign new IP use eip assign policy + // case4 assign new IP use eip assign policy // if specEgressIP.AllocatorPolicy == egress.EipAllocatorRR { randObj := rand.New(rand.NewSource(time.Now().UnixNano())) @@ -1039,6 +1073,7 @@ func assignIP(from *egress.EgressGateway, req reconcile.Request, specEgressIP eg ) return assignedIP, nil } else { + // case 5: use default eip assignedIP := &AssignedIP{ Node: "", IPv4: from.Spec.Ippools.Ipv4DefaultEIP, @@ -1249,28 +1284,44 @@ func NewEgressGatewayController(mgr manager.Manager, log logr.Logger, cfg *confi return err } - if err = c.Watch(source.Kind(mgr.GetCache(), &egress.EgressGateway{}), - handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressGateway")), egressGatewayPredicate{}); err != nil { + sourceEgressGateway := utils.SourceKind(mgr.GetCache(), + &egress.EgressGateway{}, + handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressGateway")), + egressTunnelPredicate{}) + err = c.Watch(sourceEgressGateway) + if err != nil { return fmt.Errorf("failed to watch EgressGateway: %w", err) } - if err = c.Watch(source.Kind(mgr.GetCache(), &corev1.Node{}), - handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("Node")), nodePredicate{}); err != nil { + sourceNode := utils.SourceKind(mgr.GetCache(), + &corev1.Node{}, + handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("Node")), + nodePredicate{}) + if err = c.Watch(sourceNode); err != nil { return fmt.Errorf("failed to watch Node: %w", err) } - if err = c.Watch(source.Kind(mgr.GetCache(), &egress.EgressPolicy{}), - handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressPolicy")), egressPolicyPredicate{}); err != nil { + sourceEgressPolicy := utils.SourceKind(mgr.GetCache(), + &egress.EgressPolicy{}, + handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressPolicy")), + egressPolicyPredicate{}) + if err = c.Watch(sourceEgressPolicy); err != nil { return fmt.Errorf("failed to watch EgressPolicy: %w", err) } - if err = c.Watch(source.Kind(mgr.GetCache(), &egress.EgressClusterPolicy{}), - handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressClusterPolicy")), egressClusterPolicyPredicate{}); err != nil { + sourceEgressClusterPolicy := utils.SourceKind(mgr.GetCache(), + &egress.EgressClusterPolicy{}, + handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressClusterPolicy")), + egressClusterPolicyPredicate{}) + if err = c.Watch(sourceEgressClusterPolicy); err != nil { return fmt.Errorf("failed to watch EgressClusterPolicy: %w", err) } - if err = c.Watch(source.Kind(mgr.GetCache(), &egress.EgressTunnel{}), - handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressTunnel")), egressTunnelPredicate{}); err != nil { + sourceEgressTunnel := utils.SourceKind(mgr.GetCache(), + &egress.EgressTunnel{}, + handler.EnqueueRequestsFromMapFunc(utils.KindToMapFlat("EgressTunnel")), + egressTunnelPredicate{}) + if err = c.Watch(sourceEgressTunnel); err != nil { return fmt.Errorf("failed to watch EgressTunnel: %w", err) } @@ -1402,26 +1453,28 @@ func (p egressClusterPolicyPredicate) Generic(_ event.GenericEvent) bool { retur type egressGatewayPredicate struct{} -func (p egressGatewayPredicate) Create(_ event.CreateEvent) bool { return true } -func (p egressGatewayPredicate) Delete(_ event.DeleteEvent) bool { return true } -func (p egressGatewayPredicate) Update(updateEvent event.UpdateEvent) bool { - oldObj, ok := updateEvent.ObjectOld.(*egress.EgressGateway) - if !ok { - return false - } - newObj, ok := updateEvent.ObjectNew.(*egress.EgressGateway) - if !ok { - return false - } - if !reflect.DeepEqual(oldObj.ObjectMeta, newObj.ObjectMeta) { +func (p egressGatewayPredicate) Create(_ event.TypedCreateEvent[*egress.EgressGateway]) bool { + return true +} +func (p egressGatewayPredicate) Delete(_ event.TypedDeleteEvent[*egress.EgressGateway]) bool { + return true +} +func (p egressGatewayPredicate) Update(updateEvent event.TypedUpdateEvent[*egress.EgressGateway]) bool { + if !reflect.DeepEqual( + updateEvent.ObjectOld.ObjectMeta, + updateEvent.ObjectNew.ObjectMeta) { return true } - if !reflect.DeepEqual(oldObj.Spec, newObj.Spec) { + if !reflect.DeepEqual( + updateEvent.ObjectOld.Spec, + updateEvent.ObjectNew.Spec) { return true } return false } -func (p egressGatewayPredicate) Generic(_ event.GenericEvent) bool { return true } +func (p egressGatewayPredicate) Generic(_ event.TypedGenericEvent[*egress.EgressGateway]) bool { + return true +} type nodePredicate struct{} diff --git a/pkg/utils/flat.go b/pkg/utils/flat.go index e2d75e5a4..511c7e13a 100644 --- a/pkg/utils/flat.go +++ b/pkg/utils/flat.go @@ -10,9 +10,12 @@ import ( "strings" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" ) var ErrInvalidRequest = errors.New("error invalid request") @@ -48,3 +51,7 @@ func ParseKindWithReq(req reconcile.Request) (string, reconcile.Request, error) }, }, nil } + +func SourceKind(cache cache.Cache, obj client.Object, h handler.EventHandler, predicates ...predicate.Predicate) source.Source { + return source.Kind(cache, obj, h, predicates...) +} diff --git a/pkg/utils/list.go b/pkg/utils/list.go new file mode 100644 index 000000000..261145eba --- /dev/null +++ b/pkg/utils/list.go @@ -0,0 +1,16 @@ +// Copyright 2022 Authors of spidernet-io +// SPDX-License-Identifier: Apache-2.0 + +package utils + +func EqualStringSlice(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i, v := range a { + if v != b[i] { + return false + } + } + return true +} diff --git a/pkg/utils/list_test.go b/pkg/utils/list_test.go new file mode 100644 index 000000000..6d99836c3 --- /dev/null +++ b/pkg/utils/list_test.go @@ -0,0 +1,30 @@ +// Copyright 2022 Authors of spidernet-io +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import "testing" + +func TestEqualStringSlice(t *testing.T) { + cases := []struct { + name string + a []string + b []string + expect bool + }{ + {"equal slices", []string{"a", "b", "c"}, []string{"a", "b", "c"}, true}, + {"different lengths", []string{"a", "b", "c"}, []string{"a", "b"}, false}, + {"different elements", []string{"a", "b", "c"}, []string{"a", "b", "d"}, false}, + {"empty slices", []string{}, []string{}, true}, + {"one empty slice", []string{"a"}, []string{}, false}, + } + + for _, v := range cases { + t.Run(v.name, func(t *testing.T) { + out := EqualStringSlice(v.a, v.b) + if out != v.expect { + t.Errorf("EqualStringSlice() got = %v, want = %v", out, v.expect) + } + }) + } +}