From bf30c3e698b736bad2090c469fb26a41b5351700 Mon Sep 17 00:00:00 2001 From: lou-lan Date: Wed, 7 Aug 2024 17:39:38 +0800 Subject: [PATCH] Fix get vxlan parent IP logic to skip IP addresses with a full subnet mask (e.g., /32 for IPv4 and /128 for IPv6). Signed-off-by: lou-lan --- docs/usage/AwsWithCilium.en.md | 20 +++++---- docs/usage/AwsWithCilium.zh.md | 23 ++++++----- pkg/agent/police.go | 49 +++++++++++++++-------- pkg/agent/vxlan/parent.go | 4 ++ pkg/controller/clusterinfo/clusterinfo.go | 10 +++-- pkg/controller/clusterinfo/k8sservice.go | 34 +++++++++++----- 6 files changed, 94 insertions(+), 46 deletions(-) diff --git a/docs/usage/AwsWithCilium.en.md b/docs/usage/AwsWithCilium.en.md index b1d6f6afd..42180c3d2 100644 --- a/docs/usage/AwsWithCilium.en.md +++ b/docs/usage/AwsWithCilium.en.md @@ -1,4 +1,4 @@ -# Using EgressGateway with AWS Cilium CNI +# EgressGateway with Cilium CNI on AWS ## Introduction @@ -6,7 +6,7 @@ This article introduces the use of EgressGateway in a Cilium CNI networking envi Compared to Cilium's Egress feature, EgressGateway supports HA. If you don't need HA, consider using Cilium's Egress feature first. -The following sections will guide you step-by-step to install EgressGateway, create a sample pod, and configure an egress policy for the pod to access the internet via the gateway node. +The following sections will guide you step-by-step to install EgressGateway, create a sample Pod, and configure an EgressPolicy for the Pod to access the internet via the gateway node. ## Create Cluster and Install Cilium @@ -22,12 +22,16 @@ Using curl, you should see a response that includes your node's public IP. ## Install EgressGateway -Add the helm repository and install EgressGateway. We enable IPv4 with `feature.enableIPv4=true` and disable IPv6 with `feature.enableIPv6=false`. We also specify to exclude the cluster's CIDR from the gateway with `feature.clusterCIDR.extraCidr[0]=172.16.0.0/16`. +Add and update the Helm repository to install egressgateway from the specified source. ```shell helm repo add egressgateway https://spidernet-io.github.io/egressgateway/ helm repo update +``` + +We enable IPv4 with `feature.enableIPv4=true` and disable IPv6 with `feature.enableIPv6=false`. We can optionally specify `feature.clusterCIDR.extraCidr` the internal CIDR of the cluster during installation, which will modify the behavior of the `EgressPolicy`. If you create an `EgressPolicy` CR and do not specify `spec.destSubnet`, the EgressGateway will forward all traffic from the Pod, except for the internal CIDR, to the gateway node. Conversely, if `spec.destSubnet` is specified, the EgressGateway will only forward the designated traffic to the gateway node. +```shell helm install egress --wait \ --debug egressgateway/egressgateway \ --set feature.enableIPv4=true \ @@ -41,7 +45,7 @@ List the current nodes. ```shell ~ kubectl get nodes -A -owide -NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP +NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP ip-172-16-103-117.ec2.internal Ready 25m v1.30.0-eks-036c24b 172.16.103.117 34.239.162.85 ip-172-16-61-234.ec2.internal Ready 25m v1.30.0-eks-036c24b 172.16.61.234 54.147.15.230 ip-172-16-62-200.ec2.internal Ready 25m v1.30.0-eks-036c24b 172.16.62.200 54.147.16.130 @@ -50,11 +54,11 @@ ip-172-16-62-200.ec2.internal Ready 25m v1.30.0-eks-036c24b 1 We select `ip-172-16-103-117.ec2.internal` and `ip-172-16-62-200.ec2.internal` as gateway nodes. Label the nodes with `egress=true`. ```shell -kubectl label node ip-172-16-103-117.ec2.internal egress=true -kubectl label node ip-172-16-62-200.ec2.internal egress=true +kubectl label node ip-172-16-103-117.ec2.internal role=gateway +kubectl label node ip-172-16-62-200.ec2.internal role=gateway ``` -Create the EgressGateway CR, using `egress: "true"` to select nodes as exit gateways. +Create the EgressGateway CR, using `role: gateway` to select nodes as exit gateways. ```yaml apiVersion: egressgateway.spidernet.io/v1beta1 @@ -65,7 +69,7 @@ spec: nodeSelector: selector: matchLabels: - egress: "true" + role: gateway ``` ## Create a Test Pod diff --git a/docs/usage/AwsWithCilium.zh.md b/docs/usage/AwsWithCilium.zh.md index 380f63217..77af136ba 100644 --- a/docs/usage/AwsWithCilium.zh.md +++ b/docs/usage/AwsWithCilium.zh.md @@ -23,18 +23,21 @@ curl ipinfo.io ## 安装 EgressGateway -添加 helm 仓库，并安装 EgressGateway。我们通过 `feature.enableIPv4=true` 来启用 IPv4，`feature.enableIPv6=false` 来禁用 IPv6。 -我们通过 `feature.clusterCIDR.extraCidr[0]=172.16.0.0/16` 来指定排除集群的 CIDR 走到网关。 +添加和更新 Helm 仓库以从指定来源安装 EgressGateway。 ```shell helm repo add egressgateway https://spidernet-io.github.io/egressgateway/ helm repo update +``` + +我们 `feature.enableIPv4=true` 启用 IPv4 ，通过 `feature.enableIPv6=false` 禁用 IPv6。在安装过程中，我们可以通过 ``feature.clusterCIDR.extraCidr`` 集群的内部 CIDR，这将修改 `EgressPolicy` 的行为。如果您创建一个 `EgressPolicy` CR 并且没有指定 `spec.destSubnet`，EgressGateway 将把 Pod 的所有访问外部的流量（内部 CIDR 除外）转发到网关节点。相反，如果指定了 `spec.destSubnet`，EgressGateway 将仅将指定的流量转发到网关节点。 +```shell helm install egress --wait \ - --debug egressgateway/egressgateway \ - --set feature.enableIPv4=true \ - --set feature.enableIPv6=false \ - --set feature.clusterCIDR.extraCidr[0]=172.16.0.0/16 + --debug egressgateway/egressgateway \ + --set feature.enableIPv4=true \ + --set feature.enableIPv6=false \ + --set feature.clusterCIDR.extraCidr[0]=172.16.0.0/16 ``` ## 创建 EgressGateway CR @@ -52,11 +55,11 @@ ip-172-16-62-200.ec2.internal Ready 25m v1.30.0-eks-036c24b 1 我们选择 `ip-172-16-103-117.ec2.internal` 和 `ip-172-16-62-200.ec2.internal` 作为网关节点。给节点设置 `egress=true` 标签。 ```shell -kubectl label node ip-172-16-103-117.ec2.internal egress=true -kubectl label node ip-172-16-62-200.ec2.internal egress=true +kubectl label node ip-172-16-103-117.ec2.internal role=gateway +kubectl label node ip-172-16-62-200.ec2.internal role=gateway ``` -创建 EgressGateway CR，我们通过 `egress: "true"` 来选择节点作为出口网关。 +创建 EgressGateway CR，我们通过 `role: gateway` 来选择节点作为出口网关。 ```yaml apiVersion: egressgateway.spidernet.io/v1beta1 @@ -67,7 +70,7 @@ spec: nodeSelector: selector: matchLabels: - egress: "true" + role: gateway ``` ## 创建测试 Pod diff --git a/pkg/agent/police.go b/pkg/agent/police.go index 7e975fea4..2d4abe8d9 100644 --- a/pkg/agent/police.go +++ b/pkg/agent/police.go @@ -37,6 +37,9 @@ import ( const ( EgressClusterCIDRIPv4 = "egress-cluster-cidr-ipv4" EgressClusterCIDRIPv6 = "egress-cluster-cidr-ipv6" + + Mark = 0xff000000 + Mask = 0xffffffff ) type policeReconciler struct { @@ -257,7 +260,7 @@ func (r *policeReconciler) initApplyPolicy() error { } restore := iptables.Rule{ Match: iptables.MatchCriteria{}.CTDirectionOriginal(iptables.DirectionReply), - Action: iptables.RestoreConnMarkAction{RestoreMask: 0xffffffff}, + Action: iptables.RestoreConnMarkAction{RestoreMask: Mask}, Comment: []string{ "label for restoring connections, rule is from the EgressGateway", }, @@ -554,7 +557,7 @@ func (r *policeReconciler) buildPolicyRule(policyName string, mark uint32, versi CTDirectionOriginal(iptables.DirectionOriginal) } - action := iptables.SetMaskedMarkAction{Mark: mark, Mask: 0xffffffff} + action := iptables.SetMaskedMarkAction{Mark: mark, Mask: Mask} rule := &iptables.Rule{Match: matchCriteria, Action: action, Comment: []string{ fmt.Sprintf("Set mark for EgressPolicy %s", policyName), }} @@ -571,13 +574,21 @@ func buildNatStaticRule(base uint32) map[string][]iptables.Rule { }, }, { - Match: iptables.MatchCriteria{}.MarkMatchesWithMask(base, 0xffffffff), + Match: iptables.MatchCriteria{}.MarkMatchesWithMask(base, Mask), Action: iptables.AcceptAction{}, Comment: []string{ "Accept for egress traffic from pod going to EgressTunnel", }, }, - }} + }, + "PREROUTING": { + { + Match: iptables.MatchCriteria{}.MarkMatchesWithMask(base, Mark), + Action: iptables.AcceptAction{}, + Comment: []string{"EgressGateway traffic accept datapath rule"}, + }, + }, + } return res } @@ -764,14 +775,14 @@ func (r *policeReconciler) reconcileTunnel(ctx context.Context, req reconcile.Re func buildFilterStaticRule(base uint32) map[string][]iptables.Rule { res := map[string][]iptables.Rule{ "FORWARD": {{ - Match: iptables.MatchCriteria{}.MarkMatchesWithMask(base, 0xffffffff), + Match: iptables.MatchCriteria{}.MarkMatchesWithMask(base, Mask), Action: iptables.AcceptAction{}, Comment: []string{ "Accept for egress traffic from pod going to EgressTunnel", }, }}, "OUTPUT": {{ - Match: iptables.MatchCriteria{}.MarkMatchesWithMask(base, 0xffffffff), + Match: iptables.MatchCriteria{}.MarkMatchesWithMask(base, Mask), Action: iptables.AcceptAction{}, Comment: []string{ "Accept for egress traffic from pod going to EgressTunnel", @@ -787,8 +798,8 @@ func buildMangleStaticRule(base uint32, forward := []iptables.Rule{ { - Match: iptables.MatchCriteria{}.MarkMatchesWithMask(base, 0xff000000), - Action: iptables.SetMaskedMarkAction{Mark: base, Mask: 0xffffffff}, + Match: iptables.MatchCriteria{}.MarkMatchesWithMask(base, Mark), + Action: iptables.SetMaskedMarkAction{Mark: base, Mask: Mask}, Comment: []string{ "Accept for egress traffic from pod going to EgressTunnel", }, @@ -796,7 +807,7 @@ func buildMangleStaticRule(base uint32, } postrouting := []iptables.Rule{{ - Match: iptables.MatchCriteria{}.MarkMatchesWithMask(base, 0xffffffff), + Match: iptables.MatchCriteria{}.MarkMatchesWithMask(base, Mask), Action: iptables.AcceptAction{}, Comment: []string{ "Accept for egress traffic from pod going to EgressTunnel", @@ -819,17 +830,23 @@ func buildMangleStaticRule(base uint32, Comment: []string{"EgressGateway reply datapath rule, rule is from the EgressGateway"}, }) prerouting = append(prerouting, iptables.Rule{ - Match: iptables.MatchCriteria{}.MarkMatchesWithMask(base, 0xff000000), + Match: iptables.MatchCriteria{}.MarkMatchesWithMask(base, Mark), Action: iptables.AcceptAction{}, Comment: []string{"EgressGateway reply datapath rule, rule is from the EgressGateway"}, }) postrouting = append(postrouting, iptables.Rule{ - Match: iptables.MatchCriteria{}.MarkMatchesWithMask(replyMark, 0xffffffff), - Action: iptables.SetMaskedMarkAction{Mark: 0x00000000, Mask: 0xffffffff}, + Match: iptables.MatchCriteria{}.MarkMatchesWithMask(replyMark, Mask), + Action: iptables.SetMaskedMarkAction{Mark: 0x00000000, Mask: Mask}, Comment: []string{ "clear the Mark of the inner package, rule is from the EgressGateway", }, }) + } else { + prerouting = append(prerouting, iptables.Rule{ + Match: iptables.MatchCriteria{}.MarkMatchesWithMask(base, Mark), + Action: iptables.AcceptAction{}, + Comment: []string{"EgressGateway traffic accept datapath rule"}, + }) } res := map[string][]iptables.Rule{ @@ -848,21 +865,21 @@ func buildPreroutingReplyRouting(vxlanName string, base uint32, replyMark string return []iptables.Rule{ { Match: iptables.MatchCriteria{}.InInterface(vxlanName).SrcMacSource(mac).CTDirectionOriginal(iptables.DirectionOriginal), - Action: iptables.SetMaskedMarkAction{Mark: mark, Mask: 0xffffffff}, + Action: iptables.SetMaskedMarkAction{Mark: mark, Mask: Mask}, Comment: []string{ "Mark the traffic from the EgressGateway tunnel, rule is from the EgressGateway", }, }, { - Match: iptables.MatchCriteria{}.MarkMatchesWithMask(mark, 0xffffffff), - Action: iptables.SaveConnMarkAction{SaveMask: 0xffffffff}, + Match: iptables.MatchCriteria{}.MarkMatchesWithMask(mark, Mask), + Action: iptables.SaveConnMarkAction{SaveMask: Mask}, Comment: []string{ "Save mark to the connection, rule is from the EgressGateway", }, }, { Match: iptables.MatchCriteria{}.InInterface(vxlanName).SrcMacSource(mac), - Action: iptables.SetMaskedMarkAction{Mark: base, Mask: 0xffffffff}, + Action: iptables.SetMaskedMarkAction{Mark: base, Mask: Mask}, Comment: []string{ "Clear Mark of the inner package, rule is from the EgressGateway", }, diff --git a/pkg/agent/vxlan/parent.go b/pkg/agent/vxlan/parent.go index d85bcdae3..9ea623cd8 100644 --- a/pkg/agent/vxlan/parent.go +++ b/pkg/agent/vxlan/parent.go @@ -62,6 +62,10 @@ func GetParentByDefaultRoute(cli NetLink) func(version int) (*Parent, error) { if !addr.IP.IsGlobalUnicast() { continue } + ones, bits := addr.Mask.Size() + if ones == 32 && bits == 32 || ones == 128 && bits == 128 { + continue + } return &Parent{Name: link.Attrs().Name, IP: addr.IP, Index: link.Attrs().Index}, nil } return nil, fmt.Errorf("failed to find parent interface") diff --git a/pkg/controller/clusterinfo/clusterinfo.go b/pkg/controller/clusterinfo/clusterinfo.go index bb82d4e57..5f4aa2250 100644 --- a/pkg/controller/clusterinfo/clusterinfo.go +++ b/pkg/controller/clusterinfo/clusterinfo.go @@ -13,6 +13,7 @@ import ( calicov1 "github.com/tigera/operator/pkg/apis/crd.projectcalico.org/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" @@ -65,9 +66,12 @@ func NewController(mgr manager.Manager, log logr.Logger, cfg *config.Config) err for { err = r.cli.List(context.Background(), &calicov1.IPPoolList{}) if err != nil { - log.Error(err, "failed to list CalicoIPPool", "error", err) - time.Sleep(time.Second * 3) - continue + if meta.IsNoMatchError(err) { + log.Info("not found CalicoIPPool CRD in current cluster, skipping watch") + } else { + log.Error(err, "failed to list Calico IPPool, skipping watch.", "error", err) + } + return } sourceCalicoIPPool := utils.SourceKind(r.mgr.GetCache(), &calicov1.IPPool{}, diff --git a/pkg/controller/clusterinfo/k8sservice.go b/pkg/controller/clusterinfo/k8sservice.go index abd7ba23e..13bed369f 100644 --- a/pkg/controller/clusterinfo/k8sservice.go +++ b/pkg/controller/clusterinfo/k8sservice.go @@ -6,20 +6,39 @@ package clusterinfo import ( "context" "fmt" - "github.com/spidernet-io/egressgateway/pkg/utils/ip" + corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" "strings" + + "github.com/spidernet-io/egressgateway/pkg/utils/ip" ) -var kubeControllerManagerPodLabel = map[string]string{"component": "kube-controller-manager"} +// kubeControllerManagerPodLabelList Store the labels for the Kubernetes Controller Manager. +// We use these labels to retrieve the Kubernetes Controller Manager Pod arguments in order +// to obtain the cluster CIDR. Since different clusters have different Kubernetes Controller +// Manager labels, the content of the labels also varies. +var kubeControllerManagerPodLabelList = []map[string]string{ + { + "component": "kube-controller-manager", + }, + { + "k8s-app": "kube-controller-manager", + }, +} func GetClusterCIDR(ctx context.Context, cli client.Client) (ipv4, ipv6 []string, err error) { - pods, err := listPodByLabel(ctx, cli, kubeControllerManagerPodLabel) - if err != nil { - return nil, nil, err + for _, item := range kubeControllerManagerPodLabelList { + pods, err := listPodByLabel(ctx, cli, item) + if err != nil { + return nil, nil, err + } + if len(pods) < 1 { + continue + } + return parseCIDRFromControllerManager(&pods[0], "--service-cluster-ip-range=") } - return parseCIDRFromControllerManager(&pods[0], "--service-cluster-ip-range=") + return nil, nil, nil } func listPodByLabel(ctx context.Context, cli client.Client, @@ -31,9 +50,6 @@ func listPodByLabel(ctx context.Context, cli client.Client, return nil, err } pods := podList.Items - if len(pods) == 0 { - return nil, fmt.Errorf("failed to get pod") - } return pods, nil }