Fix the default egress gateway problem (#1045)

Signed-off-by: bzsuni <[email protected]>

.github/workflows/auto-pr-ci.yaml

@@ -147,9 +147,11 @@ jobs:
               echo "RUN_UNITEST_ENABLED=false" >> $GITHUB_ENV
           else
               # call by auto-nightly-ci, the event is schedule or its workflow_dispatch
+              if ${{ github.event_name != 'workflow_dispatch' }}; then
+                echo "RUN_TAG=main" >> $GITHUB_ENV
+              fi
               # use main sha for ci image tag
               echo "trigger by workflow_call"
-              echo "RUN_TAG=main" >> $GITHUB_ENV
               echo "RUN_E2E_LABEL=" >> $GITHUB_ENV
               echo "RUN_E2E_ENABLED=true" >> $GITHUB_ENV
               echo "RUN_UNITEST_ENABLED=true" >> $GITHUB_ENV

Makefile.defs

@@ -182,7 +182,7 @@ E2E_KIND_IPV6_POD_CIDR = fd40::/48
 
 #====== calico
 CALICO_VERSION ?=
-MAMUAL_CALICO_VERSION ?= v3.26.4
+DEFAULT_CALICO_VERSION ?= v3.26.4
 INSTALL_TIME_OUT = 300s
 ifeq ($(E2E_CHINA_IMAGE_REGISTRY),true)
     CALICO_REGISTRY ?= docker.m.daocloud.io

test/Makefile

@@ -93,6 +93,7 @@ install_calico: KIND_CLUSTER_NAME ?= $(E2E_KIND_CLUSTER_NAME)
 install_calico:
 	echo "install calico"
 	export CALICO_VERSION=$(CALICO_VERSION); \
+	export DEFAULT_CALICO_VERSION=$(DEFAULT_CALICO_VERSION); \
 	export HTTP_PROXY=$(HTTP_PROXY); \
 	export CALICO_REGISTRY=$(CALICO_REGISTRY); \
 	export INSTALL_TIME_OUT=$(INSTALL_TIME_OUT); \

test/e2e/common/egw.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/google/uuid"
 	"k8s.io/apimachinery/pkg/api/errors"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -251,3 +252,40 @@ func CheckEgressGatewayStatusSynced(ctx context.Context, cli client.Client, egw
 		}
 	}
 }
+
+func DeleteEgressGateway(ctx context.Context, cli client.Client, egw *egressv1.EgressGateway, timeout time.Duration) error {
+	ctx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	err := DeleteObj(ctx, cli, egw)
+	if err != nil {
+		return err
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+			return e2eerr.ErrTimeout
+		default:
+			err = cli.Get(ctx, types.NamespacedName{Name: egw.Name}, egw)
+			if apierrors.IsNotFound(err) {
+				return nil
+			}
+			time.Sleep(time.Second / 2)
+		}
+	}
+}
+
+func GetClusterDefualtEgressGateway(ctx context.Context, cli client.Client) (*egressv1.EgressGateway, error) {
+	egwList := new(egressv1.EgressGatewayList)
+	err := cli.List(ctx, egwList)
+	if err != nil {
+		return nil, err
+	}
+	for _, v := range egwList.Items {
+		if v.Spec.ClusterDefault {
+			return &v, nil
+		}
+	}
+	return nil, nil
+}

test/e2e/constant/constant.go

@@ -0,0 +1,6 @@
+// Copyright 2022 Authors of spidernet-io
+// SPDX-License-Identifier: Apache-2.0
+
+package constant
+
+var WebhookMsgClusterDefaultGateway = "A cluster can only have one default gateway"

test/e2e/egressgateway/default_egressgateway_test.go

@@ -5,6 +5,7 @@ package egressgateway_test
 
 import (
 	"context"
+	"strings"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -16,6 +17,7 @@ import (
 
 	egressv1 "github.com/spidernet-io/egressgateway/pkg/k8s/apis/v1beta1"
 	"github.com/spidernet-io/egressgateway/test/e2e/common"
+	e2econstant "github.com/spidernet-io/egressgateway/test/e2e/constant"
 )
 
 var _ = Describe("Test default egress gateway", Label("DefaultEgressGateway", "G00017"), Ordered, func() {
@@ -88,7 +90,7 @@ var _ = Describe("Test default egress gateway", Label("DefaultEgressGateway", "G
 			err = common.DeleteObj(ctx, cli, nsDefaultEgw)
 			Expect(err).NotTo(HaveOccurred())
 
-			err = common.DeleteObj(ctx, cli, clusterDefaultEgw)
+			err = common.DeleteEgressGateway(ctx, cli, clusterDefaultEgw, time.Second*3)
 			Expect(err).NotTo(HaveOccurred())
 
 			ns := &corev1.Namespace{}
@@ -108,6 +110,9 @@ var _ = Describe("Test default egress gateway", Label("DefaultEgressGateway", "G
 		// create global egress gateway
 		ctx := context.Background()
 		err := cli.Create(ctx, clusterDefaultEgw)
+		if err != nil && strings.Contains(err.Error(), e2econstant.WebhookMsgClusterDefaultGateway) {
+			Skip("The default egressgateway already exists, skip this test case")
+		}
 		Expect(err).NotTo(HaveOccurred())
 
 		// create egress policy1

test/e2e/egresspolicy/egresspolicy_test.go

@@ -6,6 +6,7 @@ package egresspolicy_test
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -22,6 +23,7 @@ import (
 
 	egressv1 "github.com/spidernet-io/egressgateway/pkg/k8s/apis/v1beta1"
 	"github.com/spidernet-io/egressgateway/test/e2e/common"
+	e2econstant "github.com/spidernet-io/egressgateway/test/e2e/constant"
 )
 
 var _ = Describe("EgressPolicy", Serial, func() {
@@ -828,6 +830,10 @@ var _ = Describe("EgressPolicy", Serial, func() {
 				egw.Spec.ClusterDefault = true
 			})
 
+			if err != nil && strings.Contains(err.Error(), e2econstant.WebhookMsgClusterDefaultGateway) {
+				Skip("The default egressgateway already exists, skip this test case")
+			}
+
 			Expect(err).NotTo(HaveOccurred())
 
 			DeferCleanup(func() {

test/scripts/installCalico.sh

@@ -26,7 +26,7 @@ CALICO_NODE=${DEST_CALICO_YAML_DIR}/calico_node.yaml
 if [ -z "${CALICO_VERSION}" ]; then
   [ -n "${HTTP_PROXY}" ] && { calico_info_json=$(curl --retry 3 --retry-delay 5 -x "${HTTP_PROXY}" -s https://api.github.com/repos/projectcalico/calico/releases/latest); echo ${calico_info_json}; calico_tag=$(echo ${calico_info_json} | jq -r '.tag_name'); }
   [ -z "${HTTP_PROXY}" ] && { calico_info_json=$(curl --retry 3 --retry-delay 5 -s https://api.github.com/repos/projectcalico/calico/releases/latest); echo ${calico_info_json}; calico_tag=$(echo ${calico_info_json} | jq -r '.tag_name'); }
-  [ "${calico_tag}" == "null" ] && { echo "failed to get the calico version, specify the version manually"; calico_tag=${MANUAL_CALICO_VERSION}; }
+  [ "${calico_tag}" == "null" ] && { echo "failed to get the calico version, specify the version manually"; calico_tag=${DEFAULT_CALICO_VERSION}; }
 else
   calico_tag=${CALICO_VERSION}
 fi