From 8edd18e1e7c680d9583c851230e25e1c817cb34b Mon Sep 17 00:00:00 2001 From: Surya Seetharaman Date: Fri, 24 Jan 2025 13:02:27 +0100 Subject: [PATCH] Backport UDN tests into 4.18 Signed-off-by: Surya Seetharaman --- test/extended/networking/egressip_helpers.go | 106 +- test/extended/networking/livemigration.go | 220 +++- test/extended/networking/multinetpolicy.go | 24 +- .../networking/network_segmentation.go | 1142 +++++++++++++++-- ...twork_segmentation_endpointslice_mirror.go | 40 +- .../networking/network_segmentation_policy.go | 340 +++++ 6 files changed, 1628 insertions(+), 244 deletions(-) create mode 100644 test/extended/networking/network_segmentation_policy.go diff --git a/test/extended/networking/egressip_helpers.go b/test/extended/networking/egressip_helpers.go index 5e001e5f9b12..e6da27d8c9f5 100644 --- a/test/extended/networking/egressip_helpers.go +++ b/test/extended/networking/egressip_helpers.go @@ -255,8 +255,68 @@ func findDefaultInterfaceForOpenShiftSDN(oc *exutil.CLI, nodeName string) (strin return defaultRoutes[0].Dev, nil } +type ovnKubePodInfo struct { + podName string + containerName string +} + // findBridgePhysicalInterface returns the name of the physical interface that belogs to on node . func findBridgePhysicalInterface(oc *exutil.CLI, nodeName, bridgeName string) (string, error) { + ovnkubePodInfo, err := ovnkubePod(oc, nodeName) + if err != nil { + return "", err + } + + out, err := adminExecInPod( + oc, + "openshift-ovn-kubernetes", + ovnkubePodInfo.podName, + ovnkubePodInfo.containerName, + fmt.Sprintf("ovs-vsctl list-ports %s", bridgeName), + ) + if err != nil { + return "", fmt.Errorf("failed to get list of ports on bridge %s:, error: %v", + bridgeName, err) + } + for _, port := range strings.Split(out, "\n") { + out, err = adminExecInPod( + oc, + "openshift-ovn-kubernetes", + ovnkubePodInfo.podName, + ovnkubePodInfo.containerName, + fmt.Sprintf("ovs-vsctl get Port %s Interfaces", port), + ) + if err != nil { + return "", fmt.Errorf("failed to get port %s on bridge %s: error: %v", + bridgeName, port, err) + + } + // remove brackets on list of interfaces + ifaces := strings.TrimPrefix(strings.TrimSuffix(out, "]"), "[") + for _, iface := range strings.Split(ifaces, ",") { + out, err = adminExecInPod( + oc, + "openshift-ovn-kubernetes", + ovnkubePodInfo.podName, + ovnkubePodInfo.containerName, + fmt.Sprintf("ovs-vsctl get Interface %s Type", strings.TrimSpace(iface)), + ) + if err != nil { + return "", fmt.Errorf("failed to get Interface %q Type on bridge %q:, error: %v", + iface, bridgeName, err) + + } + // If system Type we know this is the OVS port is the NIC + if out == "system" { + return port, nil + } + } + } + return "", fmt.Errorf("Could not find a physical interface connected to bridge %s on node %s (pod %s)", + bridgeName, nodeName, ovnkubePodInfo.podName) +} + +func ovnkubePod(oc *exutil.CLI, nodeName string) (ovnKubePodInfo, error) { var podName string var out string var err error @@ -268,7 +328,7 @@ func findBridgePhysicalInterface(oc *exutil.CLI, nodeName, bridgeName string) (s "--field-selector", fmt.Sprintf("spec.nodeName=%s", nodeName), "-l", "app=ovnkube-node") if err != nil { - return "", err + return ovnKubePodInfo{}, err } outReader := bufio.NewScanner(strings.NewReader(out)) re := regexp.MustCompile("^pod/(.*)") @@ -281,13 +341,13 @@ func findBridgePhysicalInterface(oc *exutil.CLI, nodeName, bridgeName string) (s break } if podName == "" { - return "", fmt.Errorf("Could not find a valid ovnkube-node pod on node '%s'", nodeName) + return ovnKubePodInfo{}, fmt.Errorf("Could not find a valid ovnkube-node pod on node '%s'", nodeName) } ovnkubePod, err := oc.AdminKubeClient().CoreV1().Pods("openshift-ovn-kubernetes").Get(context.Background(), podName, metav1.GetOptions{}) if err != nil { - return "", fmt.Errorf("couldn't get %s pod in openshift-ovn-kubernetes namespace: %v", podName, err) + return ovnKubePodInfo{}, fmt.Errorf("couldn't get %s pod in openshift-ovn-kubernetes namespace: %v", podName, err) } ovnkubeContainerName := "" @@ -299,42 +359,12 @@ func findBridgePhysicalInterface(oc *exutil.CLI, nodeName, bridgeName string) (s } } if ovnkubeContainerName == "" { - return "", fmt.Errorf("didn't find ovnkube-node or ovnkube-controller container in %s pod", podName) - } - - out, err = adminExecInPod(oc, "openshift-ovn-kubernetes", podName, ovnkubeContainerName, fmt.Sprintf("ovs-vsctl list-ports %s", bridgeName)) - if err != nil { - return "", fmt.Errorf("failed to get list of ports on bridge %s:, error: %v", - bridgeName, err) - } - for _, port := range strings.Split(out, "\n") { - out, err = adminExecInPod( - oc, "openshift-ovn-kubernetes", podName, ovnkubeContainerName, - fmt.Sprintf("ovs-vsctl get Port %s Interfaces", port)) - if err != nil { - return "", fmt.Errorf("failed to get port %s on bridge %s: error: %v", - bridgeName, port, err) - - } - // remove brackets on list of interfaces - ifaces := strings.TrimPrefix(strings.TrimSuffix(out, "]"), "[") - for _, iface := range strings.Split(ifaces, ",") { - out, err = adminExecInPod( - oc, "openshift-ovn-kubernetes", podName, ovnkubeContainerName, - fmt.Sprintf("ovs-vsctl get Interface %s Type", strings.TrimSpace(iface))) - if err != nil { - return "", fmt.Errorf("failed to get Interface %q Type on bridge %q:, error: %v", - iface, bridgeName, err) - - } - // If system Type we know this is the OVS port is the NIC - if out == "system" { - return port, nil - } - } + return ovnKubePodInfo{}, fmt.Errorf("didn't find ovnkube-node or ovnkube-controller container in %s pod", podName) } - return "", fmt.Errorf("Could not find a physical interface connected to bridge %s on node %s (pod %s)", - bridgeName, nodeName, podName) + return ovnKubePodInfo{ + podName: podName, + containerName: ovnkubeContainerName, + }, nil } // adminExecInPod runs a command as admin in the provides pod inside the provided namespace. diff --git a/test/extended/networking/livemigration.go b/test/extended/networking/livemigration.go index 29bd8c96d3b3..fc548734a1be 100644 --- a/test/extended/networking/livemigration.go +++ b/test/extended/networking/livemigration.go @@ -31,8 +31,10 @@ import ( ) var _ = Describe("[sig-network][OCPFeatureGate:PersistentIPsForVirtualization][Feature:Layer2LiveMigration] Kubevirt Virtual Machines", func() { - oc := exutil.NewCLIWithPodSecurityLevel("network-segmentation-e2e", admissionapi.LevelBaseline) + // disable automatic namespace creation, we need to add the required UDN label + oc := exutil.NewCLIWithoutNamespace("network-segmentation-e2e") f := oc.KubeFramework() + f.NamespacePodSecurityLevel = admissionapi.LevelBaseline InOVNKubernetesContext(func() { var ( @@ -64,10 +66,19 @@ var _ = Describe("[sig-network][OCPFeatureGate:PersistentIPsForVirtualization][F ) DescribeTableSubtree("created using", - func(createNetworkFn func(netConfig networkAttachmentConfigParams)) { + func(createNetworkFn func(netConfig networkAttachmentConfigParams) networkAttachmentConfig) { DescribeTable("[Suite:openshift/network/virtualization] should keep ip", func(netConfig networkAttachmentConfigParams, vmResource string, opCmd func(cli *kubevirt.Client, vmNamespace, vmName string)) { var err error + l := map[string]string{ + "e2e-framework": f.BaseName, + } + if netConfig.role == "primary" { + l[RequiredUDNNamespaceLabel] = "" + } + ns, err := f.CreateNamespace(context.TODO(), f.BaseName, l) + Expect(err).NotTo(HaveOccurred()) + f.Namespace = ns netConfig.namespace = f.Namespace.Name // correctCIDRFamily makes use of the ginkgo framework so it needs to be in the testcase netConfig.cidr = correctCIDRFamily(oc, cidrIPv4, cidrIPv6) @@ -77,7 +88,16 @@ var _ = Describe("[sig-network][OCPFeatureGate:PersistentIPsForVirtualization][F isDualStack := getIPFamilyForCluster(f) == DualStack - createNetworkFn(netConfig) + provisionedNetConfig := createNetworkFn(netConfig) + + for _, node := range workerNodes { + Eventually(func() bool { + isNetProvisioned, err := isNetworkProvisioned(oc, node.Name, provisionedNetConfig.networkName) + return err == nil && isNetProvisioned + }).WithPolling(time.Second).WithTimeout(udnCrReadyTimeout).Should( + BeTrueBecause("the network must be ready before creating workloads"), + ) + } httpServerPods := prepareHTTPServerPods(f, netConfig, workerNodes) vmCreationParams := kubevirt.CreationTemplateParams{ @@ -95,7 +115,19 @@ var _ = Describe("[sig-network][OCPFeatureGate:PersistentIPsForVirtualization][F waitForVMReadiness(virtClient, vmCreationParams.VMNamespace, vmCreationParams.VMName) By("Retrieving addresses before test operation") - initialAddresses := obtainAddresses(virtClient, netConfig, vmName) + var initialAddresses []string + Eventually(func(g Gomega) []string { + GinkgoHelper() + + var err error + initialAddresses, err = obtainAddresses(virtClient, vmName) + g.Expect(err).NotTo(HaveOccurred(), "Failed to obtain IP addresses for VM") + return initialAddresses + }). + WithPolling(time.Second). + WithTimeout(5 * time.Minute). + ShouldNot(BeEmpty()) + expectedNumberOfAddresses := 1 if isDualStack { expectedNumberOfAddresses = 2 @@ -104,13 +136,24 @@ var _ = Describe("[sig-network][OCPFeatureGate:PersistentIPsForVirtualization][F httpServerPodsIPs := httpServerTestPodsMultusNetworkIPs(netConfig, httpServerPods) - By("Check east/west traffic before test operation") + By(fmt.Sprintf("Check east/west traffic before test operation using IPs: %v", httpServerPodsIPs)) checkEastWestTraffic(virtClient, vmName, httpServerPodsIPs) opCmd(virtClient, f.Namespace.Name, vmName) By("Retrieving addresses after test operation") - obtainedAddresses := obtainAddresses(virtClient, netConfig, vmName) + var obtainedAddresses []string + Eventually(func(g Gomega) []string { + GinkgoHelper() + + var err error + obtainedAddresses, err = obtainAddresses(virtClient, vmName) + g.Expect(err).NotTo(HaveOccurred(), "Failed to obtain IP addresses for VM after the migrate or restart operation") + return obtainedAddresses + }). + WithPolling(time.Second). + WithTimeout(5 * time.Minute). + ShouldNot(BeEmpty()) Expect(obtainedAddresses).To(ConsistOf(initialAddresses)) By("Check east/west after test operation") @@ -183,23 +226,92 @@ var _ = Describe("[sig-network][OCPFeatureGate:PersistentIPsForVirtualization][F restartVM, )) }, - Entry("NetworkAttachmentDefinitions", func(c networkAttachmentConfigParams) { + Entry("NetworkAttachmentDefinitions", func(c networkAttachmentConfigParams) networkAttachmentConfig { netConfig := newNetworkAttachmentConfig(c) nad := generateNAD(netConfig) By(fmt.Sprintf("Creating NetworkAttachmentDefinitions %s/%s", nad.Namespace, nad.Name)) _, err := nadClient.NetworkAttachmentDefinitions(c.namespace).Create(context.Background(), nad, metav1.CreateOptions{}) - Expect(err).NotTo((HaveOccurred())) + Expect(err).NotTo(HaveOccurred()) + return netConfig }), - Entry("UserDefinedNetwork", func(c networkAttachmentConfigParams) { + Entry("[OCPFeatureGate:NetworkSegmentation] UserDefinedNetwork", func(c networkAttachmentConfigParams) networkAttachmentConfig { udnManifest := generateUserDefinedNetworkManifest(&c) By(fmt.Sprintf("Creating UserDefinedNetwork %s/%s", c.namespace, c.name)) Expect(applyManifest(c.namespace, udnManifest)).To(Succeed()) - Expect(waitForUserDefinedNetworkReady(c.namespace, c.name, udnCrReadyTimeout)).To(Succeed()) + Eventually(userDefinedNetworkReadyFunc(oc.DynamicClient(), c.namespace, c.name), udnCrReadyTimeout, time.Second).Should(Succeed()) + + nad, err := nadClient.NetworkAttachmentDefinitions(c.namespace).Get( + context.Background(), c.name, metav1.GetOptions{}, + ) + Expect(err).NotTo(HaveOccurred()) + return networkAttachmentConfig{networkAttachmentConfigParams{networkName: networkName(nad.Spec.Config)}} })) }) }) }) +var _ = Describe("[sig-network][Feature:Layer2LiveMigration][OCPFeatureGate:NetworkSegmentation][Suite:openshift/network/virtualization] primary UDN smoke test", func() { + // disable automatic namespace creation, we need to add the required UDN label + oc := exutil.NewCLIWithoutNamespace("network-segmentation-e2e") + f := oc.KubeFramework() + f.NamespacePodSecurityLevel = admissionapi.LevelBaseline + + const ( + nadName = "blue" + cidrIPv4 = "203.203.0.0/16" + cidrIPv6 = "2014:100:200::0/60" + ) + + InOVNKubernetesContext(func() { + var ( + cs clientset.Interface + nadClient nadclient.K8sCniCncfIoV1Interface + ) + + BeforeEach(func() { + cs = f.ClientSet + + ns, err := f.CreateNamespace(context.TODO(), f.BaseName, map[string]string{ + "e2e-framework": f.BaseName, + RequiredUDNNamespaceLabel: "", + }) + f.Namespace = ns + nadClient, err = nadclient.NewForConfig(f.ClientConfig()) + Expect(err).NotTo(HaveOccurred()) + }) + + It("assert the primary UDN feature works as expected", func() { + netConfig := networkAttachmentConfigParams{ + name: nadName, + topology: "layer2", + role: "primary", + allowPersistentIPs: true, + namespace: f.Namespace.Name, + cidr: correctCIDRFamily(oc, cidrIPv4, cidrIPv6), + } + + nad := generateNAD(newNetworkAttachmentConfig(netConfig)) + By(fmt.Sprintf("Creating NetworkAttachmentDefinitions %s/%s", nad.Namespace, nad.Name)) + _, err := nadClient.NetworkAttachmentDefinitions(f.Namespace.Name).Create( + context.Background(), + nad, + metav1.CreateOptions{}, + ) + Expect(err).NotTo(HaveOccurred()) + + workerNodes, err := getWorkerNodesOrdered(cs) + Expect(err).NotTo(HaveOccurred()) + + httpServerPods := prepareHTTPServerPods(f, netConfig, workerNodes) + Expect(httpServerPods).NotTo(BeEmpty()) + Expect(podNetworkStatus(httpServerPods[0])).To( + HaveLen(2), + "the pod network status must feature both the cluster default network and the primary UDN attachment", + ) + }) + }) +}) + var _ = Describe("[sig-network][Feature:Layer2LiveMigration][Suite:openshift/network/virtualization] Kubevirt Virtual Machines", func() { It("Placeholder test for GA", func() { Expect(1).To(Equal(1)) // we just need a test to run to ensure the platform comes up correctly @@ -262,50 +374,28 @@ func waitForVMIMSuccess(vmClient *kubevirt.Client, namespace, vmName string) { return migrationCompletedStr }).WithPolling(time.Second).WithTimeout(5 * time.Minute).Should(Equal("true")) migrationFailedStr, err := vmClient.GetJSONPath("vmim", vmName, "{@.status.migrationState.failed}") - Expect(err).NotTo((HaveOccurred())) + Expect(err).NotTo(HaveOccurred()) Expect(migrationFailedStr).To(BeEmpty()) } -func addressFromStatus(cli *kubevirt.Client, vmName string) []string { - GinkgoHelper() - addressesStr, err := cli.GetJSONPath("vmi", vmName, "{@.status.interfaces[0].ipAddresses}") - Expect(err).NotTo((HaveOccurred())) +func addressFromStatus(cli *kubevirt.Client, vmName string) ([]string, error) { var addresses []string - Expect(json.Unmarshal([]byte(addressesStr), &addresses)).To(Succeed()) - return addresses -} - -func addressFromGuest(cli *kubevirt.Client, vmName string) []string { - GinkgoHelper() - Expect(cli.Login(vmName, vmName)).To(Succeed()) - output, err := cli.Console(vmName, "ip -j a show dev eth0") - Expect(err).NotTo((HaveOccurred())) - // [{"ifindex":2,"ifname":"eth0","flags":["BROADCAST","MULTICAST","UP","LOWER_UP"],"mtu":1300,"qdisc":"fq_codel","operstate":"UP","group":"default","txqlen":1000,"link_type":"ether","address":"02:ba:c3:00:00:0a","broadcast":"ff:ff:ff:ff:ff:ff","altnames":["enp1s0"],"addr_info":[{"family":"inet","local":"100.10.0.1","prefixlen":24,"broadcast":"100.10.0.255","scope":"global","dynamic":true,"noprefixroute":true,"label":"eth0","valid_life_time":86313548,"preferred_life_time":86313548},{"family":"inet6","local":"fe80::ba:c3ff:fe00:a","prefixlen":64,"scope":"link","valid_life_time":4294967295,"preferred_life_time":4294967295}]}] - type address struct { - IP string `json:"local,omitempty"` - Scope string `json:"scope,omitempty"` + addressesStr, err := cli.GetJSONPath("vmi", vmName, "{@.status.interfaces[0].ipAddresses}") + if err != nil { + return nil, fmt.Errorf("failed to extract the IP addresses from VM %q: %w", vmName, err) } - type iface struct { - Name string `json:"ifname,omitempty"` - Addresses []address `json:"addr_info,omitempty"` + + if addressesStr == "" { + return nil, nil } - ifaces := []iface{} - Expect(json.Unmarshal([]byte(output), &ifaces)).To(Succeed()) - addresses := []string{} - Expect(ifaces).NotTo((BeEmpty())) - for _, address := range ifaces[0].Addresses { - if address.Scope == "link" { - continue - } - addresses = append(addresses, address.IP) + + if err := json.Unmarshal([]byte(addressesStr), &addresses); err != nil { + return nil, fmt.Errorf("failed to unmarshal addresses %q: %w", addressesStr, err) } - return addresses + return addresses, nil } -func obtainAddresses(virtClient *kubevirt.Client, netConfig networkAttachmentConfigParams, vmName string) []string { - if netConfig.role == "primary" { - return addressFromGuest(virtClient, vmName) - } +func obtainAddresses(virtClient *kubevirt.Client, vmName string) ([]string, error) { return addressFromStatus(virtClient, vmName) } @@ -378,6 +468,9 @@ func podNetworkStatus(pod *v1.Pod, predicates ...func(nadapi.NetworkStatus) bool return nil, err } + if len(predicates) == 0 { + return netStatus, nil + } var netStatusMeetingPredicates []nadapi.NetworkStatus for i := range netStatus { for _, predicate := range predicates { @@ -468,3 +561,40 @@ func checkEastWestTraffic(virtClient *kubevirt.Client, vmiName string, podIPsByN } } } + +func isNetworkProvisioned(oc *exutil.CLI, nodeName string, networkName string) (bool, error) { + ovnkubePodInfo, err := ovnkubePod(oc, nodeName) + if err != nil { + return false, err + } + + lsName := logicalSwitchName(networkName) + out, err := adminExecInPod( + oc, + "openshift-ovn-kubernetes", + ovnkubePodInfo.podName, + ovnkubePodInfo.containerName, + fmt.Sprintf("ovn-nbctl list logical-switch %s", lsName), + ) + if err != nil { + return false, fmt.Errorf("failed to find a logical switch for network %q: %w", networkName, err) + } + + return strings.Contains(out, lsName), nil +} + +func logicalSwitchName(networkName string) string { + netName := strings.ReplaceAll(networkName, "-", ".") + netName = strings.ReplaceAll(netName, "/", ".") + return fmt.Sprintf("%s_ovn_layer2_switch", netName) +} + +func networkName(netSpecConfig string) string { + GinkgoHelper() + type netConfig struct { + Name string `json:"name,omitempty"` + } + var nc netConfig + Expect(json.Unmarshal([]byte(netSpecConfig), &nc)).To(Succeed()) + return nc.Name +} diff --git a/test/extended/networking/multinetpolicy.go b/test/extended/networking/multinetpolicy.go index f654681bc9c2..e9c70c755a5d 100644 --- a/test/extended/networking/multinetpolicy.go +++ b/test/extended/networking/multinetpolicy.go @@ -132,21 +132,37 @@ func getClusterNetwork(c operatorclientv1.NetworkInterface) *operatorv1.Network } func podShouldReach(oc *exutil.CLI, podName, address string) { + namespacePodShouldReach(oc, "", podName, address) +} + +func namespacePodShouldReach(oc *exutil.CLI, namespace, podName, address string) { out := "" o.EventuallyWithOffset(1, func() error { var err error - out, err = oc.AsAdmin().Run("exec").Args(podName, "--", "curl", "--connect-timeout", "1", address).Output() + if namespace == "" { + out, err = oc.AsAdmin().Run("exec").Args(podName, "--", "curl", "--connect-timeout", "1", "--max-time", "5", address).Output() + } else { + out, err = oc.AsAdmin().Run("exec").Args(podName, "-n", namespace, "--", "curl", "--connect-timeout", "1", "--max-time", "5", address).Output() + } return err - }, "30s", "1s").ShouldNot(o.HaveOccurred(), "cmd output: %s", out) + }, "30s", "5s").ShouldNot(o.HaveOccurred(), "cmd output: %s", out) } func podShouldNotReach(oc *exutil.CLI, podName, address string) { + namespacePodShouldNotReach(oc, "", podName, address) +} + +func namespacePodShouldNotReach(oc *exutil.CLI, namespace, podName, address string) { out := "" o.EventuallyWithOffset(1, func() error { var err error - out, err = oc.AsAdmin().Run("exec").Args(podName, "--", "curl", "--connect-timeout", "1", address).Output() + if namespace == "" { + out, err = oc.AsAdmin().Run("exec").Args(podName, "--", "curl", "--connect-timeout", "1", "--max-time", "5", address).Output() + } else { + out, err = oc.AsAdmin().Run("exec").Args(podName, "-n", namespace, "--", "curl", "--connect-timeout", "1", "--max-time", "5", address).Output() + } return err - }, "30s", "1s").Should(o.HaveOccurred(), "cmd output: %s", out) + }, "30s", "5s").Should(o.HaveOccurred(), "cmd output: %s", out) } func mustParseIPAndMask(in string) *net.IPNet { diff --git a/test/extended/networking/network_segmentation.go b/test/extended/networking/network_segmentation.go index 9d77bf0c742f..484b801f3b31 100644 --- a/test/extended/networking/network_segmentation.go +++ b/test/extended/networking/network_segmentation.go @@ -13,6 +13,9 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" nadapi "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" nadclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1" @@ -35,12 +38,38 @@ import ( exutil "github.com/openshift/origin/test/extended/util" ) +const openDefaultPortsAnnotation = "k8s.ovn.org/open-default-ports" +const RequiredUDNNamespaceLabel = "k8s.ovn.org/primary-user-defined-network" + +// NOTE: We are observing pod creation requests taking more than two minutes t +// reach the CNI for the CNI to do the necessary plumbing. This is causing tests +// to timeout since pod doesn't go into ready state. +// See https://issues.redhat.com/browse/OCPBUGS-48362 for details. We can revisit +// these values when that bug is fixed but given the Kubernetes test default for a +// pod to startup is 5mins: https://github.com/kubernetes/kubernetes/blob/60c4c2b2521fb454ce69dee737e3eb91a25e0535/test/e2e/framework/timeouts.go#L22-L23 +// we are not too far from the mark or against test policy +const podReadyPollTimeout = 4 * time.Minute +const podReadyPollInterval = 6 * time.Second + +// NOTE: Upstream, we use either the default of gomega which is 1sec polltimeout with 10ms pollinterval OR +// the tests have hardcoded values with 5sec being common for polltimeout and 10ms for pollinterval +// This is being changed to be 10seconds poll timeout to account for infrastructure complexity between +// OpenShift and KIND clusters. Also changing the polling interval to be 1 second so that in both +// Eventually and Consistently blocks we get at least 10 retries (10/1) in good conditions and 5 retries (10/2) in +// bad conditions since connectToServer util has a 2 second timeout. +// FIXME: Timeout increased to 30 seconds because default network controller does not receive the pod event after its annotations +// are updated. Reduce timeout back to sensible value once issue is understood. +const serverConnectPollTimeout = 30 * time.Second +const serverConnectPollInterval = 1 * time.Second + var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks]", func() { // TODO: so far, only the isolation tests actually require this PSA ... Feels wrong to run everything priviliged. // I've tried to have multiple kubeframeworks (from multiple OCs) running (with different project names) but // it didn't work. - oc := exutil.NewCLIWithPodSecurityLevel("network-segmentation-e2e", admissionapi.LevelPrivileged) + // disable automatic namespace creation, we need to add the required UDN label + oc := exutil.NewCLIWithoutNamespace("network-segmentation-e2e") f := oc.KubeFramework() + f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged InOVNKubernetesContext(func() { const ( @@ -68,16 +97,25 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User }) DescribeTableSubtree("created using", - func(createNetworkFn func(c networkAttachmentConfigParams) error) { + func(createNetworkFn func(c *networkAttachmentConfigParams) error) { DescribeTable( "can perform east/west traffic between nodes", func( - netConfig networkAttachmentConfigParams, + netConfig *networkAttachmentConfigParams, clientPodConfig podConfiguration, serverPodConfig podConfiguration, ) { var err error + l := map[string]string{ + "e2e-framework": f.BaseName, + } + if netConfig.role == "primary" { + l[RequiredUDNNamespaceLabel] = "" + } + ns, err := f.CreateNamespace(context.TODO(), f.BaseName, l) + Expect(err).NotTo(HaveOccurred()) + f.Namespace = ns netConfig.namespace = f.Namespace.Name // correctCIDRFamily makes use of the ginkgo framework so it needs to be in the testcase @@ -119,12 +157,12 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User } By("asserting the *client* pod can contact the server pod exposed endpoint") - podShouldReach(oc, clientPodConfig.name, formatHostAndPort(net.ParseIP(serverIP), port)) + namespacePodShouldReach(oc, f.Namespace.Name, clientPodConfig.name, formatHostAndPort(net.ParseIP(serverIP), port)) } }, Entry( "for two pods connected over a L2 primary UDN", - networkAttachmentConfigParams{ + &networkAttachmentConfigParams{ name: nadName, topology: "layer2", role: "primary", @@ -138,7 +176,7 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User ), Entry( "two pods connected over a L3 primary UDN", - networkAttachmentConfigParams{ + &networkAttachmentConfigParams{ name: nadName, topology: "layer3", role: "primary", @@ -155,12 +193,21 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User DescribeTable( "is isolated from the default network", func( - netConfigParams networkAttachmentConfigParams, + netConfigParams *networkAttachmentConfigParams, udnPodConfig podConfiguration, ) { + l := map[string]string{ + "e2e-framework": f.BaseName, + } + if netConfigParams.role == "primary" { + l[RequiredUDNNamespaceLabel] = "" + } + ns, err := f.CreateNamespace(context.TODO(), f.BaseName, l) + Expect(err).NotTo(HaveOccurred()) + f.Namespace = ns By("Creating second namespace for default network pods") defaultNetNamespace := f.Namespace.Name + "-default" - _, err := cs.CoreV1().Namespaces().Create(context.Background(), &v1.Namespace{ + _, err = cs.CoreV1().Namespaces().Create(context.Background(), &v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: defaultNetNamespace, }, @@ -189,7 +236,13 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User }, InitialDelaySeconds: 1, PeriodSeconds: 1, - FailureThreshold: 1, + // FIXME: On OCP we have seen readiness probe failures happening for the UDN pod which + // causes immediate container restarts - the first readiness probe failure usually happens because + // connection gets reset by the pod since normally a liveness probe fails first causing a + // restart that also causes the readiness probes to start failing. + // Hence increase the failure threshold to 3 tries. + FailureThreshold: 3, + TimeoutSeconds: 3, } pod.Spec.Containers[0].LivenessProbe = &v1.Probe{ ProbeHandler: v1.ProbeHandler{ @@ -200,7 +253,24 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User }, InitialDelaySeconds: 1, PeriodSeconds: 1, - FailureThreshold: 1, + // FIXME: On OCP we have seen liveness probe failures happening for the UDN pod which + // causes immediate container restarts. Hence increase the failure threshold to 3 tries + // TBD: We unfortunately don't know why the 1st liveness probe timesout - once we know the + // why we could bring this back to 1 even though 1 is still aggressive. + FailureThreshold: 3, + // FIXME: On OCP, we have seen this flake in the CI; example: + // Pod event: Type=Warning Reason=Unhealthy Message=Liveness probe failed: Get "http://[fd01:0:0:5::2ed]:9000/healthz": + // context deadline exceeded (Client.Timeout exceeded while awaiting headers) LastTimestamp=2025-01-21 15:16:43 +0000 UTC Count=1 + // Pod event: Type=Normal Reason=Killing Message=Container agnhost-container failed liveness probe, will be restarted + // LastTimestamp=2025-01-21 15:16:43 +0000 UTC Count=1 + // Pod event: Type=Warning Reason=Unhealthy Message=Readiness probe failed: Get "http://[fd01:0:0:5::2ed]:9000/healthz": + // context deadline exceeded (Client.Timeout exceeded while awaiting headers) LastTimestamp=2025-01-21 15:16:43 +0000 UTC Count=1 + // Pod event: Type=Warning Reason=Unhealthy Message=Readiness probe failed: Get "http://[fd01:0:0:5::2ed]:9000/healthz": + // read tcp [fd01:0:0:5::2]:33400->[fd01:0:0:5::2ed]:9000: read: connection reset by peer LastTimestamp=2025-01-21 15:16:43 +0000 UTC Count=1 + // While we don't know why 1second wasn't enough to receive the headers for the liveness probe + // it is clear the TCP conn is getting established but 1second is not enough to complete the probe. + // Let's increase the timeout to 3seconds till we understand what causes the 1st probe failure. + TimeoutSeconds: 3, } pod.Spec.Containers[0].StartupProbe = &v1.Probe{ ProbeHandler: v1.ProbeHandler{ @@ -212,6 +282,8 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User InitialDelaySeconds: 1, PeriodSeconds: 1, FailureThreshold: 3, + // FIXME: Figure out why it sometimes takes more than 3seconds for the healthcheck to complete + TimeoutSeconds: 3, } // add NET_ADMIN to change pod routes pod.Spec.Containers[0].SecurityContext = &v1.SecurityContext{ @@ -260,7 +332,7 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User By("checking the default network pod can't reach UDN pod on IP " + destIP) Consistently(func() bool { return connectToServer(podConfiguration{namespace: defaultPod.Namespace, name: defaultPod.Name}, destIP, port) != nil - }, 5*time.Second).Should(BeTrue()) + }, serverConnectPollTimeout, serverConnectPollInterval).Should(BeTrue()) } defaultIPv4, defaultIPv6, err := podIPsForDefaultNetwork( @@ -277,15 +349,15 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User By("checking the default network client pod can reach default pod on IP " + destIP) Eventually(func() bool { return connectToServer(podConfiguration{namespace: defaultClientPod.Namespace, name: defaultClientPod.Name}, destIP, defaultPort) == nil - }).Should(BeTrue()) + }, serverConnectPollTimeout, serverConnectPollInterval).Should(BeTrue()) By("checking the UDN pod can't reach the default network pod on IP " + destIP) Consistently(func() bool { return connectToServer(udnPodConfig, destIP, defaultPort) != nil - }, 5*time.Second).Should(BeTrue()) + }, serverConnectPollTimeout, serverConnectPollInterval).Should(BeTrue()) } // connectivity check is run every second + 1sec initialDelay - // By this time we have spent at least 8 seconds doing the above checks + // By this time we have spent at least 20 seconds doing the above consistently checks udnPod, err = cs.CoreV1().Pods(udnPod.Namespace).Get(context.Background(), udnPod.Name, metav1.GetOptions{}) Expect(err).NotTo(HaveOccurred()) Expect(udnPod.Status.ContainerStatuses[0].RestartCount).To(Equal(int32(0))) @@ -311,9 +383,47 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User ping, "-I", "eth0", "-c", "1", "-W", "1", hostIP.IP, ) return err == nil - }, 4*time.Second).Should(BeFalse()) + }, 4*time.Second, 1*time.Second).Should(BeFalse()) } + By("asserting UDN pod can reach the kapi service in the default network") + // Use the service name to get test the DNS access + Consistently(func() bool { + _, err := e2ekubectl.RunKubectl( + udnPodConfig.namespace, + "exec", + udnPodConfig.name, + "--", + "curl", + "--connect-timeout", + // FIXME: We have seen in OCP CI that it can take two seconds or maybe more + // for a single curl to succeed. Example: + // STEP: asserting UDN pod can reach the kapi service in the default network @ 01/20/25 00:38:42.32 + // I0120 00:38:42.320808 70120 builder.go:121] Running '/usr/bin/kubectl + // --server=https://api.ci-op-bkg2qwwq-4edbf.XXXXXXXXXXXXXXXXXXXXXX:6443 --kubeconfig=/tmp/kubeconfig-1734723086 + // --namespace=e2e-test-network-segmentation-e2e-kzdw7 exec udn-pod -- curl --connect-timeout 2 --insecure https://kubernetes.default/healthz' + // I0120 00:38:44.108334 70120 builder.go:146] stderr: " % Total % Received % Xferd Average Speed Time Time Time Current\n Dload Upload Total Spent Left Speed\n\r 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0\r100 2 100 2 0 0 9 0 --:--:-- --:--:-- --:--:-- 9\r100 2 100 2 0 0 9 0 --:--:-- --:--:-- --:--:-- 9\n" + // I0120 00:38:44.108415 70120 builder.go:147] stdout: "ok" --> 2 seconds later + // I0120 00:38:45.109237 70120 builder.go:121] Running '/usr/bin/kubectl + // --server=https://api.ci-op-bkg2qwwq-4edbf.XXXXXXXXXXXXXXXXXXXXXX:6443 --kubeconfig=/tmp/kubeconfig-1734723086 + // --namespace=e2e-test-network-segmentation-e2e-kzdw7 exec udn-pod -- curl --connect-timeout 2 --insecure https://kubernetes.default/healthz' + // I0120 00:38:48.460089 70120 builder.go:135] rc: 28 + // around the same time we have observed OVS issues like: + // Jan 20 00:38:45.329999 ci-op-bkg2qwwq-4edbf-xv8kb-worker-b-flqxd ovs-vswitchd[1094]: ovs|03661|timeval|WARN|context switches: 0 voluntary, 695 involuntary + // Jan 20 00:38:45.329967 ci-op-bkg2qwwq-4edbf-xv8kb-worker-b-flqxd ovs-vswitchd[1094]: ovs|03660|timeval|WARN|Unreasonably long 1730ms poll interval (32ms user, 903ms system) + // which might need more investigation. Bumping the timeout to 5seconds can help with this + // but we need to figure out what exactly is causing random timeouts in CI when trying to reach kapi-server + // sometimes we have also seen more than 2seconds being taken for the timeout which also needs to be investigated: + // I0118 13:35:50.419638 87083 builder.go:121] Running '/usr/bin/kubectl + // --server=https://api.ostest.test.metalkube.org:6443 --kubeconfig=/tmp/secret/kubeconfig + // --namespace=e2e-test-network-segmentation-e2e-d4fzk exec udn-pod -- curl --connect-timeout 2 --insecure https://kubernetes.default/healthz' + // I0118 13:35:54.093268 87083 builder.go:135] rc: 28 --> takes close to 4seconds? + "5", + "--insecure", + "https://kubernetes.default/healthz") + return err == nil + }, 15*time.Second, 3*time.Second).Should(BeTrue()) + By("asserting UDN pod can't reach default services via default network interface") // route setup is already done, get kapi IPs kapi, err := cs.CoreV1().Services("default").Get(context.Background(), "kubernetes", metav1.GetOptions{}) @@ -334,12 +444,12 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User "--insecure", fmt.Sprintf("https://%s/healthz", kapiIP)) return err != nil - }, 5*time.Second).Should(BeTrue()) + }, 5*time.Second, 1*time.Second).Should(BeTrue()) } }, Entry( "with L2 primary UDN", - networkAttachmentConfigParams{ + &networkAttachmentConfigParams{ name: nadName, topology: "layer2", role: "primary", @@ -350,7 +460,7 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User ), Entry( "with L3 primary UDN", - networkAttachmentConfigParams{ + &networkAttachmentConfigParams{ name: nadName, topology: "layer3", role: "primary", @@ -369,27 +479,30 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User userDefinedv6Subnet string, ) { - + l := map[string]string{ + "e2e-framework": f.BaseName, + RequiredUDNNamespaceLabel: "", + } + ns, err := f.CreateNamespace(context.TODO(), f.BaseName, l) + Expect(err).NotTo(HaveOccurred()) + f.Namespace = ns red := "red" blue := "blue" namespaceRed := f.Namespace.Name + "-" + red namespaceBlue := f.Namespace.Name + "-" + blue - netConfig := networkAttachmentConfigParams{ - topology: topology, - cidr: correctCIDRFamily(oc, userDefinedv4Subnet, userDefinedv6Subnet), - role: "primary", - } for _, namespace := range []string{namespaceRed, namespaceBlue} { By("Creating namespace " + namespace) _, err := cs.CoreV1().Namespaces().Create(context.Background(), &v1.Namespace{ ObjectMeta: metav1.ObjectMeta{ - Name: namespace, + Name: namespace, + Labels: l, }, }, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) defer func() { + By("Removing namespace " + namespace) Expect(cs.CoreV1().Namespaces().Delete( context.Background(), namespace, @@ -400,22 +513,41 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User networkNamespaceMap := map[string]string{namespaceRed: red, namespaceBlue: blue} for namespace, network := range networkNamespaceMap { By("creating the network " + network + " in namespace " + namespace) - netConfig.namespace = namespace - netConfig.name = network + netConfig := &networkAttachmentConfigParams{ + topology: topology, + cidr: correctCIDRFamily(oc, userDefinedv4Subnet, userDefinedv6Subnet), + role: "primary", + namespace: namespace, + name: network, + } Expect(createNetworkFn(netConfig)).To(Succeed()) + // update the name because createNetworkFn may mutate the netConfig.name + // for cluster scope objects (i.g.: CUDN cases) to enable parallel testing. + networkNamespaceMap[namespace] = netConfig.name + } + red = networkNamespaceMap[namespaceRed] + blue = networkNamespaceMap[namespaceBlue] + workerNodes, err := getWorkerNodesOrdered(cs) Expect(err).NotTo(HaveOccurred()) pods := []*v1.Pod{} - redIPs := []string{} - blueIPs := []string{} + redIPs := map[string]bool{} + blueIPs := map[string]bool{} + podIPs := []string{} + bluePort := int(9091) + redPort := int(9092) for namespace, network := range networkNamespaceMap { for i := 0; i < numberOfPods; i++ { + httpServerPort := redPort + if network != red { + httpServerPort = bluePort + } podConfig := *podConfig( fmt.Sprintf("%s-pod-%d", network, i), withCommand(func() []string { - return httpServerContainerCmd(port) + return httpServerContainerCmd(uint16(httpServerPort)) }), ) podConfig.namespace = namespace @@ -444,10 +576,11 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User 0, ) Expect(err).NotTo(HaveOccurred()) + podIPs = append(podIPs, podIP) if network == red { - redIPs = append(redIPs, podIP) + redIPs[podIP] = true } else { - blueIPs = append(blueIPs, podIP) + blueIPs[podIP] = true } } } @@ -455,69 +588,32 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User By("ensuring pods only communicate with pods in their network") for _, pod := range pods { isRedPod := strings.Contains(pod.Name, red) - ips := redIPs + expectedHostname := red if !isRedPod { - ips = blueIPs + expectedHostname = blue } - for _, ip := range ips { - result, err := e2ekubectl.RunKubectl( - pod.Namespace, - "exec", - pod.Name, - "--", - "curl", - "--connect-timeout", - "2", - net.JoinHostPort(ip, fmt.Sprintf("%d", port)+"/hostname"), - ) - Expect(err).NotTo(HaveOccurred()) - if isRedPod { - Expect(strings.Contains(result, red)).To(BeTrue()) + for _, ip := range podIPs { + isRedIP := redIPs[ip] + httpServerPort := redPort + if !isRedIP { + httpServerPort = bluePort + } + sameNetwork := isRedPod == isRedIP + if !sameNetwork { + _, err := connectToServerWithPath(pod.Namespace, pod.Name, ip, "/hostname", httpServerPort) + Expect(err).Should(HaveOccurred(), "should isolate from different networks") } else { - Expect(strings.Contains(result, blue)).To(BeTrue()) + Eventually(func(g Gomega) { + result, err := connectToServerWithPath(pod.Namespace, pod.Name, ip, "/hostname", httpServerPort) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(result).To(ContainSubstring(expectedHostname)) + }). + WithTimeout(serverConnectPollTimeout). + WithPolling(serverConnectPollInterval). + Should(Succeed(), "should not isolate from same network") } } } - - By("Deleting pods in network blue except " + fmt.Sprintf("%s-pod-%d", blue, numberOfPods-1)) - for i := 0; i < numberOfPods-1; i++ { - err := cs.CoreV1().Pods(namespaceBlue).Delete( - context.Background(), - fmt.Sprintf("%s-pod-%d", blue, i), - metav1.DeleteOptions{}, - ) - Expect(err).NotTo(HaveOccurred()) - } - - podIP, err := podIPsForUserDefinedPrimaryNetwork( - cs, - namespaceBlue, - fmt.Sprintf("%s-pod-%d", blue, numberOfPods-1), - namespacedName(namespaceBlue, blue), - 0, - ) - Expect(err).NotTo(HaveOccurred()) - - By("Remaining blue pod cannot communicate with red networks overlapping CIDR") - for _, ip := range redIPs { - if podIP == ip { - //don't try with your own IP - continue - } - _, err := e2ekubectl.RunKubectl( - namespaceBlue, - "exec", - fmt.Sprintf("%s-pod-%d", blue, numberOfPods-1), - "--", - "curl", - "--connect-timeout", - "2", - net.JoinHostPort(ip, fmt.Sprintf("%d", port)), - ) - if err == nil { - framework.Failf("connection succeeded but expected timeout") - } - } }, // can completely fill the L2 topology because it does not depend on the size of the clusters hostsubnet Entry( @@ -527,43 +623,64 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User "203.203.0.0/29", "2014:100:200::0/125", ), - // limit the number of pods to 10 + // limit the number of pods to 5 Entry( "with L3 primary UDN", "layer3", - 10, + 5, userDefinedNetworkIPv4Subnet, userDefinedNetworkIPv6Subnet, ), ) }, - Entry("NetworkAttachmentDefinitions", func(c networkAttachmentConfigParams) error { - netConfig := newNetworkAttachmentConfig(c) + Entry("NetworkAttachmentDefinitions", func(c *networkAttachmentConfigParams) error { + netConfig := newNetworkAttachmentConfig(*c) nad := generateNAD(netConfig) _, err := nadClient.NetworkAttachmentDefinitions(c.namespace).Create(context.Background(), nad, metav1.CreateOptions{}) return err }), - Entry("UserDefinedNetwork", func(c networkAttachmentConfigParams) error { - udnManifest := generateUserDefinedNetworkManifest(&c) + Entry("UserDefinedNetwork", func(c *networkAttachmentConfigParams) error { + udnManifest := generateUserDefinedNetworkManifest(c) cleanup, err := createManifest(c.namespace, udnManifest) DeferCleanup(cleanup) - Expect(waitForUserDefinedNetworkReady(c.namespace, c.name, udnCrReadyTimeout)).To(Succeed()) + Eventually(userDefinedNetworkReadyFunc(oc.AdminDynamicClient(), c.namespace, c.name), udnCrReadyTimeout, time.Second).Should(Succeed()) + return err + }), + Entry("ClusterUserDefinedNetwork", func(c *networkAttachmentConfigParams) error { + cudnName := randomNetworkMetaName() + c.name = cudnName + cudnManifest := generateClusterUserDefinedNetworkManifest(c) + cleanup, err := createManifest("", cudnManifest) + DeferCleanup(func() { + cleanup() + By(fmt.Sprintf("delete pods in %s namespace to unblock CUDN CR & associate NAD deletion", c.namespace)) + Expect(cs.CoreV1().Pods(c.namespace).DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{})).To(Succeed()) + _, err := e2ekubectl.RunKubectl("", "delete", "clusteruserdefinednetwork", cudnName, "--wait", fmt.Sprintf("--timeout=%ds", 120)) + Expect(err).NotTo(HaveOccurred()) + }) + Eventually(clusterUserDefinedNetworkReadyFunc(oc.AdminDynamicClient(), c.name), 5*time.Second, time.Second).Should(Succeed()) return err }), ) - Context("UserDefinedNetwork", func() { + Context("UserDefinedNetwork CRD controller", func() { const ( testUdnName = "test-net" userDefinedNetworkResource = "userdefinednetwork" ) BeforeEach(func() { + namespace, err := f.CreateNamespace(context.TODO(), f.BaseName, map[string]string{ + "e2e-framework": f.BaseName, + }) + Expect(err).NotTo(HaveOccurred()) + f.Namespace = namespace + By("create tests UserDefinedNetwork") cleanup, err := createManifest(f.Namespace.Name, newUserDefinedNetworkManifest(testUdnName)) DeferCleanup(cleanup) Expect(err).NotTo(HaveOccurred()) - Expect(waitForUserDefinedNetworkReady(f.Namespace.Name, testUdnName, 5*time.Second)).To(Succeed()) + Eventually(userDefinedNetworkReadyFunc(oc.AdminDynamicClient(), f.Namespace.Name, testUdnName), udnCrReadyTimeout, time.Second).Should(Succeed()) }) It("should create NetworkAttachmentDefinition according to spec", func() { @@ -622,11 +739,12 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User _ = nadClient.NetworkAttachmentDefinitions(f.Namespace.Name).Delete(ctx, testUdnName, metav1.DeleteOptions{}) _, err := nadClient.NetworkAttachmentDefinitions(f.Namespace.Name).Get(ctx, testUdnName, metav1.GetOptions{}) return err - }).ShouldNot(HaveOccurred(), + }, udnInUseDeleteTimeout, deleteNetworkInterval).ShouldNot(HaveOccurred(), "should fail to delete UserDefinedNetwork associated NetworkAttachmentDefinition when used") By("verify UserDefinedNetwork status reports consuming pod") - assertUDNStatusReportsConsumers(f.Namespace.Name, testUdnName, testPodName) + err = validateUDNStatusReportsConsumers(oc.AdminDynamicClient(), f.Namespace.Name, testUdnName, testPodName) + Expect(err).ToNot(HaveOccurred()) By("delete test pod") err = cs.CoreV1().Pods(f.Namespace.Name).Delete(context.Background(), testPodName, metav1.DeleteOptions{}) @@ -655,15 +773,23 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User primaryUdnName = "primary-net" ) + l := map[string]string{ + "e2e-framework": f.BaseName, + RequiredUDNNamespaceLabel: "", + } + ns, err := f.CreateNamespace(context.TODO(), f.BaseName, l) + Expect(err).NotTo(HaveOccurred()) + f.Namespace = ns + By("create primary network NetworkAttachmentDefinition") primaryNetNad := generateNAD(newNetworkAttachmentConfig(networkAttachmentConfigParams{ role: "primary", topology: "layer3", name: primaryNadName, networkName: primaryNadName, - cidr: "10.10.100.0/24", + cidr: correctCIDRFamily(oc, userDefinedNetworkIPv4Subnet, userDefinedNetworkIPv6Subnet), })) - _, err := nadClient.NetworkAttachmentDefinitions(f.Namespace.Name).Create(context.Background(), primaryNetNad, metav1.CreateOptions{}) + _, err = nadClient.NetworkAttachmentDefinitions(f.Namespace.Name).Create(context.Background(), primaryNetNad, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) By("create primary network UserDefinedNetwork") @@ -671,27 +797,492 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User DeferCleanup(cleanup) Expect(err).NotTo(HaveOccurred()) - conditionsJSON, err := e2ekubectl.RunKubectl(f.Namespace.Name, "get", "userdefinednetwork", primaryUdnName, "-o", "jsonpath={.status.conditions}") + expectedMessage := fmt.Sprintf("primary network already exist in namespace %q: %q", f.Namespace.Name, primaryNadName) + Eventually(func(g Gomega) []metav1.Condition { + conditionsJSON, err := e2ekubectl.RunKubectl(f.Namespace.Name, "get", "userdefinednetwork", primaryUdnName, "-o", "jsonpath={.status.conditions}") + g.Expect(err).NotTo(HaveOccurred()) + var actualConditions []metav1.Condition + g.Expect(json.Unmarshal([]byte(conditionsJSON), &actualConditions)).To(Succeed()) + return normalizeConditions(actualConditions) + }, 5*time.Second, 1*time.Second).Should(SatisfyAny( + ConsistOf(metav1.Condition{ + Type: "NetworkCreated", + Status: metav1.ConditionFalse, + Reason: "SyncError", + Message: expectedMessage, + }), + ConsistOf(metav1.Condition{ + Type: "NetworkReady", + Status: metav1.ConditionFalse, + Reason: "SyncError", + Message: expectedMessage, + }), + )) + }) + + Context("ClusterUserDefinedNetwork CRD Controller", func() { + const clusterUserDefinedNetworkResource = "clusteruserdefinednetwork" + + var testTenantNamespaces []string + var defaultNetNamespace *v1.Namespace + + BeforeEach(func() { + namespace, err := f.CreateNamespace(context.TODO(), f.BaseName, map[string]string{ + "e2e-framework": f.BaseName, + RequiredUDNNamespaceLabel: "", + }) + f.Namespace = namespace + Expect(err).NotTo(HaveOccurred()) + testTenantNamespaces = []string{ + f.Namespace.Name + "blue", + f.Namespace.Name + "red", + } + + By("Creating test tenants namespaces") + for _, nsName := range testTenantNamespaces { + _, err := cs.CoreV1().Namespaces().Create(context.Background(), &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsName, + Labels: map[string]string{RequiredUDNNamespaceLabel: ""}, + }}, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() error { + err := cs.CoreV1().Namespaces().Delete(context.Background(), nsName, metav1.DeleteOptions{}) + return err + }) + } + // default cluster network namespace, for use when only testing secondary UDNs/NADs + defaultNetNamespace = &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: f.Namespace.Name + "-default", + }, + } + f.AddNamespacesToDelete(defaultNetNamespace) + _, err = cs.CoreV1().Namespaces().Create(context.Background(), defaultNetNamespace, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + testTenantNamespaces = append(testTenantNamespaces, defaultNetNamespace.Name) + }) + + var testClusterUdnName string + + BeforeEach(func() { + testClusterUdnName = randomNetworkMetaName() + By("create test CR") + cleanup, err := createManifest("", newClusterUDNManifest(testClusterUdnName, testTenantNamespaces...)) + DeferCleanup(func() error { + cleanup() + _, _ = e2ekubectl.RunKubectl("", "delete", clusterUserDefinedNetworkResource, testClusterUdnName) + Eventually(func() error { + _, err := e2ekubectl.RunKubectl("", "get", clusterUserDefinedNetworkResource, testClusterUdnName) + return err + }, 1*time.Minute, 3*time.Second).Should(MatchError(ContainSubstring(fmt.Sprintf("clusteruserdefinednetworks.k8s.ovn.org %q not found", testClusterUdnName)))) + return nil + }) + Expect(err).NotTo(HaveOccurred()) + Eventually(clusterUserDefinedNetworkReadyFunc(oc.AdminDynamicClient(), testClusterUdnName), 5*time.Second, time.Second).Should(Succeed()) + }) + + It("should create NAD according to spec in each target namespace and report active namespaces", func() { + Eventually( + validateClusterUDNStatusReportsActiveNamespacesFunc(oc.AdminDynamicClient(), testClusterUdnName, testTenantNamespaces...), + 1*time.Minute, 3*time.Second).Should(Succeed()) + + udnUidRaw, err := e2ekubectl.RunKubectl("", "get", clusterUserDefinedNetworkResource, testClusterUdnName, "-o", "jsonpath='{.metadata.uid}'") + Expect(err).NotTo(HaveOccurred(), "should get the ClsuterUserDefinedNetwork UID") + testUdnUID := strings.Trim(udnUidRaw, "'") + + By("verify a NetworkAttachmentDefinition is created according to spec") + for _, testNsName := range testTenantNamespaces { + assertClusterNADManifest(nadClient, testNsName, testClusterUdnName, testUdnUID) + } + }) + + It("when CR is deleted, should delete all managed NAD in each target namespace", func() { + By("delete test CR") + _, err := e2ekubectl.RunKubectl("", "delete", clusterUserDefinedNetworkResource, testClusterUdnName) + Expect(err).NotTo(HaveOccurred()) + + for _, nsName := range testTenantNamespaces { + By(fmt.Sprintf("verify a NAD has been deleted from namesapce %q", nsName)) + Eventually(func() bool { + _, err := nadClient.NetworkAttachmentDefinitions(nsName).Get(context.Background(), testClusterUdnName, metav1.GetOptions{}) + return err != nil && kerrors.IsNotFound(err) + }, time.Second*3, time.Second*1).Should(BeTrue(), + "NADs in target namespaces should be deleted following ClusterUserDefinedNetwork deletion") + } + }) + + It("should create NAD in new created namespaces that apply to namespace-selector", func() { + testNewNs := f.Namespace.Name + "green" + + By("add new target namespace to CR namespace-selector") + patch := fmt.Sprintf(`[{"op": "add", "path": "./spec/namespaceSelector/matchExpressions/0/values/-", "value": "%s"}]`, testNewNs) + _, err := e2ekubectl.RunKubectl("", "patch", clusterUserDefinedNetworkResource, testClusterUdnName, "--type=json", "-p="+patch) + Expect(err).NotTo(HaveOccurred()) + Eventually(clusterUserDefinedNetworkReadyFunc(oc.AdminDynamicClient(), testClusterUdnName), 5*time.Second, time.Second).Should(Succeed()) + Eventually( + validateClusterUDNStatusReportsActiveNamespacesFunc(oc.AdminDynamicClient(), testClusterUdnName, testTenantNamespaces...), + 1*time.Minute, 3*time.Second).Should(Succeed()) + + By("create the new target namespace") + _, err = cs.CoreV1().Namespaces().Create(context.Background(), &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNewNs, + Labels: map[string]string{RequiredUDNNamespaceLabel: ""}, + }}, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() error { + err := cs.CoreV1().Namespaces().Delete(context.Background(), testNewNs, metav1.DeleteOptions{}) + return err + }) + + expectedActiveNamespaces := append(testTenantNamespaces, testNewNs) + Eventually( + validateClusterUDNStatusReportsActiveNamespacesFunc(oc.AdminDynamicClient(), testClusterUdnName, expectedActiveNamespaces...), + 1*time.Minute, 3*time.Second).Should(Succeed()) + + udnUidRaw, err := e2ekubectl.RunKubectl("", "get", clusterUserDefinedNetworkResource, testClusterUdnName, "-o", "jsonpath='{.metadata.uid}'") + Expect(err).NotTo(HaveOccurred(), "should get the ClsuterUserDefinedNetwork UID") + testUdnUID := strings.Trim(udnUidRaw, "'") + + By("verify a NAD exist in new namespace according to spec") + assertClusterNADManifest(nadClient, testNewNs, testClusterUdnName, testUdnUID) + }) + + When("namespace-selector is mutated", func() { + It("should create NAD in namespaces that apply to mutated namespace-selector", func() { + testNewNs := f.Namespace.Name + "green" + + By("create new namespace") + _, err := cs.CoreV1().Namespaces().Create(context.Background(), &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: testNewNs, + Labels: map[string]string{RequiredUDNNamespaceLabel: ""}, + }}, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() error { + err := cs.CoreV1().Namespaces().Delete(context.Background(), testNewNs, metav1.DeleteOptions{}) + return err + }) + + By("add new namespace to CR namespace-selector") + patch := fmt.Sprintf(`[{"op": "add", "path": "./spec/namespaceSelector/matchExpressions/0/values/-", "value": "%s"}]`, testNewNs) + _, err = e2ekubectl.RunKubectl("", "patch", clusterUserDefinedNetworkResource, testClusterUdnName, "--type=json", "-p="+patch) + Expect(err).NotTo(HaveOccurred()) + + By("verify status reports the new added namespace as active") + expectedActiveNs := append(testTenantNamespaces, testNewNs) + Eventually( + validateClusterUDNStatusReportsActiveNamespacesFunc(oc.AdminDynamicClient(), testClusterUdnName, expectedActiveNs...), + 1*time.Minute, 3*time.Second).Should(Succeed()) + + By("verify a NAD is created in new target namespace according to spec") + udnUidRaw, err := e2ekubectl.RunKubectl("", "get", clusterUserDefinedNetworkResource, testClusterUdnName, "-o", "jsonpath='{.metadata.uid}'") + Expect(err).NotTo(HaveOccurred(), "should get the ClusterUserDefinedNetwork UID") + testUdnUID := strings.Trim(udnUidRaw, "'") + assertClusterNADManifest(nadClient, testNewNs, testClusterUdnName, testUdnUID) + }) + + It("should delete managed NAD in namespaces that no longer apply to namespace-selector", func() { + By("remove one active namespace from CR namespace-selector") + activeTenantNs := testTenantNamespaces[1] + patch := fmt.Sprintf(`[{"op": "replace", "path": "./spec/namespaceSelector/matchExpressions/0/values", "value": [%q]}]`, activeTenantNs) + _, err := e2ekubectl.RunKubectl("", "patch", clusterUserDefinedNetworkResource, testClusterUdnName, "--type=json", "-p="+patch) + Expect(err).NotTo(HaveOccurred()) + + By("verify status reports remained target namespaces only as active") + expectedActiveNs := []string{activeTenantNs} + Eventually( + validateClusterUDNStatusReportsActiveNamespacesFunc(oc.AdminDynamicClient(), testClusterUdnName, expectedActiveNs...), + 1*time.Minute, 3*time.Second).Should(Succeed()) + + removedTenantNs := testTenantNamespaces[0] + By("verify managed NAD not exist in removed target namespace") + Eventually(func() bool { + _, err := nadClient.NetworkAttachmentDefinitions(removedTenantNs).Get(context.Background(), testClusterUdnName, metav1.GetOptions{}) + return err != nil && kerrors.IsNotFound(err) + }, time.Second*300, time.Second*1).Should(BeTrue(), + "NAD in target namespaces should be deleted following CR namespace-selector mutation") + }) + }) + + Context("pod connected to ClusterUserDefinedNetwork", func() { + const testPodName = "test-pod-cluster-udn" + + var ( + udnInUseDeleteTimeout = 65 * time.Second + deleteNetworkTimeout = 5 * time.Second + deleteNetworkInterval = 1 * time.Second + + inUseNetTestTenantNamespace string + ) + + BeforeEach(func() { + inUseNetTestTenantNamespace = defaultNetNamespace.Name + + By("create pod in one of the test tenant namespaces") + networkAttachments := []nadapi.NetworkSelectionElement{ + {Name: testClusterUdnName, Namespace: inUseNetTestTenantNamespace}, + } + cfg := podConfig(testPodName, withNetworkAttachment(networkAttachments)) + cfg.namespace = inUseNetTestTenantNamespace + runUDNPod(cs, inUseNetTestTenantNamespace, *cfg, setRuntimeDefaultPSA) + }) + + It("CR & managed NADs cannot be deleted when being used", func() { + By("verify CR cannot be deleted") + cmd := e2ekubectl.NewKubectlCommand("", "delete", clusterUserDefinedNetworkResource, testClusterUdnName) + cmd.WithTimeout(time.NewTimer(deleteNetworkTimeout).C) + _, err := cmd.Exec() + Expect(err).To(HaveOccurred(), "should fail to delete ClusterUserDefinedNetwork when used") + + By("verify CR associate NAD cannot be deleted") + Eventually(func() error { + ctx, cancel := context.WithTimeout(context.Background(), deleteNetworkTimeout) + defer cancel() + _ = nadClient.NetworkAttachmentDefinitions(inUseNetTestTenantNamespace).Delete(ctx, testClusterUdnName, metav1.DeleteOptions{}) + _, err := nadClient.NetworkAttachmentDefinitions(inUseNetTestTenantNamespace).Get(ctx, testClusterUdnName, metav1.GetOptions{}) + return err + }, udnInUseDeleteTimeout, deleteNetworkInterval).ShouldNot(HaveOccurred(), + "should fail to delete UserDefinedNetwork associated NetworkAttachmentDefinition when used") + + By("verify CR status reports consuming pod") + err = validateClusterUDNStatusReportConsumers(oc.AdminDynamicClient(), testClusterUdnName, inUseNetTestTenantNamespace, testPodName) + Expect(err).NotTo(HaveOccurred()) + + By("delete test pod") + err = cs.CoreV1().Pods(inUseNetTestTenantNamespace).Delete(context.Background(), testPodName, metav1.DeleteOptions{}) + Expect(err).ToNot(HaveOccurred()) + + By("verify CR is gone") + Eventually(func() error { + _, err := e2ekubectl.RunKubectl("", "get", clusterUserDefinedNetworkResource, testClusterUdnName) + return err + }, udnInUseDeleteTimeout, deleteNetworkInterval).Should(HaveOccurred(), + "ClusterUserDefinedNetwork should be deleted following test pod deletion") + + By("verify CR associate NADs are gone") + for _, nsName := range testTenantNamespaces { + Eventually(func() bool { + _, err := nadClient.NetworkAttachmentDefinitions(nsName).Get(context.Background(), testClusterUdnName, metav1.GetOptions{}) + return err != nil && kerrors.IsNotFound(err) + }, deleteNetworkTimeout, deleteNetworkInterval).Should(BeTrue(), + "NADs in target namespaces should be deleted following ClusterUserDefinedNetwork deletion") + } + }) + }) + }) + + It("when primary network exist, ClusterUserDefinedNetwork status should report not-ready", func() { + namespace, err := f.CreateNamespace(context.TODO(), f.BaseName, map[string]string{ + "e2e-framework": f.BaseName, + RequiredUDNNamespaceLabel: "", + }) + Expect(err).NotTo(HaveOccurred()) + f.Namespace = namespace + testTenantNamespaces := []string{ + f.Namespace.Name + "blue", + f.Namespace.Name + "red", + } + By("Creating test tenants namespaces") + for _, nsName := range testTenantNamespaces { + _, err := cs.CoreV1().Namespaces().Create(context.Background(), &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsName, + Labels: map[string]string{RequiredUDNNamespaceLabel: ""}, + }}, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() error { + err := cs.CoreV1().Namespaces().Delete(context.Background(), nsName, metav1.DeleteOptions{}) + return err + }) + } + + By("create primary network NAD in one of the tenant namespaces") + const primaryNadName = "some-primary-net" + primaryNetTenantNs := testTenantNamespaces[0] + primaryNetNad := generateNAD(newNetworkAttachmentConfig(networkAttachmentConfigParams{ + role: "primary", + topology: "layer3", + name: primaryNadName, + networkName: primaryNadName, + cidr: correctCIDRFamily(oc, userDefinedNetworkIPv4Subnet, userDefinedNetworkIPv6Subnet), + })) + _, err = nadClient.NetworkAttachmentDefinitions(primaryNetTenantNs).Create(context.Background(), primaryNetNad, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) - var actualConditions []metav1.Condition - Expect(json.Unmarshal([]byte(conditionsJSON), &actualConditions)).To(Succeed()) - Expect(actualConditions[0].Type).To(Equal("NetworkReady")) - Expect(actualConditions[0].Status).To(Equal(metav1.ConditionFalse)) - Expect(actualConditions[0].Reason).To(Equal("SyncError")) - expectedMessage := fmt.Sprintf("primary network already exist in namespace %q: %q", f.Namespace.Name, primaryNadName) - Expect(actualConditions[0].Message).To(Equal(expectedMessage)) + By("create primary Cluster UDN CR") + cudnName := randomNetworkMetaName() + cleanup, err := createManifest(f.Namespace.Name, newPrimaryClusterUDNManifest(oc, cudnName, testTenantNamespaces...)) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + cleanup() + _, err := e2ekubectl.RunKubectl("", "delete", "clusteruserdefinednetwork", cudnName, "--wait", fmt.Sprintf("--timeout=%ds", 60)) + Expect(err).NotTo(HaveOccurred()) + }) + + expectedMessage := fmt.Sprintf("primary network already exist in namespace %q: %q", primaryNetTenantNs, primaryNadName) + Eventually(func(g Gomega) []metav1.Condition { + conditionsJSON, err := e2ekubectl.RunKubectl(f.Namespace.Name, "get", "clusteruserdefinednetwork", cudnName, "-o", "jsonpath={.status.conditions}") + g.Expect(err).NotTo(HaveOccurred()) + var actualConditions []metav1.Condition + g.Expect(json.Unmarshal([]byte(conditionsJSON), &actualConditions)).To(Succeed()) + return normalizeConditions(actualConditions) + }, 5*time.Second, 1*time.Second).Should(SatisfyAny( + ConsistOf(metav1.Condition{ + Type: "NetworkReady", + Status: metav1.ConditionFalse, + Reason: "NetworkAttachmentDefinitionSyncError", + Message: expectedMessage, + }), + ConsistOf(metav1.Condition{ + Type: "NetworkCreated", + Status: metav1.ConditionFalse, + Reason: "NetworkAttachmentDefinitionSyncError", + Message: expectedMessage, + }), + )) + }) + + Context("UDN Pod", func() { + const ( + testUdnName = "test-net" + testPodName = "test-pod-udn" + ) + + var udnPod *v1.Pod + + BeforeEach(func() { + l := map[string]string{ + "e2e-framework": f.BaseName, + RequiredUDNNamespaceLabel: "", + } + ns, err := f.CreateNamespace(context.TODO(), f.BaseName, l) + Expect(err).NotTo(HaveOccurred()) + f.Namespace = ns + By("create tests UserDefinedNetwork") + cleanup, err := createManifest(f.Namespace.Name, newPrimaryUserDefinedNetworkManifest(oc, testUdnName)) + DeferCleanup(cleanup) + Expect(err).NotTo(HaveOccurred()) + Eventually(userDefinedNetworkReadyFunc(oc.AdminDynamicClient(), f.Namespace.Name, testUdnName), udnCrReadyTimeout, time.Second).Should(Succeed()) + By("create UDN pod") + cfg := podConfig(testPodName, withCommand(func() []string { + return httpServerContainerCmd(port) + })) + cfg.namespace = f.Namespace.Name + udnPod = runUDNPod(cs, f.Namespace.Name, *cfg, nil) + }) + + It("should react to k8s.ovn.org/open-default-ports annotations changes", func() { + By("Creating second namespace for default network pod") + defaultNetNamespace := f.Namespace.Name + "-default" + _, err := cs.CoreV1().Namespaces().Create(context.Background(), &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultNetNamespace, + }, + }, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + defer func() { + Expect(cs.CoreV1().Namespaces().Delete(context.Background(), defaultNetNamespace, metav1.DeleteOptions{})).To(Succeed()) + }() + + By("creating default network client pod") + defaultClientPod := frameworkpod.CreateExecPodOrFail( + context.Background(), + f.ClientSet, + defaultNetNamespace, + "default-net-client-pod", + func(pod *v1.Pod) { + pod.Spec.Containers[0].Args = []string{"netexec"} + setRuntimeDefaultPSA(pod) + }, + ) + + udnIPv4, udnIPv6, err := podIPsForDefaultNetwork( + cs, + f.Namespace.Name, + udnPod.GetName(), + ) + Expect(err).NotTo(HaveOccurred()) + + By(fmt.Sprintf("verify default network client pod can't access UDN pod on port %d", port)) + for _, destIP := range []string{udnIPv4, udnIPv6} { + if destIP == "" { + continue + } + By("checking the default network pod can't reach UDN pod on IP " + destIP) + Consistently(func() bool { + return connectToServer(podConfiguration{namespace: defaultClientPod.Namespace, name: defaultClientPod.Name}, destIP, port) != nil + }, serverConnectPollTimeout, serverConnectPollInterval).Should(BeTrue()) + } + + By("Open UDN pod port") + udnPod.Annotations[openDefaultPortsAnnotation] = fmt.Sprintf( + `- protocol: tcp + port: %d`, port) + udnPod, err = cs.CoreV1().Pods(udnPod.Namespace).Update(context.Background(), udnPod, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By(fmt.Sprintf("verify default network client pod can access UDN pod on open port %d", port)) + for _, destIP := range []string{udnIPv4, udnIPv6} { + if destIP == "" { + continue + } + By("checking the default network pod can reach UDN pod on IP " + destIP) + Eventually(func() bool { + return connectToServer(podConfiguration{namespace: defaultClientPod.Namespace, name: defaultClientPod.Name}, destIP, port) == nil + }, serverConnectPollTimeout, serverConnectPollInterval).Should(BeTrue()) + } + + By("Update UDN pod port with the wrong syntax") + // this should clean up open ports and throw an event + udnPod.Annotations[openDefaultPortsAnnotation] = fmt.Sprintf( + `- protocol: ppp + port: %d`, port) + udnPod, err = cs.CoreV1().Pods(udnPod.Namespace).Update(context.Background(), udnPod, metav1.UpdateOptions{}) + Expect(err).NotTo(HaveOccurred()) + + By(fmt.Sprintf("verify default network client pod can't access UDN pod on port %d", port)) + for _, destIP := range []string{udnIPv4, udnIPv6} { + if destIP == "" { + continue + } + By("checking the default network pod can't reach UDN pod on IP " + destIP) + Eventually(func() bool { + return connectToServer(podConfiguration{namespace: defaultClientPod.Namespace, name: defaultClientPod.Name}, destIP, port) != nil + }, serverConnectPollTimeout, serverConnectPollInterval).Should(BeTrue()) + } + By("Verify syntax error is reported via event") + events, err := cs.CoreV1().Events(udnPod.Namespace).List(context.Background(), metav1.ListOptions{}) + Expect(err).NotTo(HaveOccurred()) + found := false + for _, event := range events.Items { + if event.Reason == "ErrorUpdatingResource" && strings.Contains(event.Message, "invalid protocol ppp") { + found = true + break + } + } + Expect(found).To(BeTrue(), "should have found an event for invalid protocol") + }) }) }) }) +// randomNetworkMetaName return pseudo random name for network related objects (NAD,UDN,CUDN). +// CUDN is cluster-scoped object, in case tests running in parallel, having random names avoids +// conflicting with other tests. +func randomNetworkMetaName() string { + return fmt.Sprintf("test-net-%s", rand.String(5)) +} + +var nadToUdnParams = map[string]string{ + "primary": "Primary", + "secondary": "Secondary", + "layer2": "Layer2", + "layer3": "Layer3", +} + func generateUserDefinedNetworkManifest(params *networkAttachmentConfigParams) string { - nadToUdnParams := map[string]string{ - "primary": "Primary", - "secondary": "Secondary", - "layer2": "Layer2", - "layer3": "Layer3", - } subnets := generateSubnetsYaml(params) return ` apiVersion: k8s.ovn.org/v1 @@ -707,6 +1298,27 @@ spec: ` } +func generateClusterUserDefinedNetworkManifest(params *networkAttachmentConfigParams) string { + subnets := generateSubnetsYaml(params) + return ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: ` + params.name + ` +spec: + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: [` + params.namespace + `] + network: + topology: ` + nadToUdnParams[params.topology] + ` + ` + params.topology + `: + role: ` + nadToUdnParams[params.role] + ` + subnets: ` + subnets + ` +` +} + func generateSubnetsYaml(params *networkAttachmentConfigParams) string { if params.topology == "layer3" { l3Subnets := generateLayer3Subnets(params.cidr) @@ -767,9 +1379,85 @@ func applyManifest(namespace, manifest string) error { return err } -func waitForUserDefinedNetworkReady(namespace, name string, timeout time.Duration) error { - _, err := e2ekubectl.RunKubectl(namespace, "wait", "userdefinednetwork", name, "--for", "condition=NetworkReady=True", "--timeout", timeout.String()) - return err +var clusterUDNGVR = schema.GroupVersionResource{ + Group: "k8s.ovn.org", + Version: "v1", + Resource: "clusteruserdefinednetworks", +} + +var udnGVR = schema.GroupVersionResource{ + Group: "k8s.ovn.org", + Version: "v1", + Resource: "userdefinednetworks", +} + +// getConditions extracts metav1 conditions from .status.conditions of an unstructured object +func getConditions(uns *unstructured.Unstructured) ([]metav1.Condition, error) { + var conditions []metav1.Condition + conditionsRaw, found, err := unstructured.NestedFieldNoCopy(uns.Object, "status", "conditions") + if err != nil { + return nil, fmt.Errorf("failed getting conditions in %s: %v", uns.GetName(), err) + } + if !found { + return nil, fmt.Errorf("conditions not found in %v", uns) + } + + conditionsJSON, err := json.Marshal(conditionsRaw) + if err != nil { + return nil, err + } + if err := json.Unmarshal(conditionsJSON, &conditions); err != nil { + return nil, err + } + + return conditions, nil +} + +// userDefinedNetworkReadyFunc returns a function that checks for the NetworkCreated/NetworkReady condition in the provided udn +func userDefinedNetworkReadyFunc(client dynamic.Interface, namespace, name string) func() error { + return func() error { + udn, err := client.Resource(udnGVR).Namespace(namespace).Get(context.Background(), name, metav1.GetOptions{}, "status") + if err != nil { + return err + } + conditions, err := getConditions(udn) + if err != nil { + return err + } + if len(conditions) == 0 { + return fmt.Errorf("no conditions found in: %v", udn) + } + for _, udnCondition := range conditions { + if (udnCondition.Type == "NetworkCreated" || udnCondition.Type == "NetworkReady") && udnCondition.Status == metav1.ConditionTrue { + return nil + } + + } + return fmt.Errorf("no NetworkCreated/NetworkReady condition found in: %v", udn) + } +} + +// userDefinedNetworkReadyFunc returns a function that checks for the NetworkCreated/NetworkReady condition in the provided cluster udn +func clusterUserDefinedNetworkReadyFunc(client dynamic.Interface, name string) func() error { + return func() error { + cUDN, err := client.Resource(clusterUDNGVR).Get(context.Background(), name, metav1.GetOptions{}, "status") + if err != nil { + return err + } + conditions, err := getConditions(cUDN) + if err != nil { + return err + } + if len(conditions) == 0 { + return fmt.Errorf("no conditions found in: %v", cUDN) + } + for _, cUDNCondition := range conditions { + if (cUDNCondition.Type == "NetworkCreated" || cUDNCondition.Type == "NetworkReady") && cUDNCondition.Status == metav1.ConditionTrue { + return nil + } + } + return fmt.Errorf("no NetworkCreated/NetworkReady condition found in: %v", cUDN) + } } func newPrimaryUserDefinedNetworkManifest(oc *exutil.CLI, name string) string { @@ -845,28 +1533,36 @@ func assertNetAttachDefManifest(nadClient nadclient.K8sCniCncfIoV1Interface, nam }`)) } -func assertUDNStatusReportsConsumers(udnNamesapce, udnName, expectedPodName string) { - conditionsRaw, err := e2ekubectl.RunKubectl(udnNamesapce, "get", "userdefinednetwork", udnName, "-o", "jsonpath='{.status.conditions}'") - Expect(err).NotTo(HaveOccurred()) - conditionsRaw = strings.ReplaceAll(conditionsRaw, `\`, ``) - conditionsRaw = strings.ReplaceAll(conditionsRaw, `'`, ``) - var conditions []metav1.Condition - Expect(json.Unmarshal([]byte(conditionsRaw), &conditions)).To(Succeed()) +func validateUDNStatusReportsConsumers(client dynamic.Interface, udnNamesapce, udnName, expectedPodName string) error { + udn, err := client.Resource(udnGVR).Namespace(udnNamesapce).Get(context.Background(), udnName, metav1.GetOptions{}) + if err != nil { + return err + } + conditions, err := getConditions(udn) + if err != nil { + return err + } conditions = normalizeConditions(conditions) expectedMsg := fmt.Sprintf("failed to delete NetworkAttachmentDefinition [%[1]s/%[2]s]: network in use by the following pods: [%[1]s/%[3]s]", udnNamesapce, udnName, expectedPodName) - found := false - for _, condition := range conditions { - if found, _ = Equal(metav1.Condition{ - Type: "NetworkReady", - Status: "False", - Reason: "SyncError", - Message: expectedMsg, - }).Match(condition); found { - break + networkReadyCondition := metav1.Condition{ + Type: "NetworkReady", + Status: metav1.ConditionFalse, + Reason: "SyncError", + Message: expectedMsg, + } + networkCreatedCondition := metav1.Condition{ + Type: "NetworkCreated", + Status: metav1.ConditionFalse, + Reason: "SyncError", + Message: expectedMsg, + } + for _, udnCondition := range conditions { + if udnCondition == networkReadyCondition || udnCondition == networkCreatedCondition { + return nil } } - Expect(found).To(BeTrue(), "expected condition not found in %v", conditions) + return fmt.Errorf("failed to find NetworkCreated/NetworkReady condition in %v", conditions) } func normalizeConditions(conditions []metav1.Condition) []metav1.Condition { @@ -877,6 +1573,158 @@ func normalizeConditions(conditions []metav1.Condition) []metav1.Condition { return conditions } +func assertClusterNADManifest(nadClient nadclient.K8sCniCncfIoV1Interface, namespace, udnName, udnUID string) { + nad, err := nadClient.NetworkAttachmentDefinitions(namespace).Get(context.Background(), udnName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + ExpectWithOffset(1, nad.Name).To(Equal(udnName)) + ExpectWithOffset(1, nad.Namespace).To(Equal(namespace)) + ExpectWithOffset(1, nad.OwnerReferences).To(Equal([]metav1.OwnerReference{{ + APIVersion: "k8s.ovn.org/v1", + Kind: "ClusterUserDefinedNetwork", + Name: udnName, + UID: types.UID(udnUID), + BlockOwnerDeletion: pointer.Bool(true), + Controller: pointer.Bool(true), + }})) + ExpectWithOffset(1, nad.Labels).To(Equal(map[string]string{"k8s.ovn.org/user-defined-network": ""})) + ExpectWithOffset(1, nad.Finalizers).To(Equal([]string{"k8s.ovn.org/user-defined-network-protection"})) + + expectedNetworkName := "cluster.udn." + udnName + expectedNadName := namespace + "/" + udnName + ExpectWithOffset(1, nad.Spec.Config).To(MatchJSON(`{ + "cniVersion":"1.0.0", + "type": "ovn-k8s-cni-overlay", + "name": "` + expectedNetworkName + `", + "netAttachDefName": "` + expectedNadName + `", + "topology": "layer2", + "role": "secondary", + "subnets": "10.100.0.0/16" + }`)) +} + +func validateClusterUDNStatusReportsActiveNamespacesFunc(client dynamic.Interface, cUDNName string, expectedActiveNsNames ...string) func() error { + return func() error { + cUDN, err := client.Resource(clusterUDNGVR).Get(context.Background(), cUDNName, metav1.GetOptions{}) + if err != nil { + return err + } + conditions, err := getConditions(cUDN) + if err != nil { + return err + } + if len(conditions) == 0 { + return fmt.Errorf("expected at least one condition in %v", cUDN) + } + + c := conditions[0] + if c.Type != "NetworkCreated" && c.Type != "NetworkReady" { + return fmt.Errorf("expected NetworkCreated/NetworkReady type in %v", c) + } + if c.Status != metav1.ConditionTrue { + return fmt.Errorf("expected True status in %v", c) + } + if c.Reason != "NetworkAttachmentDefinitionCreated" && c.Reason != "NetworkAttachmentDefinitionReady" { + return fmt.Errorf("expected NetworkAttachmentDefinitionCreated/NetworkAttachmentDefinitionReady reason in %v", c) + } + if !strings.Contains(c.Message, "NetworkAttachmentDefinition has been created in following namespaces:") { + return fmt.Errorf("expected \"NetworkAttachmentDefinition has been created in following namespaces:\" in %s", c.Message) + } + + for _, ns := range expectedActiveNsNames { + if !strings.Contains(c.Message, ns) { + return fmt.Errorf("expected to find %q namespace in %s", ns, c.Message) + } + } + return nil + } +} + +func validateClusterUDNStatusReportConsumers(client dynamic.Interface, cUDNName, udnNamespace, expectedPodName string) error { + cUDN, err := client.Resource(clusterUDNGVR).Get(context.Background(), cUDNName, metav1.GetOptions{}) + if err != nil { + return err + } + conditions, err := getConditions(cUDN) + if err != nil { + return err + } + conditions = normalizeConditions(conditions) + expectedMsg := fmt.Sprintf("failed to delete NetworkAttachmentDefinition [%[1]s/%[2]s]: network in use by the following pods: [%[1]s/%[3]s]", + udnNamespace, cUDNName, expectedPodName) + networkCreatedCondition := metav1.Condition{ + Type: "NetworkCreated", + Status: metav1.ConditionFalse, + Reason: "NetworkAttachmentDefinitionSyncError", + Message: expectedMsg, + } + networkReadyCondition := metav1.Condition{ + Type: "NetworkReady", + Status: metav1.ConditionFalse, + Reason: "NetworkAttachmentDefinitionSyncError", + Message: expectedMsg, + } + for _, clusterUDNCondition := range conditions { + if clusterUDNCondition == networkCreatedCondition || clusterUDNCondition == networkReadyCondition { + return nil + } + } + return fmt.Errorf("failed to find NetworkCreated/NetworkReady condition in %v", conditions) +} + +func newClusterUDNManifest(name string, targetNamespaces ...string) string { + targetNs := strings.Join(targetNamespaces, ",") + return ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: ` + name + ` +spec: + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: [ ` + targetNs + ` ] + network: + topology: Layer2 + layer2: + role: Secondary + subnets: ["10.100.0.0/16"] +` +} + +func newPrimaryClusterUDNManifest(oc *exutil.CLI, name string, targetNamespaces ...string) string { + targetNs := strings.Join(targetNamespaces, ",") + return ` +apiVersion: k8s.ovn.org/v1 +kind: ClusterUserDefinedNetwork +metadata: + name: ` + name + ` +spec: + namespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: [ ` + targetNs + ` ] + network: + topology: Layer3 + layer3: + role: Primary + subnets: ` + generateCIDRforClusterUDN(oc) +} + +func generateCIDRforClusterUDN(oc *exutil.CLI) string { + hasIPv4, hasIPv6, err := GetIPAddressFamily(oc) + Expect(err).NotTo(HaveOccurred()) + cidr := `[{cidr: "203.203.0.0/16"}]` + if hasIPv6 && hasIPv4 { + cidr = `[{cidr: "203.203.0.0/16"},{cidr: "2014:100:200::0/60"}]` + } else if hasIPv6 { + cidr = `[{cidr: "2014:100:200::0/60"}]` + } + return cidr +} + func setRuntimeDefaultPSA(pod *v1.Pod) { dontEscape := false noRoot := true @@ -912,6 +1760,12 @@ func withCommand(cmdGenerationFn func() []string) podOption { } } +func withLabels(labels map[string]string) podOption { + return func(pod *podConfiguration) { + pod.labels = labels + } +} + func withNetworkAttachment(networks []nadapi.NetworkSelectionElement) podOption { return func(pod *podConfiguration) { pod.attachments = networks @@ -978,7 +1832,7 @@ func runUDNPod(cs clientset.Interface, namespace string, podConfig podConfigurat return v1.PodFailed } return updatedPod.Status.Phase - }, 2*time.Minute, 6*time.Second).Should(Equal(v1.PodRunning)) + }, podReadyPollTimeout, podReadyPollInterval).Should(Equal(v1.PodRunning)) return updatedPod } @@ -1135,17 +1989,21 @@ func inRange(cidr string, ip string) error { } func connectToServer(clientPodConfig podConfiguration, serverIP string, port int) error { - _, err := e2ekubectl.RunKubectl( - clientPodConfig.namespace, + _, err := connectToServerWithPath(clientPodConfig.namespace, clientPodConfig.name, serverIP, "" /* no path */, port) + return err +} + +func connectToServerWithPath(podNamespace, podName, serverIP, path string, port int) (string, error) { + return e2ekubectl.RunKubectl( + podNamespace, "exec", - clientPodConfig.name, + podName, "--", "curl", "--connect-timeout", "2", - net.JoinHostPort(serverIP, fmt.Sprintf("%d", port)), + net.JoinHostPort(serverIP, fmt.Sprintf("%d", port))+path, ) - return err } // Returns pod's ipv4 and ipv6 addresses IN ORDER diff --git a/test/extended/networking/network_segmentation_endpointslice_mirror.go b/test/extended/networking/network_segmentation_endpointslice_mirror.go index f3d8fb8529e0..11652fa51980 100644 --- a/test/extended/networking/network_segmentation_endpointslice_mirror.go +++ b/test/extended/networking/network_segmentation_endpointslice_mirror.go @@ -27,9 +27,10 @@ import ( var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks] EndpointSlices mirroring", func() { defer GinkgoRecover() - - oc := exutil.NewCLIWithPodSecurityLevel("endpointslices-mirror-e2e", admissionapi.LevelPrivileged) + // disable automatic namespace creation, we need to add the required UDN label + oc := exutil.NewCLIWithoutNamespace("endpointslices-mirror-e2e") f := oc.KubeFramework() + f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged InOVNKubernetesContext(func() { const ( userDefinedNetworkIPv4Subnet = "203.203.0.0/16" @@ -44,8 +45,12 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User BeforeEach(func() { cs = f.ClientSet - - var err error + namespace, err := f.CreateNamespace(context.TODO(), f.BaseName, map[string]string{ + "e2e-framework": f.BaseName, + RequiredUDNNamespaceLabel: "", + }) + f.Namespace = namespace + Expect(err).NotTo(HaveOccurred()) nadClient, err = nadclient.NewForConfig(f.ClientConfig()) Expect(err).NotTo(HaveOccurred()) }) @@ -168,7 +173,7 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User udnManifest := generateUserDefinedNetworkManifest(&c) cleanup, err := createManifest(f.Namespace.Name, udnManifest) DeferCleanup(cleanup) - Expect(waitForUserDefinedNetworkReady(f.Namespace.Name, c.name, 5*time.Second)).To(Succeed()) + Eventually(userDefinedNetworkReadyFunc(oc.AdminDynamicClient(), f.Namespace.Name, c.name), 5*time.Second, time.Second).Should(Succeed()) return err }), ) @@ -180,16 +185,23 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User func( netConfig networkAttachmentConfigParams, ) { + netConfig.cidr = correctCIDRFamily(oc, userDefinedNetworkIPv4Subnet, userDefinedNetworkIPv6Subnet) + By("creating default net namespace") + defaultNSName := f.BaseName + "-default" + defaultNetNamespace, err := f.CreateNamespace(context.TODO(), defaultNSName, map[string]string{ + "e2e-framework": defaultNSName, + }) + Expect(err).NotTo(HaveOccurred()) By("creating the network") - netConfig.namespace = f.Namespace.Name + netConfig.namespace = defaultNetNamespace.Name Expect(createNetworkFn(netConfig)).To(Succeed()) By("deploying the backend pods") replicas := 3 for i := 0; i < replicas; i++ { - runUDNPod(cs, f.Namespace.Name, + runUDNPod(cs, defaultNetNamespace.Name, *podConfig(fmt.Sprintf("backend-%d", i), func(cfg *podConfiguration) { - cfg.namespace = f.Namespace.Name + cfg.namespace = defaultNetNamespace.Name // Add the net-attach annotation for secondary networks if netConfig.role == "secondary" { cfg.attachments = []nadapi.NetworkSelectionElement{{Name: netConfig.name}} @@ -208,12 +220,12 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User svc := e2eservice.CreateServiceSpec("test-service", "", false, map[string]string{"app": "test"}) familyPolicy := corev1.IPFamilyPolicyPreferDualStack svc.Spec.IPFamilyPolicy = &familyPolicy - _, err := cs.CoreV1().Services(f.Namespace.Name).Create(context.Background(), svc, metav1.CreateOptions{}) + _, err = cs.CoreV1().Services(defaultNetNamespace.Name).Create(context.Background(), svc, metav1.CreateOptions{}) framework.ExpectNoError(err, "Failed creating service %v", err) By("asserting the mirrored EndpointSlice does not exist") Eventually(func() error { - esList, err := cs.DiscoveryV1().EndpointSlices(f.Namespace.Name).List(context.TODO(), metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", "k8s.ovn.org/service-name", svc.Name)}) + esList, err := cs.DiscoveryV1().EndpointSlices(defaultNetNamespace.Name).List(context.TODO(), metav1.ListOptions{LabelSelector: fmt.Sprintf("%s=%s", "k8s.ovn.org/service-name", svc.Name)}) if err != nil { return err } @@ -229,7 +241,6 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User networkAttachmentConfigParams{ name: nadName, topology: "layer2", - cidr: fmt.Sprintf("%s,%s", userDefinedNetworkIPv4Subnet, userDefinedNetworkIPv6Subnet), role: "secondary", }, ), @@ -238,7 +249,6 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User networkAttachmentConfigParams{ name: nadName, topology: "layer3", - cidr: fmt.Sprintf("%s,%s", userDefinedNetworkIPv4Subnet, userDefinedNetworkIPv6Subnet), role: "secondary", }, ), @@ -247,14 +257,14 @@ var _ = Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:User Entry("NetworkAttachmentDefinitions", func(c networkAttachmentConfigParams) error { netConfig := newNetworkAttachmentConfig(c) nad := generateNAD(netConfig) - _, err := nadClient.NetworkAttachmentDefinitions(f.Namespace.Name).Create(context.Background(), nad, metav1.CreateOptions{}) + _, err := nadClient.NetworkAttachmentDefinitions(c.namespace).Create(context.Background(), nad, metav1.CreateOptions{}) return err }), Entry("UserDefinedNetwork", func(c networkAttachmentConfigParams) error { udnManifest := generateUserDefinedNetworkManifest(&c) - cleanup, err := createManifest(f.Namespace.Name, udnManifest) + cleanup, err := createManifest(c.namespace, udnManifest) DeferCleanup(cleanup) - Expect(waitForUserDefinedNetworkReady(f.Namespace.Name, c.name, 5*time.Second)).To(Succeed()) + Eventually(userDefinedNetworkReadyFunc(oc.AdminDynamicClient(), c.namespace, c.name), 5*time.Second, time.Second).Should(Succeed()) return err }), ) diff --git a/test/extended/networking/network_segmentation_policy.go b/test/extended/networking/network_segmentation_policy.go new file mode 100644 index 000000000000..786881e9abce --- /dev/null +++ b/test/extended/networking/network_segmentation_policy.go @@ -0,0 +1,340 @@ +package networking + +import ( + "context" + "fmt" + "net" + "strings" + + nadclient "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/typed/k8s.cni.cncf.io/v1" + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + + v1 "k8s.io/api/core/v1" + knet "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/rand" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + admissionapi "k8s.io/pod-security-admission/api" +) + +var _ = ginkgo.Describe("[sig-network][OCPFeatureGate:NetworkSegmentation][Feature:UserDefinedPrimaryNetworks] Network Policies", func() { + defer ginkgo.GinkgoRecover() + + // disable automatic namespace creation, we need to add the required UDN label + oc := exutil.NewCLIWithoutNamespace("network-segmentation-policy-e2e") + f := oc.KubeFramework() + f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged + InOVNKubernetesContext(func() { + const ( + nodeHostnameKey = "kubernetes.io/hostname" + nadName = "tenant-red" + userDefinedNetworkIPv4Subnet = "203.203.0.0/16" + userDefinedNetworkIPv6Subnet = "2014:100:200::0/60" + port = 9000 + randomStringLength = 5 + nameSpaceYellowSuffix = "yellow" + namespaceBlueSuffix = "blue" + ) + + var ( + cs clientset.Interface + nadClient nadclient.K8sCniCncfIoV1Interface + allowServerPodLabel = map[string]string{"foo": "bar"} + denyServerPodLabel = map[string]string{"abc": "xyz"} + ) + + ginkgo.BeforeEach(func() { + cs = f.ClientSet + namespace, err := f.CreateNamespace(context.TODO(), f.BaseName, map[string]string{ + "e2e-framework": f.BaseName, + RequiredUDNNamespaceLabel: "", + }) + f.Namespace = namespace + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + nadClient, err = nadclient.NewForConfig(f.ClientConfig()) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + namespaceYellow := getNamespaceName(f, nameSpaceYellowSuffix) + namespaceBlue := getNamespaceName(f, namespaceBlueSuffix) + for _, namespace := range []string{namespaceYellow, namespaceBlue} { + ginkgo.By("Creating namespace " + namespace) + ns, err := cs.CoreV1().Namespaces().Create(context.Background(), &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + Labels: map[string]string{RequiredUDNNamespaceLabel: ""}, + }, + }, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + f.AddNamespacesToDelete(ns) + } + }) + + ginkgo.AfterEach(func() { + if ginkgo.CurrentSpecReport().Failed() { + exutil.DumpPodStatesInNamespace(f.Namespace.Name, oc) + exutil.DumpPodStatesInNamespace(getNamespaceName(f, nameSpaceYellowSuffix), oc) + exutil.DumpPodStatesInNamespace(getNamespaceName(f, namespaceBlueSuffix), oc) + } + }) + + ginkgo.DescribeTable( + "pods within namespace should be isolated when deny policy is present", + func( + topology string, + clientPodConfig podConfiguration, + serverPodConfig podConfiguration, + ) { + ginkgo.By("Creating the attachment configuration") + netConfig := newNetworkAttachmentConfig(networkAttachmentConfigParams{ + name: nadName, + topology: topology, + cidr: correctCIDRFamily(oc, userDefinedNetworkIPv4Subnet, userDefinedNetworkIPv6Subnet), + role: "primary", + }) + netConfig.namespace = f.Namespace.Name + _, err := nadClient.NetworkAttachmentDefinitions(f.Namespace.Name).Create( + context.Background(), + generateNAD(netConfig), + metav1.CreateOptions{}, + ) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + workerNodes, err := getWorkerNodesOrdered(cs) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(len(workerNodes)).To(gomega.BeNumerically(">=", 1)) + + ginkgo.By("creating client/server pods") + clientPodConfig.namespace = f.Namespace.Name + clientPodConfig.nodeSelector = map[string]string{nodeHostnameKey: workerNodes[0].Name} + serverPodConfig.namespace = f.Namespace.Name + serverPodConfig.nodeSelector = map[string]string{nodeHostnameKey: workerNodes[len(workerNodes)-1].Name} + runUDNPod(cs, f.Namespace.Name, serverPodConfig, nil) + runUDNPod(cs, f.Namespace.Name, clientPodConfig, nil) + + var serverIP string + for i, cidr := range strings.Split(netConfig.cidr, ",") { + if cidr != "" { + ginkgo.By("asserting the server pod has an IP from the configured range") + serverIP, err = podIPsForUserDefinedPrimaryNetwork( + cs, + f.Namespace.Name, + serverPodConfig.name, + namespacedName(f.Namespace.Name, netConfig.name), + i, + ) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By(fmt.Sprintf("asserting the server pod IP %v is from the configured range %v", serverIP, cidr)) + subnet, err := getNetCIDRSubnet(cidr) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(inRange(subnet, serverIP)).To(gomega.Succeed()) + } + + ginkgo.By("asserting the *client* pod can contact the server pod exposed endpoint") + namespacePodShouldReach(oc, f.Namespace.Name, clientPodConfig.name, formatHostAndPort(net.ParseIP(serverIP), port)) + } + + ginkgo.By("creating a \"default deny\" network policy") + _, err = makeDenyAllPolicy(f, f.Namespace.Name) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("asserting the *client* pod can not contact the server pod exposed endpoint") + podShouldNotReach(oc, clientPodConfig.name, formatHostAndPort(net.ParseIP(serverIP), port)) + + }, + ginkgo.Entry( + "in L2 dualstack primary UDN", + "layer2", + *podConfig( + "client-pod", + ), + *podConfig( + "server-pod", + withCommand(func() []string { + return httpServerContainerCmd(port) + }), + ), + ), + ginkgo.Entry( + "in L3 dualstack primary UDN", + "layer3", + *podConfig( + "client-pod", + ), + *podConfig( + "server-pod", + withCommand(func() []string { + return httpServerContainerCmd(port) + }), + ), + ), + ) + + ginkgo.DescribeTable( + "allow ingress traffic to one pod from a particular namespace", + func( + topology string, + clientPodConfig podConfiguration, + allowServerPodConfig podConfiguration, + denyServerPodConfig podConfiguration, + ) { + + namespaceYellow := getNamespaceName(f, nameSpaceYellowSuffix) + namespaceBlue := getNamespaceName(f, namespaceBlueSuffix) + + nad := networkAttachmentConfigParams{ + topology: topology, + cidr: correctCIDRFamily(oc, userDefinedNetworkIPv4Subnet, userDefinedNetworkIPv6Subnet), + // Both yellow and blue namespaces are going to served by green network. + // Use random suffix for the network name to avoid race between tests. + networkName: fmt.Sprintf("%s-%s", "green", rand.String(randomStringLength)), + role: "primary", + } + + // Use random suffix in net conf name to avoid race between tests. + netConfName := fmt.Sprintf("sharednet-%s", rand.String(randomStringLength)) + for _, namespace := range []string{namespaceYellow, namespaceBlue} { + ginkgo.By("creating the attachment configuration for " + netConfName + " in namespace " + namespace) + netConfig := newNetworkAttachmentConfig(nad) + netConfig.namespace = namespace + netConfig.name = netConfName + + _, err := nadClient.NetworkAttachmentDefinitions(namespace).Create( + context.Background(), + generateNAD(netConfig), + metav1.CreateOptions{}, + ) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + } + + workerNodes, err := getWorkerNodesOrdered(cs) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(len(workerNodes)).To(gomega.BeNumerically(">=", 1)) + + ginkgo.By("creating client/server pods") + allowServerPodConfig.namespace = namespaceYellow + allowServerPodConfig.nodeSelector = map[string]string{nodeHostnameKey: workerNodes[len(workerNodes)-1].Name} + denyServerPodConfig.namespace = namespaceYellow + denyServerPodConfig.nodeSelector = map[string]string{nodeHostnameKey: workerNodes[len(workerNodes)-1].Name} + clientPodConfig.namespace = namespaceBlue + clientPodConfig.nodeSelector = map[string]string{nodeHostnameKey: workerNodes[0].Name} + runUDNPod(cs, namespaceYellow, allowServerPodConfig, func(pod *v1.Pod) { + setRuntimeDefaultPSA(pod) + }) + runUDNPod(cs, namespaceYellow, denyServerPodConfig, func(pod *v1.Pod) { + setRuntimeDefaultPSA(pod) + }) + runUDNPod(cs, namespaceBlue, clientPodConfig, func(pod *v1.Pod) { + setRuntimeDefaultPSA(pod) + }) + + ginkgo.By("asserting the server pods have an IP from the configured range") + var allowServerPodIP, denyServerPodIP string + for i, cidr := range strings.Split(nad.cidr, ",") { + if cidr != "" { + subnet, err := getNetCIDRSubnet(cidr) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + allowServerPodIP, err = podIPsForUserDefinedPrimaryNetwork(cs, namespaceYellow, allowServerPodConfig.name, + namespacedName(namespaceYellow, netConfName), i) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By(fmt.Sprintf("asserting the allow server pod IP %v is from the configured range %v", allowServerPodIP, cidr)) + gomega.Expect(inRange(subnet, allowServerPodIP)).To(gomega.Succeed()) + denyServerPodIP, err = podIPsForUserDefinedPrimaryNetwork(cs, namespaceYellow, denyServerPodConfig.name, + namespacedName(namespaceYellow, netConfName), i) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + ginkgo.By(fmt.Sprintf("asserting the deny server pod IP %v is from the configured range %v", denyServerPodIP, cidr)) + gomega.Expect(inRange(subnet, denyServerPodIP)).To(gomega.Succeed()) + } + } + + ginkgo.By("asserting the *client* pod can contact the allow server pod exposed endpoint") + namespacePodShouldReach(oc, clientPodConfig.namespace, clientPodConfig.name, formatHostAndPort(net.ParseIP(allowServerPodIP), port)) + + ginkgo.By("asserting the *client* pod can contact the deny server pod exposed endpoint") + namespacePodShouldReach(oc, clientPodConfig.namespace, clientPodConfig.name, formatHostAndPort(net.ParseIP(denyServerPodIP), port)) + + ginkgo.By("creating a \"default deny\" network policy") + _, err = makeDenyAllPolicy(f, namespaceYellow) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("asserting the *client* pod can not contact the allow server pod exposed endpoint") + namespacePodShouldNotReach(oc, clientPodConfig.namespace, clientPodConfig.name, formatHostAndPort(net.ParseIP(allowServerPodIP), port)) + + ginkgo.By("asserting the *client* pod can not contact the deny server pod exposed endpoint") + namespacePodShouldNotReach(oc, clientPodConfig.namespace, clientPodConfig.name, formatHostAndPort(net.ParseIP(denyServerPodIP), port)) + + ginkgo.By("creating a \"allow-traffic-to-pod\" network policy") + _, err = allowTrafficToPodFromNamespacePolicy(f, namespaceYellow, namespaceBlue, "allow-traffic-to-pod", allowServerPodLabel) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By("asserting the *client* pod can contact the allow server pod exposed endpoint") + namespacePodShouldReach(oc, clientPodConfig.namespace, clientPodConfig.name, formatHostAndPort(net.ParseIP(allowServerPodIP), port)) + + ginkgo.By("asserting the *client* pod can not contact deny server pod exposed endpoint") + namespacePodShouldNotReach(oc, clientPodConfig.namespace, clientPodConfig.name, formatHostAndPort(net.ParseIP(denyServerPodIP), port)) + }, + ginkgo.Entry( + "in L2 primary UDN", + "layer2", + *podConfig( + "client-pod", + ), + *podConfig( + "allow-server-pod", + withCommand(func() []string { + return httpServerContainerCmd(port) + }), + withLabels(allowServerPodLabel), + ), + *podConfig( + "deny-server-pod", + withCommand(func() []string { + return httpServerContainerCmd(port) + }), + withLabels(denyServerPodLabel), + ), + ), + ginkgo.Entry( + "in L3 primary UDN", + "layer3", + *podConfig( + "client-pod", + ), + *podConfig( + "allow-server-pod", + withCommand(func() []string { + return httpServerContainerCmd(port) + }), + withLabels(allowServerPodLabel), + ), + *podConfig( + "deny-server-pod", + withCommand(func() []string { + return httpServerContainerCmd(port) + }), + withLabels(denyServerPodLabel), + ), + )) + }) +}) + +func getNamespaceName(f *framework.Framework, nsSuffix string) string { + return fmt.Sprintf("%s-%s", f.Namespace.Name, nsSuffix) +} + +func allowTrafficToPodFromNamespacePolicy(f *framework.Framework, namespace, fromNamespace, policyName string, podLabel map[string]string) (*knet.NetworkPolicy, error) { + policy := &knet.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: policyName, + }, + Spec: knet.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{MatchLabels: podLabel}, + PolicyTypes: []knet.PolicyType{knet.PolicyTypeIngress}, + Ingress: []knet.NetworkPolicyIngressRule{{From: []knet.NetworkPolicyPeer{ + {NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": fromNamespace}}}}}}, + }, + } + return f.ClientSet.NetworkingV1().NetworkPolicies(namespace).Create(context.TODO(), policy, metav1.CreateOptions{}) +}