Merge pull request openstack-k8s-operators#265 from gibizer/multus-en…

…vtest

Multus envtest

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/onsi/ginkgo/v2 v2.8.3
 	github.com/onsi/gomega v1.27.1
 	github.com/openshift/api v3.9.0+incompatible
-	github.com/openstack-k8s-operators/infra-operator/apis v0.0.0-20230206011116-de79e1982b80
+	github.com/openstack-k8s-operators/infra-operator/apis v0.0.0-20230210143210-6e3aad14c3aa
 	github.com/openstack-k8s-operators/keystone-operator/api v0.0.0-20230208150008-87df8c2f32cb
 	github.com/openstack-k8s-operators/lib-common/modules/common v0.0.0-20230220103808-bd9bda2ad709
 	github.com/openstack-k8s-operators/lib-common/modules/database v0.0.0-20230215134634-d31141e5bbba

go.sum

@@ -233,8 +233,8 @@ github.com/onsi/gomega v1.27.1 h1:rfztXRbg6nv/5f+Raen9RcGoSecHIFgBBLQK3Wdj754=
 github.com/onsi/gomega v1.27.1/go.mod h1:aHX5xOykVYzWOV4WqQy0sy8BQptgukenXpCXfadcIAw=
 github.com/openshift/api v3.9.0+incompatible h1:fJ/KsefYuZAjmrr3+5U9yZIZbTOpVkDDLDLFresAeYs=
 github.com/openshift/api v3.9.0+incompatible/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY=
-github.com/openstack-k8s-operators/infra-operator/apis v0.0.0-20230206011116-de79e1982b80 h1:uIZERZo3P3wzFTvb+8eVKltXlkVDcr6LwQKTovhr3PM=
-github.com/openstack-k8s-operators/infra-operator/apis v0.0.0-20230206011116-de79e1982b80/go.mod h1:1eXDVJYo3+NPrAyvrDo9BTcc2W/v+b6Jw/cL7O0aAno=
+github.com/openstack-k8s-operators/infra-operator/apis v0.0.0-20230210143210-6e3aad14c3aa h1:HJypldaUFUol3iBea4P6UDRuCvpAVSOtnBqOSKSnW50=
+github.com/openstack-k8s-operators/infra-operator/apis v0.0.0-20230210143210-6e3aad14c3aa/go.mod h1:5kG0Ct412tO3fNkZ5b3/BwwSsV7LkSNfOB/apUlbMJI=
 github.com/openstack-k8s-operators/keystone-operator/api v0.0.0-20230208150008-87df8c2f32cb h1:2liBXr5C9LaAM1e/CQWNY7PRwDVnPp3df9Q8lvVv5BI=
 github.com/openstack-k8s-operators/keystone-operator/api v0.0.0-20230208150008-87df8c2f32cb/go.mod h1:gr1gxe+nRHt1UNzc7D5jTXN6auvufTzhqdVsubQsLDc=
 github.com/openstack-k8s-operators/lib-common/modules/common v0.0.0-20230220103808-bd9bda2ad709 h1:g3sO7qyvAz3ltQG+I81bsM3SwrI8ILYPv+s6bWT3tLU=

test/functional/base_test.go

@@ -16,18 +16,21 @@ limitations under the License.
 package functional_test
 
 import (
+	"encoding/json"
 	"time"
 
 	"github.com/google/uuid"
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/core/v1"
 	k8s_errors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
+	networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	routev1 "github.com/openshift/api/route/v1"
 	rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1"
 	keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1"
@@ -556,7 +559,7 @@ func SimulateKeystoneServiceReady(name types.NamespacedName) {
 	logger.Info("Simulated KeystoneService ready", "on", name)
 }
 
-func AssertServiceExists(name types.NamespacedName) *corev1.Service {
+func GetService(name types.NamespacedName) *corev1.Service {
 	instance := &corev1.Service{}
 	Eventually(func(g Gomega) {
 		g.Expect(k8sClient.Get(ctx, name, instance)).Should(Succeed())
@@ -572,6 +575,15 @@ func AssertRouteExists(name types.NamespacedName) *routev1.Route {
 	return instance
 }
 
+func AssertRouteNotExists(name types.NamespacedName) *routev1.Route {
+	instance := &routev1.Route{}
+	Consistently(func(g Gomega) {
+		err := k8sClient.Get(ctx, name, instance)
+		g.Expect(k8s_errors.IsNotFound(err)).To(BeTrue())
+	}, consistencyTimeout, interval).Should(Succeed())
+	return instance
+}
+
 func GetKeystoneEndpoint(name types.NamespacedName) *keystonev1.KeystoneEndpoint {
 	instance := &keystonev1.KeystoneEndpoint{}
 	Eventually(func(g Gomega) {
@@ -669,3 +681,71 @@ func NovaSchedulerNotExists(name types.NamespacedName) {
 		g.Expect(k8s_errors.IsNotFound(err)).To(BeTrue())
 	}, consistencyTimeout, interval).Should(Succeed())
 }
+
+func CreateNetworkAttachmentDefinition(name types.NamespacedName) {
+	instance := &networkv1.NetworkAttachmentDefinition{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name.Name,
+			Namespace: name.Namespace,
+		},
+		Spec: networkv1.NetworkAttachmentDefinitionSpec{
+			Config: "",
+		},
+	}
+	Expect(k8sClient.Create(ctx, instance)).Should(Succeed())
+}
+
+func DeleteNetworkAttachmentDefinition(name types.NamespacedName) {
+	Eventually(func(g Gomega) {
+		instance := &networkv1.NetworkAttachmentDefinition{}
+		err := k8sClient.Get(ctx, name, instance)
+		// if it is already gone that is OK
+		if k8s_errors.IsNotFound(err) {
+			return
+		}
+		g.Expect(err).Should(BeNil())
+
+		g.Expect(k8sClient.Delete(ctx, instance)).Should(Succeed())
+
+		err = k8sClient.Get(ctx, name, instance)
+		g.Expect(k8s_errors.IsNotFound(err)).To(BeTrue())
+	}, timeout, interval).Should(Succeed())
+}
+
+func SimulateStatefulSetReplicaReadyWithPods(name types.NamespacedName, networkIPs map[string][]string) {
+	ss := th.GetStatefulSet(name)
+	for i := 0; i < int(*ss.Spec.Replicas); i++ {
+		pod := &v1.Pod{
+			ObjectMeta: ss.Spec.Template.ObjectMeta,
+			Spec:       ss.Spec.Template.Spec,
+		}
+		pod.ObjectMeta.Namespace = name.Namespace
+		pod.ObjectMeta.GenerateName = name.Name
+
+		var netStatus []networkv1.NetworkStatus
+		for network, IPs := range networkIPs {
+			netStatus = append(
+				netStatus,
+				networkv1.NetworkStatus{
+					Name: network,
+					IPs:  IPs,
+				},
+			)
+		}
+		netStatusAnnotation, err := json.Marshal(netStatus)
+		Expect(err).NotTo(HaveOccurred())
+		pod.Annotations[networkv1.NetworkStatusAnnot] = string(netStatusAnnotation)
+
+		Expect(k8sClient.Create(ctx, pod)).Should(Succeed())
+	}
+
+	Eventually(func(g Gomega) {
+		ss := th.GetStatefulSet(name)
+		ss.Status.Replicas = 1
+		ss.Status.ReadyReplicas = 1
+		g.Expect(k8sClient.Status().Update(ctx, ss)).To(Succeed())
+
+	}, timeout, interval).Should(Succeed())
+
+	logger.Info("Simulated statefulset success", "on", name)
+}

test/functional/nova_controller_test.go

@@ -622,4 +622,102 @@ var _ = Describe("Nova controller", func() {
 			Expect(cell0DB.Finalizers).NotTo(ContainElement("Nova"))
 		})
 	})
+	When("Nova CR instance is created with NetworkAttachment and ExternalEndpoints", func() {
+		BeforeEach(func() {
+			DeferCleanup(
+				k8sClient.Delete, ctx, CreateNovaSecret(namespace, SecretName))
+			DeferCleanup(
+				k8sClient.Delete,
+				ctx,
+				CreateNovaMessageBusSecret(namespace, MessageBusSecretName),
+			)
+			DeferCleanup(
+				DeleteDBService,
+				CreateDBService(
+					namespace,
+					"openstack",
+					corev1.ServiceSpec{
+						Ports: []corev1.ServicePort{{Port: 3306}},
+					},
+				),
+			)
+			DeferCleanup(DeleteKeystoneAPI, CreateKeystoneAPI(namespace))
+
+			internalAPINADName := types.NamespacedName{Namespace: namespace, Name: "internalapi"}
+			CreateNetworkAttachmentDefinition(internalAPINADName)
+			DeferCleanup(DeleteNetworkAttachmentDefinition, internalAPINADName)
+
+			var externalEndpoints []interface{}
+			externalEndpoints = append(
+				externalEndpoints, map[string]interface{}{
+					"endpoint":        "internal",
+					"ipAddressPool":   "osp-internalapi",
+					"loadBalancerIPs": []string{"10.1.0.1", "10.1.0.2"},
+				},
+			)
+			rawSpec := map[string]interface{}{
+				"secret": SecretName,
+				"cellTemplates": map[string]interface{}{
+					"cell0": map[string]interface{}{
+						"cellDatabaseUser": "nova_cell0",
+						"hasAPIAccess":     true,
+						"conductorServiceTemplate": map[string]interface{}{
+							"networkAttachments": []string{"internalapi"},
+						},
+					},
+				},
+				"apiServiceTemplate": map[string]interface{}{
+					"networkAttachments": []string{"internalapi"},
+					"externalEndpoints":  externalEndpoints,
+				},
+				"schedulerServiceTemplate": map[string]interface{}{
+					"networkAttachments": []string{"internalapi"},
+				},
+			}
+			CreateNova(novaName, rawSpec)
+			DeferCleanup(DeleteNova, novaName)
+
+			SimulateKeystoneServiceReady(novaKeystoneServiceName)
+			SimulateMariaDBDatabaseCompleted(mariaDBDatabaseNameForAPI)
+			SimulateMariaDBDatabaseCompleted(mariaDBDatabaseNameForCell0)
+			SimulateTransportURLReady(apiTransportURLName)
+			th.SimulateJobSuccess(cell0DBSyncJobName)
+		})
+
+		It("creates all the sub CRs and passes down the network parameters", func() {
+			SimulateStatefulSetReplicaReadyWithPods(
+				novaCell0ConductorStatefulSetName,
+				map[string][]string{namespace + "/internalapi": {"10.0.0.1"}},
+			)
+			SimulateStatefulSetReplicaReadyWithPods(
+				novaSchedulerStatefulSetName,
+				map[string][]string{namespace + "/internalapi": {"10.0.0.1"}},
+			)
+			SimulateStatefulSetReplicaReadyWithPods(
+				novaAPIdeploymentName,
+				map[string][]string{namespace + "/internalapi": {"10.0.0.1"}},
+			)
+
+			th.ExpectCondition(
+				novaName,
+				ConditionGetterFunc(NovaConditionGetter),
+				condition.ReadyCondition,
+				corev1.ConditionTrue,
+			)
+
+			nova := GetNova(novaName)
+
+			conductor := GetNovaConductor(cell0ConductorName)
+			Expect(conductor.Spec.NetworkAttachments).To(
+				Equal(nova.Spec.CellTemplates["cell0"].ConductorServiceTemplate.NetworkAttachments))
+
+			api := GetNovaAPI(novaAPIName)
+			Expect(api.Spec.NetworkAttachments).To(Equal(nova.Spec.APIServiceTemplate.NetworkAttachments))
+			Expect(api.Spec.ExternalEndpoints).To(Equal(nova.Spec.APIServiceTemplate.ExternalEndpoints))
+
+			scheduler := GetNovaScheduler(novaSchedulerName)
+			Expect(scheduler.Spec.NetworkAttachments).To(Equal(nova.Spec.APIServiceTemplate.NetworkAttachments))
+		})
+
+	})
 })

test/functional/nova_scheduler_test.go

@@ -14,10 +14,12 @@ limitations under the License.
 package functional_test
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
 
 	"github.com/google/uuid"
+	networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition"
@@ -300,4 +302,141 @@ var _ = Describe("NovaScheduler controller", func() {
 			)
 		})
 	})
+	When("NovaScheduler is created with networkAttachments", func() {
+		BeforeEach(func() {
+			DeferCleanup(
+				k8sClient.Delete, ctx, CreateNovaAPISecret(namespace, SecretName))
+
+			spec := GetDefaultNovaSchedulerSpec()
+			spec["networkAttachments"] = []string{"internalapi"}
+			novaSchedulerName = CreateNovaScheduler(namespace, spec)
+			DeferCleanup(DeleteNovaScheduler, novaSchedulerName)
+		})
+
+		It("reports that the definition is missing", func() {
+			th.ExpectConditionWithDetails(
+				novaSchedulerName,
+				ConditionGetterFunc(NovaSchedulerConditionGetter),
+				condition.NetworkAttachmentsReadyCondition,
+				corev1.ConditionFalse,
+				condition.RequestedReason,
+				"NetworkAttachment resources missing: internalapi",
+			)
+			th.ExpectCondition(
+				novaSchedulerName,
+				ConditionGetterFunc(NovaSchedulerConditionGetter),
+				condition.ReadyCondition,
+				corev1.ConditionUnknown,
+			)
+		})
+		It("reports that network attachment is missing", func() {
+			internalAPINADName := types.NamespacedName{Namespace: namespace, Name: "internalapi"}
+			CreateNetworkAttachmentDefinition(internalAPINADName)
+			DeferCleanup(DeleteNetworkAttachmentDefinition, internalAPINADName)
+
+			statefulSetName := types.NamespacedName{
+				Namespace: namespace,
+				Name:      novaSchedulerName.Name,
+			}
+			ss := th.GetStatefulSet(statefulSetName)
+
+			expectedAnnotation, err := json.Marshal(
+				[]networkv1.NetworkSelectionElement{
+					{
+						Name:      "internalapi",
+						Namespace: namespace,
+					}})
+			Expect(err).ShouldNot(HaveOccurred())
+			Expect(ss.Spec.Template.ObjectMeta.Annotations).To(
+				HaveKeyWithValue("k8s.v1.cni.cncf.io/networks", string(expectedAnnotation)),
+			)
+
+			// We don't add network attachment status annotations to the Pods
+			// to simulate that the network attachments are missing.
+			SimulateStatefulSetReplicaReadyWithPods(statefulSetName, map[string][]string{})
+
+			th.ExpectConditionWithDetails(
+				novaSchedulerName,
+				ConditionGetterFunc(NovaSchedulerConditionGetter),
+				condition.NetworkAttachmentsReadyCondition,
+				corev1.ConditionFalse,
+				condition.ErrorReason,
+				"NetworkAttachments error occured "+
+					"not all pods have interfaces with ips as configured in NetworkAttachments: [internalapi]",
+			)
+		})
+		It("reports that an IP is missing", func() {
+			internalAPINADName := types.NamespacedName{Namespace: namespace, Name: "internalapi"}
+			CreateNetworkAttachmentDefinition(internalAPINADName)
+			DeferCleanup(DeleteNetworkAttachmentDefinition, internalAPINADName)
+
+			statefulSetName := types.NamespacedName{
+				Namespace: namespace,
+				Name:      novaSchedulerName.Name,
+			}
+			ss := th.GetStatefulSet(statefulSetName)
+
+			expectedAnnotation, err := json.Marshal(
+				[]networkv1.NetworkSelectionElement{
+					{
+						Name:      "internalapi",
+						Namespace: namespace,
+					}})
+			Expect(err).ShouldNot(HaveOccurred())
+			Expect(ss.Spec.Template.ObjectMeta.Annotations).To(
+				HaveKeyWithValue("k8s.v1.cni.cncf.io/networks", string(expectedAnnotation)),
+			)
+
+			// We simulat that there is no IP associated with the internalapi
+			// network attachment
+			SimulateStatefulSetReplicaReadyWithPods(
+				statefulSetName,
+				map[string][]string{namespace + "/internalapi": {}},
+			)
+
+			th.ExpectConditionWithDetails(
+				novaSchedulerName,
+				ConditionGetterFunc(NovaSchedulerConditionGetter),
+				condition.NetworkAttachmentsReadyCondition,
+				corev1.ConditionFalse,
+				condition.ErrorReason,
+				"NetworkAttachments error occured "+
+					"not all pods have interfaces with ips as configured in NetworkAttachments: [internalapi]",
+			)
+		})
+		It("reports NetworkAttachmentsReady if the Pods got the proper annotiations", func() {
+			internalAPINADName := types.NamespacedName{Namespace: namespace, Name: "internalapi"}
+			CreateNetworkAttachmentDefinition(internalAPINADName)
+			DeferCleanup(DeleteNetworkAttachmentDefinition, internalAPINADName)
+
+			statefulSetName := types.NamespacedName{
+				Namespace: namespace,
+				Name:      novaSchedulerName.Name,
+			}
+			SimulateStatefulSetReplicaReadyWithPods(
+				statefulSetName,
+				map[string][]string{namespace + "/internalapi": {"10.0.0.1"}},
+			)
+
+			th.ExpectCondition(
+				novaSchedulerName,
+				ConditionGetterFunc(NovaSchedulerConditionGetter),
+				condition.NetworkAttachmentsReadyCondition,
+				corev1.ConditionTrue,
+			)
+
+			Eventually(func(g Gomega) {
+				novaScheduler := GetNovaScheduler(novaSchedulerName)
+				g.Expect(novaScheduler.Status.NetworkAttachments).To(
+					Equal(map[string][]string{namespace + "/internalapi": {"10.0.0.1"}}))
+			}, timeout, interval).Should(Succeed())
+
+			th.ExpectCondition(
+				novaSchedulerName,
+				ConditionGetterFunc(NovaSchedulerConditionGetter),
+				condition.ReadyCondition,
+				corev1.ConditionTrue,
+			)
+		})
+	})
 })