Adding webhook 'delete' command to remove non-namespaced objects

Dockerfile

@@ -2,9 +2,11 @@ FROM golang:1.22 AS builder
 WORKDIR /go/src/github.com/k8snetworkplumbingwg/sriov-network-operator
 COPY . .
 RUN make _build-manager BIN_PATH=build/_output/cmd
+RUN make _build-sriov-network-operator-config-cleanup BIN_PATH=build/_output/cmd
 
 FROM quay.io/centos/centos:stream9
 COPY --from=builder /go/src/github.com/k8snetworkplumbingwg/sriov-network-operator/build/_output/cmd/manager /usr/bin/sriov-network-operator
+COPY --from=builder /go/src/github.com/k8snetworkplumbingwg/sriov-network-operator/build/_output/cmd/sriov-network-operator-config-cleanup /usr/bin/sriov-network-operator-config-cleanup
 COPY bindata /bindata
 ENV OPERATOR_NAME=sriov-network-operator
 CMD ["/usr/bin/sriov-network-operator"]

Makefile

@@ -60,7 +60,7 @@ GOLANGCI_LINT_VER = v1.55.2
 
 all: generate lint build
 
-build: manager _build-sriov-network-config-daemon _build-webhook
+build: manager _build-sriov-network-config-daemon _build-webhook _build-sriov-network-operator-config-cleanup
 
 _build-%:
 	WHAT=$* hack/build-go.sh

api/v1/helper.go

@@ -26,11 +26,12 @@ import (
 )
 
 const (
-	LASTNETWORKNAMESPACE    = "operator.sriovnetwork.openshift.io/last-network-namespace"
-	NETATTDEFFINALIZERNAME  = "netattdef.finalizers.sriovnetwork.openshift.io"
-	POOLCONFIGFINALIZERNAME = "poolconfig.finalizers.sriovnetwork.openshift.io"
-	ESwithModeLegacy        = "legacy"
-	ESwithModeSwitchDev     = "switchdev"
+	LASTNETWORKNAMESPACE        = "operator.sriovnetwork.openshift.io/last-network-namespace"
+	NETATTDEFFINALIZERNAME      = "netattdef.finalizers.sriovnetwork.openshift.io"
+	POOLCONFIGFINALIZERNAME     = "poolconfig.finalizers.sriovnetwork.openshift.io"
+	OPERATORCONFIGFINALIZERNAME = "operatorconfig.finalizers.sriovnetwork.openshift.io"
+	ESwithModeLegacy            = "legacy"
+	ESwithModeSwitchDev         = "switchdev"
 
 	SriovCniStateEnable  = "enable"
 	SriovCniStateDisable = "disable"

cmd/sriov-network-operator-config-cleanup/cleanup.go

@@ -0,0 +1,83 @@
+package main
+
+import (
+	"context"
+	"time"
+
+	"github.com/spf13/cobra"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+
+	snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log"
+
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/watch"
+
+	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/client/clientset/versioned/typed/sriovnetwork/v1"
+)
+
+var (
+	namespace string
+	watchTO   int
+)
+
+func init() {
+	rootCmd.Flags().StringVarP(&namespace, "namespace", "n", "", "designated SriovOperatorConfig namespace")
+	rootCmd.Flags().IntVarP(&watchTO, "watch-timeout", "w", 10, "sriov-operator config post-delete watch timeout ")
+}
+
+func runCleanupCmd(cmd *cobra.Command, args []string) error {
+	// init logger
+	snolog.InitLog()
+	setupLog := log.Log.WithName("sriov-network-operator-config-cleanup")
+	setupLog.Info("Run sriov-network-operator-config-cleanup")
+
+	// adding context timeout although client-go Delete should be non-blocking by default
+	ctx, timeoutFunc := context.WithTimeout(context.Background(), time.Second*time.Duration(watchTO))
+	defer timeoutFunc()
+
+	restConfig := ctrl.GetConfigOrDie()
+	sriovcs, err := sriovnetworkv1.NewForConfig(restConfig)
+	if err != nil {
+		setupLog.Error(err, "failed to create 'sriovnetworkv1' clientset")
+	}
+
+	err = sriovcs.SriovOperatorConfigs(namespace).Delete(context.Background(), "default", metav1.DeleteOptions{})
+	if err != nil {
+		if errors.IsNotFound(err) {
+			return nil
+		}
+		setupLog.Error(err, "failed to delete SriovOperatorConfig")
+		return err
+	}
+
+	// watching 'default' config deletion with context timeout, in case sriov-operator fails to delete 'default' config
+	watcher, err := sriovcs.SriovOperatorConfigs(namespace).Watch(ctx, metav1.ListOptions{Watch: true})
+	if err != nil {
+		setupLog.Error(err, "failed creating 'default' SriovOperatorConfig object watcher")
+		return err
+	}
+	defer watcher.Stop()
+	for {
+		select {
+		case event := <-watcher.ResultChan():
+			if event.Type == watch.Deleted {
+				setupLog.Info("'default' SriovOperatorConfig is deleted")
+				return nil
+			}
+
+		case <-ctx.Done():
+			// check whether object might has been deleted before watch event triggered
+			_, err := sriovcs.SriovOperatorConfigs(namespace).Get(context.Background(), "default", metav1.GetOptions{})
+			if err != nil {
+				if errors.IsNotFound(err) {
+					return nil
+				}
+			}
+			err = ctx.Err()
+			setupLog.Error(err, "timeout has occurred for 'default' SriovOperatorConfig deletion")
+			return err
+		}
+	}
+}

cmd/sriov-network-operator-config-cleanup/cleanup_test.go

@@ -0,0 +1,177 @@
+package main
+
+import (
+	"context"
+	"sync"
+
+	"github.com/golang/mock/gomock"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/spf13/cobra"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes/scheme"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+
+	sriovnetworkv1 "github.com/k8snetworkplumbingwg/sriov-network-operator/api/v1"
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/controllers"
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/featuregate"
+	mock_platforms "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/mock"
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms/openshift"
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/test/util"
+)
+
+type configController struct {
+	k8sManager manager.Manager
+	ctx        context.Context
+	cancel     context.CancelFunc
+	wg         *sync.WaitGroup
+}
+
+var (
+	controller               *configController
+	testNamespace            string = "sriov-network-operator"
+	defaultSriovOperatorSpec        = sriovnetworkv1.SriovOperatorConfigSpec{
+		EnableInjector:        true,
+		EnableOperatorWebhook: true,
+		LogLevel:              2,
+		FeatureGates:          nil,
+	}
+)
+
+var _ = Describe("cleanup", Ordered, func() {
+	BeforeAll(func() {
+		By("Create SriovOperatorConfig controller k8s objs")
+		config := getDefaultSriovOperatorConfig()
+		Expect(k8sClient.Create(context.Background(), config)).Should(Succeed())
+
+		somePolicy := &sriovnetworkv1.SriovNetworkNodePolicy{}
+		somePolicy.SetNamespace(testNamespace)
+		somePolicy.SetName("some-policy")
+		somePolicy.Spec = sriovnetworkv1.SriovNetworkNodePolicySpec{
+			NumVfs:       5,
+			NodeSelector: map[string]string{"foo": "bar"},
+			NicSelector:  sriovnetworkv1.SriovNetworkNicSelector{},
+			Priority:     20,
+		}
+		Expect(k8sClient.Create(context.Background(), somePolicy)).ToNot(HaveOccurred())
+		DeferCleanup(func() {
+			err := k8sClient.Delete(context.Background(), somePolicy)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		controller = newConfigController()
+
+	})
+
+	It("test webhook cleanup flow", func() {
+		controller.start()
+		defer controller.stop()
+
+		cmd := &cobra.Command{}
+		namespace = testNamespace
+		// verify that finalizer has been added, by controller, upon object creation
+		config := &sriovnetworkv1.SriovOperatorConfig{}
+		Eventually(func() []string {
+			// wait for SriovOperatorConfig flags to get updated
+			err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "default", Namespace: testNamespace}, config)
+			if err != nil {
+				return nil
+			}
+			return config.Finalizers
+		}, util.APITimeout, util.RetryInterval).Should(Equal([]string{sriovnetworkv1.OPERATORCONFIGFINALIZERNAME}))
+
+		Expect(runCleanupCmd(cmd, []string{})).Should(Succeed())
+		config = &sriovnetworkv1.SriovOperatorConfig{}
+		err := util.WaitForNamespacedObjectDeleted(config, k8sClient, testNamespace, "default", util.RetryInterval, util.APITimeout)
+		Expect(err).NotTo(HaveOccurred())
+
+	})
+
+	It("test 'default' config cleanup timeout", func() {
+		// in this test case sriov-operator controller has been scaled down.
+		// we are testing returned ctx timeout error, for not being able to delete 'default' config object
+		config := getDefaultSriovOperatorConfig()
+		config.Finalizers = []string{sriovnetworkv1.OPERATORCONFIGFINALIZERNAME}
+		Expect(k8sClient.Create(context.Background(), config)).Should(Succeed())
+
+		cmd := &cobra.Command{}
+		namespace = testNamespace
+		// verify that finalizer has been added, by controller, upon object creation
+		config = &sriovnetworkv1.SriovOperatorConfig{}
+		Eventually(func() []string {
+			// wait for SriovOperatorConfig flags to get updated
+			err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "default", Namespace: testNamespace}, config)
+			if err != nil {
+				return nil
+			}
+			return config.Finalizers
+		}, util.APITimeout, util.RetryInterval).Should(Equal([]string{sriovnetworkv1.OPERATORCONFIGFINALIZERNAME}))
+
+		watchTO = 1
+		err := runCleanupCmd(cmd, []string{})
+		Expect(err.Error()).To(ContainSubstring("context deadline exceeded"))
+	})
+})
+
+func getDefaultSriovOperatorConfig() *sriovnetworkv1.SriovOperatorConfig {
+	return &sriovnetworkv1.SriovOperatorConfig{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "default",
+			Namespace: testNamespace,
+		},
+		Spec: defaultSriovOperatorSpec,
+	}
+}
+
+func newConfigController() *configController {
+	// setup controller manager
+	By("Setup controller manager")
+	k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{
+		Scheme: scheme.Scheme,
+	})
+	Expect(err).ToNot(HaveOccurred())
+
+	t := GinkgoT()
+	mockCtrl := gomock.NewController(t)
+	platformHelper := mock_platforms.NewMockInterface(mockCtrl)
+	platformHelper.EXPECT().GetFlavor().Return(openshift.OpenshiftFlavorDefault).AnyTimes()
+	platformHelper.EXPECT().IsOpenshiftCluster().Return(false).AnyTimes()
+	platformHelper.EXPECT().IsHypershift().Return(false).AnyTimes()
+
+	err = (&controllers.SriovOperatorConfigReconciler{
+		Client:         k8sManager.GetClient(),
+		Scheme:         k8sManager.GetScheme(),
+		PlatformHelper: platformHelper,
+		FeatureGate:    featuregate.New(),
+	}).SetupWithManager(k8sManager)
+	Expect(err).ToNot(HaveOccurred())
+
+	ctx, cancel := context.WithCancel(context.Background())
+	wg := sync.WaitGroup{}
+	controller = &configController{
+		k8sManager: k8sManager,
+		ctx:        ctx,
+		cancel:     cancel,
+		wg:         &wg,
+	}
+
+	return controller
+}
+
+func (c *configController) start() {
+	c.wg.Add(1)
+	go func() {
+		defer c.wg.Done()
+		defer GinkgoRecover()
+		By("Start controller manager")
+		err := c.k8sManager.Start(c.ctx)
+		Expect(err).ToNot(HaveOccurred())
+	}()
+}
+
+func (c *configController) stop() {
+	c.cancel()
+	c.wg.Wait()
+}

cmd/sriov-network-operator-config-cleanup/main.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"flag"
+	"os"
+
+	"github.com/spf13/cobra"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+
+	snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log"
+)
+
+const (
+	componentName = "sriov-network-operator-config-cleanup"
+)
+
+var (
+	rootCmd = &cobra.Command{
+		Use:   componentName,
+		Short: "Removes 'default' SriovOperatorConfig",
+		Long: `Removes 'default' SriovOperatorConfig in order to cleanup non-namespaced objects e.g clusterroles/clusterrolebinding/validating/mutating webhooks
+
+Example: sriov-network-operator-config-cleanup -n <sriov-operator ns>`,
+		RunE: runCleanupCmd,
+	}
+)
+
+func main() {
+	klog.InitFlags(nil)
+	snolog.BindFlags(flag.CommandLine)
+	rootCmd.PersistentFlags().AddGoFlagSet(flag.CommandLine)
+
+	if err := rootCmd.Execute(); err != nil {
+		log.Log.Error(err, "Error executing sriov-network-operator-config-cleanup")
+		os.Exit(1)
+	}
+}