Update operator to enable LoadBalancer kube-controller

api/v1/installation_types.go

@@ -673,13 +673,17 @@ type IPPool struct {
 	// AllowedUse controls what the IP pool will be used for.  If not specified or empty, defaults to
 	// ["Tunnel", "Workload"] for back-compatibility
 	AllowedUses []IPPoolAllowedUse `json:"allowedUses,omitempty" validate:"omitempty"`
+
+	// AssignmentMode determines if IP addresses from this pool should be  assigned automatically or on request only
+	AssignmentMode pcv1.AssignmentMode `json:"assignmentMode,omitempty" validate:"omitempty,assignmentMode"`
 }
 
 type IPPoolAllowedUse string
 
 const (
-	IPPoolAllowedUseWorkload IPPoolAllowedUse = "Workload"
-	IPPoolAllowedUseTunnel   IPPoolAllowedUse = "Tunnel"
+	IPPoolAllowedUseWorkload     IPPoolAllowedUse = "Workload"
+	IPPoolAllowedUseTunnel       IPPoolAllowedUse = "Tunnel"
+	IPPoolAllowedUseLoadBalancer IPPoolAllowedUse = "LoadBalancer"
 )
 
 // ToProjectCalicoV1 converts an IPPool to a crd.projectcalico.org/v1 IPPool resource.
@@ -736,6 +740,8 @@ func (p *IPPool) ToProjectCalicoV1() (*pcv1.IPPool, error) {
 		pool.Spec.AllowedUses = append(pool.Spec.AllowedUses, pcv1.IPPoolAllowedUse(use))
 	}
 
+	pool.Spec.AssignmentMode = p.AssignmentMode
+
 	return &pool, nil
 }
 
@@ -791,6 +797,8 @@ func (p *IPPool) FromProjectCalicoV1(crd pcv1.IPPool) {
 	for _, use := range crd.Spec.AllowedUses {
 		p.AllowedUses = append(p.AllowedUses, IPPoolAllowedUse(use))
 	}
+
+	p.AssignmentMode = crd.Spec.AssignmentMode
 }
 
 // CNIPluginType describes the type of CNI plugin used.

pkg/apis/crd.projectcalico.org/v1/ippool.go

@@ -74,13 +74,17 @@ type IPPoolSpec struct {
 	// AllowedUse controls what the IP pool will be used for.  If not specified or empty, defaults to
 	// ["Tunnel", "Workload"] for back-compatibility
 	AllowedUses []IPPoolAllowedUse `json:"allowedUses,omitempty" validate:"omitempty"`
+
+	// AssignmentMode determines if IP addresses from this pool should be  assigned automatically or on request only
+	AssignmentMode AssignmentMode `json:"assignmentMode,omitempty" validate:"omitempty,assignmentMode"`
 }
 
 type IPPoolAllowedUse string
 
 const (
-	IPPoolAllowedUseWorkload IPPoolAllowedUse = "Workload"
-	IPPoolAllowedUseTunnel   IPPoolAllowedUse = "Tunnel"
+	IPPoolAllowedUseWorkload      IPPoolAllowedUse = "Workload"
+	IPPoolAllowedUseTunnel        IPPoolAllowedUse = "Tunnel"
+	IPPoolsAllowedUseLoadBalancer IPPoolAllowedUse = "LoadBalancer"
 )
 
 type VXLANMode string
@@ -99,6 +103,13 @@ const (
 	IPIPModeCrossSubnet IPIPMode = "CrossSubnet"
 )
 
+type AssignmentMode string
+
+const (
+	Automatic AssignmentMode = "Automatic"
+	Manual    AssignmentMode = "Manual"
+)
+
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
 // IPPoolList contains a list of IPPool resources.

pkg/controller/ippool/defaults.go

@@ -235,6 +235,10 @@ func fillDefaults(ctx context.Context, client client.Client, instance *operator.
 				}
 			}
 		}
+
+		if pool.AssignmentMode == "" {
+			pool.AssignmentMode = crdv1.Automatic
+		}
 	}
 	return nil
 }

pkg/controller/ippool/pool_controller_test.go

@@ -452,6 +452,7 @@ var _ = table.DescribeTable("Test OpenShift IP pool defaulting",
 					BlockSize:             &twentySix,
 					AllowedUses:           []operator.IPPoolAllowedUse{operator.IPPoolAllowedUseWorkload, operator.IPPoolAllowedUseTunnel},
 					DisableNewAllocations: &false_,
+					AssignmentMode:        "Automatic",
 				},
 			},
 		}),
@@ -481,6 +482,7 @@ var _ = table.DescribeTable("Test OpenShift IP pool defaulting",
 					BlockSize:             &twentySix,
 					AllowedUses:           []operator.IPPoolAllowedUse{operator.IPPoolAllowedUseWorkload, operator.IPPoolAllowedUseTunnel},
 					DisableNewAllocations: &false_,
+					AssignmentMode:        "Automatic",
 				},
 			},
 		}),
@@ -518,6 +520,7 @@ var _ = table.DescribeTable("Test OpenShift IP pool defaulting",
 					BlockSize:             &twentySix,
 					AllowedUses:           []operator.IPPoolAllowedUse{operator.IPPoolAllowedUseWorkload, operator.IPPoolAllowedUseTunnel},
 					DisableNewAllocations: &false_,
+					AssignmentMode:        "Automatic",
 				},
 			},
 		}),

pkg/controller/ippool/validation.go

@@ -42,9 +42,30 @@ func ValidatePools(instance *operator.Installation) error {
 		}
 		names[pool.Name] = true
 
+		// Check if pool is for LoadBalancer
+		isLoadBalancer := false
+		for _, u := range pool.AllowedUses {
+			if u == operator.IPPoolAllowedUseLoadBalancer {
+				isLoadBalancer = true
+			}
+		}
+
+		// Check if pool is set as LoadBalancer no other allowed use is specified
+		if isLoadBalancer {
+			for _, u := range pool.AllowedUses {
+				if u != operator.IPPoolAllowedUseLoadBalancer {
+					return fmt.Errorf("IP pool %s AllowedUse LoadBalancer cannot be used with Workload/Tunnel", pool.Name)
+				}
+			}
+		}
+
 		// Verify NAT outgoing values.
 		switch pool.NATOutgoing {
-		case operator.NATOutgoingEnabled, operator.NATOutgoingDisabled:
+		case operator.NATOutgoingEnabled:
+		case operator.NATOutgoingDisabled:
+			if isLoadBalancer {
+				return fmt.Errorf("IP pool %s NATOutgoing cannot be disabled with allowedUse LoadBalancer", pool.Name)
+			}
 		default:
 			return fmt.Errorf("%s is invalid for natOutgoing, should be one of %s",
 				pool.NATOutgoing, strings.Join(operator.NATOutgoingTypesString, ","))
@@ -54,6 +75,11 @@ func ValidatePools(instance *operator.Installation) error {
 		if pool.NodeSelector == "" {
 			return fmt.Errorf("IP pool nodeSelector should not be empty")
 		}
+
+		if isLoadBalancer && pool.NodeSelector != "all()" {
+			return fmt.Errorf("IP pool nodeSelector should be set to all() if allowedUse is LoadBalancer")
+		}
+
 		if instance.Spec.CNI == nil {
 			// We expect this to be defaulted by the core Installation controller prior to the IP pool controller
 			// being invoked, but check just in case.
@@ -67,8 +93,10 @@ func ValidatePools(instance *operator.Installation) error {
 
 		// Verify the Encapsulation mode is valid.
 		switch pool.Encapsulation {
-		case operator.EncapsulationIPIP, operator.EncapsulationIPIPCrossSubnet:
-		case operator.EncapsulationVXLAN, operator.EncapsulationVXLANCrossSubnet:
+		case operator.EncapsulationIPIP, operator.EncapsulationIPIPCrossSubnet, operator.EncapsulationVXLAN, operator.EncapsulationVXLANCrossSubnet:
+			if isLoadBalancer {
+				return fmt.Errorf("IP pool encapsulation must be none if allowedUse is LoadBalancer")
+			}
 		case operator.EncapsulationNone:
 		default:
 			return fmt.Errorf("%s is invalid for ipPool.encapsulation, should be one of %s",
@@ -104,6 +132,10 @@ func ValidatePools(instance *operator.Installation) error {
 				}
 			}
 		}
+
+		if isLoadBalancer && *pool.DisableBGPExport {
+			return fmt.Errorf("IP pool disable bgp export must be false when AllowedUse is LoadBalancer")
+		}
 	}
 	return nil
 }

pkg/controller/migration/convert/ippools.go

@@ -263,6 +263,7 @@ func convertPool(src crdv1.IPPool) (operatorv1.IPPool, error) {
 
 	p.NodeSelector = src.Spec.NodeSelector
 	p.DisableBGPExport = &src.Spec.DisableBGPExport
+	p.AssignmentMode = src.Spec.AssignmentMode
 
 	return p, nil
 }

pkg/crds/calico/crd.projectcalico.org_ippools.yaml

@@ -94,6 +94,11 @@ spec:
                   If not specified, then this is defaulted to "Never" (i.e. VXLAN
                   tunneling is disabled).
                 type: string
+              assignmentMode:
+                description: AssignmentMode determines if IP addresses
+                  from this pool should be  assigned automatically or
+                  on request only
+                type: string
             required:
             - cidr
             type: object

pkg/crds/calico/crd.projectcalico.org_kubecontrollersconfigurations.yaml

@@ -99,6 +99,13 @@ spec:
                           with the Calico datastore. [Default: 5m]'
                         type: string
                     type: object
+                  loadBalancer:
+                    description: LoadBalancer enables and configures the LoadBalancer
+                      controller. Enabled by default, set to nil to disable.
+                    properties:
+                      assignIPs:
+                        type: string
+                    type: object
                 type: object
               debugProfilePort:
                 description: DebugProfilePort configures the port to serve memory
@@ -214,6 +221,13 @@ spec:
                               5m]'
                             type: string
                         type: object
+                      loadBalancer:
+                        description: LoadBalancer enables and configures the LoadBalancer
+                          controller. Enabled by default, set to nil to disable.
+                        properties:
+                          assignIPs:
+                            type: string
+                        type: object
                     type: object
                   debugProfilePort:
                     description: DebugProfilePort configures the port to serve memory

pkg/crds/operator/operator.tigera.io_installations.yaml

@@ -1083,6 +1083,11 @@ spec:
                           items:
                             type: string
                           type: array
+                        assignmentMode:
+                          description: AssignmentMode determines if IP addresses from
+                            this pool should be  assigned automatically or on request
+                            only
+                          type: string
                         blockSize:
                           description: |-
                             BlockSize specifies the CIDR prefex length to use when allocating per-node IP blocks from
@@ -8467,6 +8472,11 @@ spec:
                               items:
                                 type: string
                               type: array
+                            assignmentMode:
+                              description: AssignmentMode determines if IP addresses
+                                from this pool should be  assigned automatically or
+                                on request only
+                              type: string
                             blockSize:
                               description: |-
                                 BlockSize specifies the CIDR prefex length to use when allocating per-node IP blocks from

pkg/render/kubecontrollers/kube-controllers.go

@@ -107,7 +107,7 @@ type KubeControllersConfiguration struct {
 
 func NewCalicoKubeControllers(cfg *KubeControllersConfiguration) *kubeControllersComponent {
 	kubeControllerRolePolicyRules := kubeControllersRoleCommonRules(cfg, KubeController)
-	enabledControllers := []string{"node"}
+	enabledControllers := []string{"node", "loadbalancer"}
 	if cfg.Installation.Variant == operatorv1.TigeraSecureEnterprise {
 		kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, kubeControllersRoleEnterpriseCommonRules(cfg)...)
 		kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules,
@@ -365,6 +365,11 @@ func kubeControllersRoleCommonRules(cfg *KubeControllersConfiguration, kubeContr
 			Resources: []string{"pods"},
 			Verbs:     []string{"get", "list", "watch"},
 		},
+		{
+			APIGroups: []string{""},
+			Resources: []string{"services", "services/status"},
+			Verbs:     []string{"get", "list", "update", "watch"},
+		},
 		{
 			// IPAM resources are manipulated in response to node and block updates, as well as periodic triggers.
 			APIGroups: []string{"crd.projectcalico.org"},
@@ -373,7 +378,7 @@ func kubeControllersRoleCommonRules(cfg *KubeControllersConfiguration, kubeContr
 		},
 		{
 			APIGroups: []string{"crd.projectcalico.org"},
-			Resources: []string{"blockaffinities", "ipamblocks", "ipamhandles", "networksets"},
+			Resources: []string{"blockaffinities", "ipamblocks", "ipamhandles", "networksets", "ipamconfigs"},
 			Verbs:     []string{"get", "list", "create", "update", "delete", "watch"},
 		},
 		{

pkg/render/kubecontrollers/kube-controllers_test.go

@@ -186,7 +186,7 @@ var _ = Describe("kube-controllers rendering tests", func() {
 		// Verify env
 		expectedEnv := []corev1.EnvVar{
 			{Name: "DATASTORE_TYPE", Value: "kubernetes"},
-			{Name: "ENABLED_CONTROLLERS", Value: "node"},
+			{Name: "ENABLED_CONTROLLERS", Value: "node,loadbalancer"},
 			{Name: "KUBE_CONTROLLERS_CONFIG_NAME", Value: "default"},
 			{Name: "FIPS_MODE_ENABLED", Value: "false"},
 			{Name: "DISABLE_KUBE_CONTROLLERS_CONFIG_API", Value: "false"},
@@ -249,14 +249,14 @@ var _ = Describe("kube-controllers rendering tests", func() {
 		Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/kube-controllers:" + components.ComponentTigeraKubeControllers.Version))
 		envs := dp.Spec.Template.Spec.Containers[0].Env
 		Expect(envs).To(ContainElement(corev1.EnvVar{
-			Name: "ENABLED_CONTROLLERS", Value: "node,service,federatedservices,usage",
+			Name: "ENABLED_CONTROLLERS", Value: "node,loadbalancer,service,federatedservices,usage",
 		}))
 
 		Expect(len(dp.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(1))
 		Expect(len(dp.Spec.Template.Spec.Volumes)).To(Equal(1))
 
 		clusterRole := rtest.GetResource(resources, kubecontrollers.KubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole)
-		Expect(clusterRole.Rules).To(HaveLen(20))
+		Expect(clusterRole.Rules).To(HaveLen(21))
 
 		ms := rtest.GetResource(resources, kubecontrollers.KubeControllerMetrics, common.CalicoNamespace, "", "v1", "Service").(*corev1.Service)
 		Expect(ms.Spec.ClusterIP).To(Equal("None"), "metrics service should be headless")
@@ -343,7 +343,7 @@ var _ = Describe("kube-controllers rendering tests", func() {
 		Expect(dp.Spec.Template.Spec.Volumes[0].ConfigMap.Name).To(Equal("tigera-ca-bundle"))
 
 		clusterRole := rtest.GetResource(resources, kubecontrollers.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole)
-		Expect(clusterRole.Rules).To(HaveLen(18))
+		Expect(clusterRole.Rules).To(HaveLen(19))
 		Expect(clusterRole.Rules).To(ContainElement(
 			rbacv1.PolicyRule{
 				APIGroups: []string{""},
@@ -390,7 +390,7 @@ var _ = Describe("kube-controllers rendering tests", func() {
 		envs := dp.Spec.Template.Spec.Containers[0].Env
 		Expect(envs).To(ContainElement(corev1.EnvVar{
 			Name:  "ENABLED_CONTROLLERS",
-			Value: "node,service,federatedservices,usage",
+			Value: "node,loadbalancer,service,federatedservices,usage",
 		}))
 
 		Expect(len(dp.Spec.Template.Spec.Containers[0].VolumeMounts)).To(Equal(1))
@@ -546,7 +546,7 @@ var _ = Describe("kube-controllers rendering tests", func() {
 		Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/kube-controllers:" + components.ComponentTigeraKubeControllers.Version))
 
 		clusterRole := rtest.GetResource(resources, kubecontrollers.EsKubeControllerRole, "", "rbac.authorization.k8s.io", "v1", "ClusterRole").(*rbacv1.ClusterRole)
-		Expect(clusterRole.Rules).To(HaveLen(18))
+		Expect(clusterRole.Rules).To(HaveLen(19))
 		Expect(clusterRole.Rules).To(ContainElement(
 			rbacv1.PolicyRule{
 				APIGroups: []string{""},