diff --git a/parts/windows/kuberneteswindowssetup.ps1 b/parts/windows/kuberneteswindowssetup.ps1 index 8a795ba6c69..96196368cf3 100644 --- a/parts/windows/kuberneteswindowssetup.ps1 +++ b/parts/windows/kuberneteswindowssetup.ps1 @@ -159,6 +159,7 @@ $global:NetworkPlugin = "{{GetParameter "networkPlugin"}}" $global:VNetCNIPluginsURL = "{{GetParameter "vnetCniWindowsPluginsURL"}}" $global:IsDualStackEnabled = {{if IsIPv6DualStackFeatureEnabled}}$true{{else}}$false{{end}} $global:IsAzureCNIOverlayEnabled = {{if IsAzureCNIOverlayFeatureEnabled}}$true{{else}}$false{{end}} +$global:CiliumDataplaneEnabled = {{if CiliumDataplaneEnabled}}$true{{else}}$false{{end}} # Kubelet credential provider $global:CredentialProviderURL = "{{GetParameter "windowsCredentialProviderURL"}}" @@ -398,6 +399,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` diff --git a/pkg/agent/baker.go b/pkg/agent/baker.go index 2599c132832..7f8df08cb97 100644 --- a/pkg/agent/baker.go +++ b/pkg/agent/baker.go @@ -623,6 +623,9 @@ func getContainerServiceFuncMap(config *datamodel.NodeBootstrappingConfiguration "IsAzureCNIOverlayFeatureEnabled": func() bool { return cs.Properties.OrchestratorProfile.KubernetesConfig.IsUsingNetworkPluginMode("overlay") }, + "CiliumDataplaneEnabled": func() bool { + return cs.Properties.OrchestratorProfile.KubernetesConfig.EbpfDataplane == datamodel.EbpfDataplane_cilium + }, "GetBase64EncodedEnvironmentJSON": func() string { customEnvironmentJSON, _ := cs.Properties.GetCustomEnvironmentJSON(false) return base64.StdEncoding.EncodeToString([]byte(customEnvironmentJSON)) diff --git a/pkg/agent/datamodel/types.go b/pkg/agent/datamodel/types.go index 34a52005053..8f7a821f37c 100644 --- a/pkg/agent/datamodel/types.go +++ b/pkg/agent/datamodel/types.go @@ -622,6 +622,22 @@ type KubernetesAddon struct { Data string `json:"data,omitempty"` } +// EbpfDataplane controls the eBPF networking dataplane. +type EbpfDataplane int32 + +const ( + // none means don't install an eBPF dataplane. + EbpfDataplane_none EbpfDataplane = 0 + // cilium means use Cilium as the eBPF dataplane. + EbpfDataplane_cilium EbpfDataplane = 1 + // unspecified means the cx didn't provide a value. + // This is used only during validation / defaulting, never written to the database. + EbpfDataplane_unspecified EbpfDataplane = 3 + // invalid means the cx provided a value that isn't an enum in the API version. + // This will always be rejected by validation (and therefore never written to the database). + EbpfDataplane_invalid EbpfDataplane = 4 +) + // KubernetesConfig contains the Kubernetes config structure, containing Kubernetes specific configuration. type KubernetesConfig struct { KubernetesImageBase string `json:"kubernetesImageBase,omitempty"` @@ -678,6 +694,7 @@ type KubernetesConfig struct { MaximumLoadBalancerRuleCount int `json:"maximumLoadBalancerRuleCount,omitempty"` PrivateAzureRegistryServer string `json:"privateAzureRegistryServer,omitempty"` NetworkPluginMode string `json:"networkPluginMode,omitempty"` + EbpfDataplane EbpfDataplane `json:"ebpfDataplane,omitempty"` } /* diff --git a/pkg/agent/testdata/AKSWindows2019+CustomCloud+ootcredentialprovider/CustomData b/pkg/agent/testdata/AKSWindows2019+CustomCloud+ootcredentialprovider/CustomData index f8109abc29c..f1701e7e66a 100644 --- a/pkg/agent/testdata/AKSWindows2019+CustomCloud+ootcredentialprovider/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+CustomCloud+ootcredentialprovider/CustomData @@ -155,6 +155,7 @@ $global:NetworkPlugin = "azure" $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/binaries/azure-vnet-cni-singletenancy-windows-amd64-v1.1.3.zip" $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +$global:CiliumDataplaneEnabled = $false # Kubelet credential provider $global:CredentialProviderURL = "https://acs-mirror.azureedge.net/cloud-provider-azure/v1.29.0/binaries/azure-acr-credential-provider-windows-amd64-v1.29.0.tar.gz" @@ -394,6 +395,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` diff --git a/pkg/agent/testdata/AKSWindows2019+CustomCloud/CustomData b/pkg/agent/testdata/AKSWindows2019+CustomCloud/CustomData index 253c7b69050..b4a681543d0 100644 --- a/pkg/agent/testdata/AKSWindows2019+CustomCloud/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+CustomCloud/CustomData @@ -155,6 +155,7 @@ $global:NetworkPlugin = "azure" $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/binaries/azure-vnet-cni-singletenancy-windows-amd64-v1.1.3.zip" $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +$global:CiliumDataplaneEnabled = $false # Kubelet credential provider $global:CredentialProviderURL = "" @@ -394,6 +395,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` diff --git a/pkg/agent/testdata/AKSWindows2019+CustomVnet/CustomData b/pkg/agent/testdata/AKSWindows2019+CustomVnet/CustomData index 284a4f83ac0..6514c3f247b 100644 --- a/pkg/agent/testdata/AKSWindows2019+CustomVnet/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+CustomVnet/CustomData @@ -155,6 +155,7 @@ $global:NetworkPlugin = "azure" $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/binaries/azure-vnet-cni-singletenancy-windows-amd64-v1.1.3.zip" $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +$global:CiliumDataplaneEnabled = $false # Kubelet credential provider $global:CredentialProviderURL = "" @@ -388,6 +389,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` diff --git a/pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CustomData b/pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CustomData index 5faff0b98d2..8f747000cac 100644 --- a/pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+EnablePrivateClusterHostsConfigAgent/CustomData @@ -155,6 +155,7 @@ $global:NetworkPlugin = "azure" $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/binaries/azure-vnet-cni-singletenancy-windows-amd64-v1.1.3.zip" $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +$global:CiliumDataplaneEnabled = $false # Kubelet credential provider $global:CredentialProviderURL = "" @@ -388,6 +389,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` diff --git a/pkg/agent/testdata/AKSWindows2019+K8S116/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S116/CustomData index 1ecc56d1bf4..85beabfb6a7 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S116/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+K8S116/CustomData @@ -155,6 +155,7 @@ $global:NetworkPlugin = "azure" $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/binaries/azure-vnet-cni-singletenancy-windows-amd64-v1.1.3.zip" $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +$global:CiliumDataplaneEnabled = $false # Kubelet credential provider $global:CredentialProviderURL = "" @@ -388,6 +389,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` diff --git a/pkg/agent/testdata/AKSWindows2019+K8S117/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S117/CustomData index 3f9a704daed..69768afbf5a 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S117/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+K8S117/CustomData @@ -155,6 +155,7 @@ $global:NetworkPlugin = "azure" $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/binaries/azure-vnet-cni-singletenancy-windows-amd64-v1.1.3.zip" $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +$global:CiliumDataplaneEnabled = $false # Kubelet credential provider $global:CredentialProviderURL = "" @@ -388,6 +389,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` diff --git a/pkg/agent/testdata/AKSWindows2019+K8S118/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S118/CustomData index 2d6eeb764cb..594d1a45f60 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S118/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+K8S118/CustomData @@ -155,6 +155,7 @@ $global:NetworkPlugin = "azure" $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/binaries/azure-vnet-cni-singletenancy-windows-amd64-v1.1.3.zip" $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +$global:CiliumDataplaneEnabled = $false # Kubelet credential provider $global:CredentialProviderURL = "" @@ -388,6 +389,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` diff --git a/pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CustomData index 8aa89dad16a..023b534df8b 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+K8S119+CSI/CustomData @@ -155,6 +155,7 @@ $global:NetworkPlugin = "azure" $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/binaries/azure-vnet-cni-singletenancy-windows-amd64-v1.1.3.zip" $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +$global:CiliumDataplaneEnabled = $false # Kubelet credential provider $global:CredentialProviderURL = "" @@ -388,6 +389,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` diff --git a/pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CustomData index 033ad369e94..7bdbd44f77c 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+K8S119+FIPS/CustomData @@ -155,6 +155,7 @@ $global:NetworkPlugin = "azure" $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/binaries/azure-vnet-cni-singletenancy-windows-amd64-v1.1.3.zip" $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +$global:CiliumDataplaneEnabled = $false # Kubelet credential provider $global:CredentialProviderURL = "" @@ -388,6 +389,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` diff --git a/pkg/agent/testdata/AKSWindows2019+K8S119/CustomData b/pkg/agent/testdata/AKSWindows2019+K8S119/CustomData index c25634a3039..bdc506f88d5 100644 --- a/pkg/agent/testdata/AKSWindows2019+K8S119/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+K8S119/CustomData @@ -155,6 +155,7 @@ $global:NetworkPlugin = "azure" $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/binaries/azure-vnet-cni-singletenancy-windows-amd64-v1.1.3.zip" $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +$global:CiliumDataplaneEnabled = $false # Kubelet credential provider $global:CredentialProviderURL = "" @@ -388,6 +389,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` diff --git a/pkg/agent/testdata/AKSWindows2019+KubeletClientTLSBootstrapping/CustomData b/pkg/agent/testdata/AKSWindows2019+KubeletClientTLSBootstrapping/CustomData index 695c41b6847..c2b41040729 100644 --- a/pkg/agent/testdata/AKSWindows2019+KubeletClientTLSBootstrapping/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+KubeletClientTLSBootstrapping/CustomData @@ -155,6 +155,7 @@ $global:NetworkPlugin = "azure" $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/binaries/azure-vnet-cni-singletenancy-windows-amd64-v1.1.3.zip" $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +$global:CiliumDataplaneEnabled = $false # Kubelet credential provider $global:CredentialProviderURL = "" @@ -388,6 +389,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` diff --git a/pkg/agent/testdata/AKSWindows2019+KubeletServingCertificateRotation/CustomData b/pkg/agent/testdata/AKSWindows2019+KubeletServingCertificateRotation/CustomData index 9ceea729fea..0258d6dabc3 100644 --- a/pkg/agent/testdata/AKSWindows2019+KubeletServingCertificateRotation/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+KubeletServingCertificateRotation/CustomData @@ -155,6 +155,7 @@ $global:NetworkPlugin = "azure" $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/binaries/azure-vnet-cni-singletenancy-windows-amd64-v1.1.3.zip" $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +$global:CiliumDataplaneEnabled = $false # Kubelet credential provider $global:CredentialProviderURL = "https://acs-mirror.azureedge.net/cloud-provider-azure/v1.29.7/binaries/azure-acr-credential-provider-windows-amd64-v1.29.7.tar.gz" @@ -388,6 +389,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` diff --git a/pkg/agent/testdata/AKSWindows2019+ManagedIdentity/CustomData b/pkg/agent/testdata/AKSWindows2019+ManagedIdentity/CustomData index c7609e6cd0f..7991bb55786 100644 --- a/pkg/agent/testdata/AKSWindows2019+ManagedIdentity/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+ManagedIdentity/CustomData @@ -155,6 +155,7 @@ $global:NetworkPlugin = "azure" $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/binaries/azure-vnet-cni-singletenancy-windows-amd64-v1.1.3.zip" $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +$global:CiliumDataplaneEnabled = $false # Kubelet credential provider $global:CredentialProviderURL = "" @@ -388,6 +389,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` diff --git a/pkg/agent/testdata/AKSWindows2019+SecurityProfile/CustomData b/pkg/agent/testdata/AKSWindows2019+SecurityProfile/CustomData index 9004135a2a1..01423f4542b 100644 --- a/pkg/agent/testdata/AKSWindows2019+SecurityProfile/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+SecurityProfile/CustomData @@ -155,6 +155,7 @@ $global:NetworkPlugin = "azure" $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/binaries/azure-vnet-cni-singletenancy-windows-amd64-v1.1.3.zip" $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +$global:CiliumDataplaneEnabled = $false # Kubelet credential provider $global:CredentialProviderURL = "" @@ -388,6 +389,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` diff --git a/pkg/agent/testdata/AKSWindows2019+ootcredentialprovider/CustomData b/pkg/agent/testdata/AKSWindows2019+ootcredentialprovider/CustomData index fc86b64604a..1fad931880b 100644 --- a/pkg/agent/testdata/AKSWindows2019+ootcredentialprovider/CustomData +++ b/pkg/agent/testdata/AKSWindows2019+ootcredentialprovider/CustomData @@ -155,6 +155,7 @@ $global:NetworkPlugin = "azure" $global:VNetCNIPluginsURL = "https://acs-mirror.azureedge.net/azure-cni/v1.1.3/binaries/azure-vnet-cni-singletenancy-windows-amd64-v1.1.3.zip" $global:IsDualStackEnabled = $false $global:IsAzureCNIOverlayEnabled = $false +$global:CiliumDataplaneEnabled = $false # Kubelet credential provider $global:CredentialProviderURL = "https://acs-mirror.azureedge.net/cloud-provider-azure/v1.29.0/binaries/azure-acr-credential-provider-windows-amd64-v1.29.0.tar.gz" @@ -388,6 +389,7 @@ try -VNetCIDR $global:VNetCIDR ` -IsDualStackEnabled $global:IsDualStackEnabled ` -IsAzureCNIOverlayEnabled $global:IsAzureCNIOverlayEnabled + if ($TargetEnvironment -ieq "AzureStackCloud") { GenerateAzureStackCNIConfig ` diff --git a/staging/cse/windows/azurecnifunc.ps1 b/staging/cse/windows/azurecnifunc.ps1 index e9dfae072dc..6c0dd89cf97 100644 --- a/staging/cse/windows/azurecnifunc.ps1 +++ b/staging/cse/windows/azurecnifunc.ps1 @@ -45,12 +45,18 @@ function Set-AzureCNIConfig [Parameter(Mandatory=$false)][bool] $IsAzureCNIOverlayEnabled ) - Logs-To-Event -TaskName "AKS.WindowsCSE.SetAzureCNIConfig" -TaskMessage "Start to set Azure CNI config. IsDualStackEnabled: $global:IsDualStackEnabled, IsAzureCNIOverlayEnabled: $global:IsAzureCNIOverlayEnabled, IsDisableWindowsOutboundNat: $global:IsDisableWindowsOutboundNat" + Logs-To-Event -TaskName "AKS.WindowsCSE.SetAzureCNIConfig" -TaskMessage "Start to set Azure CNI config. IsDualStackEnabled: $global:IsDualStackEnabled, IsAzureCNIOverlayEnabled: $global:IsAzureCNIOverlayEnabled, IsDisableWindowsOutboundNat: $global:IsDisableWindowsOutboundNat, CiliumDataplaneEnabled: $global:CiliumDataplaneEnabled" $fileName = [Io.path]::Combine("$AzureCNIConfDir", "10-azure.conflist") $configJson = Get-Content $fileName | ConvertFrom-Json $configJson.plugins.dns.Nameservers[0] = $KubeDnsServiceIp $configJson.plugins.dns.Search[0] = $KubeDnsSearchPath + + if (Test-Path variable:global:CiliumDataplaneEnabled) { + if($global:CiliumDataplaneEnabled) { + $configJson.plugins.ipam.type = "azure-cns" + } + } if ($global:IsDisableWindowsOutboundNat) { # Replace OutBoundNAT with LoopbackDSR for IMDS acess if AKS cluster disabled Windows OutBoundNAT. diff --git a/staging/cse/windows/azurecnifunc.tests.ps1 b/staging/cse/windows/azurecnifunc.tests.ps1 index 9fa006b4ce9..a4ff18fb419 100644 --- a/staging/cse/windows/azurecnifunc.tests.ps1 +++ b/staging/cse/windows/azurecnifunc.tests.ps1 @@ -29,6 +29,7 @@ Describe 'Set-AzureCNIConfig' { $isDualStackEnabled = $false $KubeDnsServiceIp = "10.0.0.10" $global:IsDisableWindowsOutboundNat = $false + $global:CiliumDataplaneEnabled = $false $global:KubeproxyFeatureGates = @("WinDSR=true") $azureCNIConfigFile = [Io.path]::Combine($azureCNIConfDir, "10-azure.conflist") @@ -55,8 +56,27 @@ Describe 'Set-AzureCNIConfig' { } } - Context 'WinDSR is enabled by default' { - It "Should remove ROUTE" { + Context 'Cilium (ebpf dataplane) is enabled' { + It "Should use azure-cns as IPAM" { + Set-Default-AzureCNI "AzureCNI.Default.conflist" + + $global:CiliumDataplaneEnabled = $true + Set-AzureCNIConfig -AzureCNIConfDir $azureCNIConfDir ` + -KubeDnsSearchPath $kubeDnsSearchPath ` + -KubeClusterCIDR $kubeClusterCIDR ` + -KubeServiceCIDR $kubeServiceCIDR ` + -VNetCIDR $vNetCIDR ` + -IsDualStackEnabled $isDualStackEnabled + + $actualConfigJson = Read-Format-Json $azureCNIConfigFile + $expectedConfigJson = Read-Format-Json ([Io.path]::Combine($azureCNIConfDir, "AzureCNI.Expect.CiliumNodeSubnet.conflist")) + $difference = Compare-Object $actualConfigJson $expectedConfigJson + $difference | Should -Be $null + } + } + + Context 'WinDSR is enabled, ebpf dataplane disabled by default' { + It "Should remove ROUTE and use azure-vnet-ipam for IPAM" { Set-Default-AzureCNI "AzureCNI.Default.conflist" Set-AzureCNIConfig -AzureCNIConfDir $azureCNIConfDir ` diff --git a/staging/cse/windows/azurecnifunc.tests.suites/AzureCNI.Expect.CiliumNodeSubnet.conflist b/staging/cse/windows/azurecnifunc.tests.suites/AzureCNI.Expect.CiliumNodeSubnet.conflist new file mode 100644 index 00000000000..e754004d7fc --- /dev/null +++ b/staging/cse/windows/azurecnifunc.tests.suites/AzureCNI.Expect.CiliumNodeSubnet.conflist @@ -0,0 +1,87 @@ +{ + "cniVersion": "0.3.0", + "name": "azure", + "adapterName": "", + "plugins": [ + { + "type": "azure-vnet", + "mode": "bridge", + "bridge": "azure0", + "capabilities": { + "portMappings": true, + "dns": true + }, + "ipam": { + "type": "azure-cns" + }, + "dns": { + "Nameservers": [ + "10.0.0.10", + "168.63.129.16" + ], + "Search": [ + "svc.cluster.local" + ] + }, + "AdditionalArgs": [ + { + "Name": "EndpointPolicy", + "Value": { + "Type": "OutBoundNAT", + "ExceptionList": [ + "10.224.0.0/12", + "10.224.1.0/12" + ] + } + }, + { + "Name": "EndpointPolicy", + "Value": { + "Type": "ACL", + "Protocols": "6", + "Action": "Block", + "Direction": "Out", + "RemoteAddresses": "168.63.129.16/32", + "RemotePorts": "80", + "Priority": 200, + "RuleType": "Switch" + } + }, + { + "Name": "EndpointPolicy", + "Value": { + "Type": "ACL", + "Protocols": "6", + "Action": "Block", + "Direction": "Out", + "RemoteAddresses": "168.63.129.16/32", + "RemotePorts": "32526", + "Priority": 200, + "RuleType": "Switch" + } + }, + { + "Name": "EndpointPolicy", + "Value": { + "Type": "ACL", + "Action": "Allow", + "Direction": "In", + "Priority": 65500 + } + }, + { + "Name": "EndpointPolicy", + "Value": { + "Type": "ACL", + "Action": "Allow", + "Direction": "Out", + "Priority": 65500 + } + } + ], + "windowsSettings": { + "enableLoopbackDSR": true + } + } + ] +}