From 796514ab0aad17c4414ed40113d2aab789fe89c0 Mon Sep 17 00:00:00 2001 From: zhzhuang-zju Date: Thu, 24 Oct 2024 10:46:31 +0800 Subject: [PATCH] karmada operator add the support for custom dns domain Signed-off-by: zhzhuang-zju --- operator/pkg/certs/certs.go | 11 +++++----- operator/pkg/certs/certs_test.go | 20 +++++++++++++------ .../pkg/controlplane/apiserver/apiserver.go | 19 +++++++++--------- .../controlplane/apiserver/apiserver_test.go | 10 +++++----- .../pkg/controlplane/apiserver/mainfests.go | 2 +- operator/pkg/controlplane/etcd/etcd.go | 20 ++++++++++--------- operator/pkg/controlplane/etcd/etcd_test.go | 9 +++++---- operator/pkg/controlplane/etcd/mainfests.go | 2 +- operator/pkg/controlplane/etcd/util.go | 4 ++-- operator/pkg/controlplane/search/search.go | 8 ++++---- .../pkg/controlplane/search/search_test.go | 6 +++--- operator/pkg/init.go | 4 ++++ operator/pkg/tasks/init/apiserver.go | 2 ++ operator/pkg/tasks/init/cert.go | 1 + operator/pkg/tasks/init/component.go | 1 + operator/pkg/tasks/init/data.go | 1 + operator/pkg/tasks/init/etcd.go | 2 +- operator/pkg/tasks/init/test_helpers.go | 6 ++++++ operator/pkg/tasks/init/upload.go | 2 +- 19 files changed, 79 insertions(+), 51 deletions(-) diff --git a/operator/pkg/certs/certs.go b/operator/pkg/certs/certs.go index 30dae79c5a39..e539ebd8b6df 100644 --- a/operator/pkg/certs/certs.go +++ b/operator/pkg/certs/certs.go @@ -55,6 +55,7 @@ const ( type AltNamesMutatorConfig struct { Name string Namespace string + DNSDomain string ControlplaneAddress string Components *operatorv1alpha1.KarmadaComponents } @@ -452,8 +453,8 @@ func makeAltNamesMutator(f func(cfg *AltNamesMutatorConfig) (*certutil.AltNames, } func etcdServerAltNamesMutator(cfg *AltNamesMutatorConfig) (*certutil.AltNames, error) { - etcdClientServiceDNS := fmt.Sprintf("%s.%s.svc.cluster.local", util.KarmadaEtcdClientName(cfg.Name), cfg.Namespace) - etcdPeerServiceDNS := fmt.Sprintf("*.%s.%s.svc.cluster.local", util.KarmadaEtcdName(cfg.Name), cfg.Namespace) + etcdClientServiceDNS := fmt.Sprintf("%s.%s.svc.%s", util.KarmadaEtcdClientName(cfg.Name), cfg.Namespace, cfg.DNSDomain) + etcdPeerServiceDNS := fmt.Sprintf("*.%s.%s.svc.%s", util.KarmadaEtcdName(cfg.Name), cfg.Namespace, cfg.DNSDomain) altNames := &certutil.AltNames{ DNSNames: []string{"localhost", etcdClientServiceDNS, etcdPeerServiceDNS}, @@ -474,7 +475,7 @@ func apiServerAltNamesMutator(cfg *AltNamesMutatorConfig) (*certutil.AltNames, e "kubernetes", "kubernetes.default", "kubernetes.default.svc", - fmt.Sprintf("*.%s.svc.cluster.local", constants.KarmadaSystemNamespace), + fmt.Sprintf("*.%s.svc.%s", constants.KarmadaSystemNamespace, cfg.DNSDomain), fmt.Sprintf("*.%s.svc", constants.KarmadaSystemNamespace), }, IPs: []net.IP{ @@ -487,9 +488,9 @@ func apiServerAltNamesMutator(cfg *AltNamesMutatorConfig) (*certutil.AltNames, e // if its altNames contains 'karmada-demo-aggregated-apiserver.karmada-system.svc'; // 2.When karmada-apiserver access webhook, the cert of 'karmada-demo-webhook' will be verified to see // if its altNames contains 'karmada-demo-webhook.test.svc'. - // Therefore, the certificate's altNames should contain both 'karmada-system.svc.cluster.local' and 'test.svc.cluster.local'. + // Therefore, the certificate's altNames should contain both 'karmada-system.svc.${dnsDomain}' and 'test.svc.${dnsDomain}'. if cfg.Namespace != constants.KarmadaSystemNamespace { - appendSANsToAltNames(altNames, []string{fmt.Sprintf("*.%s.svc.cluster.local", cfg.Namespace), + appendSANsToAltNames(altNames, []string{fmt.Sprintf("*.%s.svc.%s", cfg.Namespace, cfg.DNSDomain), fmt.Sprintf("*.%s.svc", cfg.Namespace)}) } diff --git a/operator/pkg/certs/certs_test.go b/operator/pkg/certs/certs_test.go index c0eb1dd5f011..0cf06ed3d4bb 100644 --- a/operator/pkg/certs/certs_test.go +++ b/operator/pkg/certs/certs_test.go @@ -44,7 +44,7 @@ var ( "kubernetes", "kubernetes.default", "kubernetes.default.svc", - fmt.Sprintf("*.%s.svc.cluster.local", constants.KarmadaSystemNamespace), + fmt.Sprintf("*.%s.svc.%s", constants.KarmadaSystemNamespace, constants.KarmadaDefaultDNSDomain), fmt.Sprintf("*.%s.svc", constants.KarmadaSystemNamespace), } expectedAPIServerAltIPs = []net.IP{net.IPv4(127, 0, 0, 1)} @@ -94,7 +94,9 @@ func TestKarmadaCertRootCA(t *testing.T) { func TestKarmadaCertAdmin(t *testing.T) { certConfig := KarmadaCertAdmin() - err := certConfig.AltNamesMutatorFunc(&AltNamesMutatorConfig{}, certConfig) + err := certConfig.AltNamesMutatorFunc(&AltNamesMutatorConfig{ + DNSDomain: constants.KarmadaDefaultDNSDomain, + }, certConfig) if err != nil { t.Fatalf("AltNamesMutatorFunc() returned error: %v", err) } @@ -135,7 +137,9 @@ func TestKarmadaCertAdmin(t *testing.T) { func TestKarmadaCertApiserver(t *testing.T) { certConfig := KarmadaCertApiserver() - err := certConfig.AltNamesMutatorFunc(&AltNamesMutatorConfig{}, certConfig) + err := certConfig.AltNamesMutatorFunc(&AltNamesMutatorConfig{ + DNSDomain: constants.KarmadaDefaultDNSDomain, + }, certConfig) if err != nil { t.Fatalf("AltNamesMutatorFunc() returned error: %v", err) } @@ -176,6 +180,7 @@ func TestKarmadaCertClient(t *testing.T) { err := certConfig.AltNamesMutatorFunc(&AltNamesMutatorConfig{ ControlplaneAddress: newControlPlaneAddress, + DNSDomain: constants.KarmadaDefaultDNSDomain, Components: &v1alpha1.KarmadaComponents{ KarmadaAPIServer: &v1alpha1.KarmadaAPIServer{ CertSANs: []string{newCertSAN}, @@ -277,7 +282,10 @@ func TestKarmadaCertEtcdCA(t *testing.T) { func TestKarmadaCertEtcdServer(t *testing.T) { certConfig := KarmadaCertEtcdServer() - cfg := &AltNamesMutatorConfig{Namespace: constants.KarmadaSystemNamespace} + cfg := &AltNamesMutatorConfig{ + Namespace: constants.KarmadaSystemNamespace, + DNSDomain: constants.KarmadaDefaultDNSDomain, + } err := certConfig.AltNamesMutatorFunc(cfg, certConfig) if err != nil { t.Fatalf("AltNamesMutatorFunc() returned error: %v", err) @@ -287,8 +295,8 @@ func TestKarmadaCertEtcdServer(t *testing.T) { expectedUsages := []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth} expectedDNSNames := []string{ "localhost", - fmt.Sprintf("%s.%s.svc.cluster.local", util.KarmadaEtcdClientName(cfg.Name), cfg.Namespace), - fmt.Sprintf("*.%s.%s.svc.cluster.local", util.KarmadaEtcdName(cfg.Name), cfg.Namespace), + fmt.Sprintf("%s.%s.svc.%s", util.KarmadaEtcdClientName(cfg.Name), cfg.Namespace, constants.KarmadaDefaultDNSDomain), + fmt.Sprintf("*.%s.%s.svc.%s", util.KarmadaEtcdName(cfg.Name), cfg.Namespace, constants.KarmadaDefaultDNSDomain), } expectedIPs := []net.IP{net.IPv4(127, 0, 0, 1)} diff --git a/operator/pkg/controlplane/apiserver/apiserver.go b/operator/pkg/controlplane/apiserver/apiserver.go index feae902c95a5..50dfd44c6554 100644 --- a/operator/pkg/controlplane/apiserver/apiserver.go +++ b/operator/pkg/controlplane/apiserver/apiserver.go @@ -34,8 +34,8 @@ import ( ) // EnsureKarmadaAPIServer creates karmada apiserver deployment and service resource -func EnsureKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents, name, namespace string, featureGates map[string]bool) error { - if err := installKarmadaAPIServer(client, cfg.KarmadaAPIServer, cfg.Etcd, name, namespace, featureGates); err != nil { +func EnsureKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents, name, namespace, dnsDomain string, featureGates map[string]bool) error { + if err := installKarmadaAPIServer(client, cfg.KarmadaAPIServer, cfg.Etcd, name, namespace, dnsDomain, featureGates); err != nil { return fmt.Errorf("failed to install karmada apiserver, err: %w", err) } @@ -43,21 +43,22 @@ func EnsureKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.Ka } // EnsureKarmadaAggregatedAPIServer creates karmada aggregated apiserver deployment and service resource -func EnsureKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents, name, namespace string, featureGates map[string]bool) error { - if err := installKarmadaAggregatedAPIServer(client, cfg.KarmadaAggregatedAPIServer, cfg.Etcd, name, namespace, featureGates); err != nil { +func EnsureKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaComponents, name, namespace, dnsDomain string, featureGates map[string]bool) error { + if err := installKarmadaAggregatedAPIServer(client, cfg.KarmadaAggregatedAPIServer, cfg.Etcd, name, namespace, dnsDomain, featureGates); err != nil { return err } return createKarmadaAggregatedAPIServerService(client, name, namespace) } -func installKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAPIServer, etcdCfg *operatorv1alpha1.Etcd, name, namespace string, _ map[string]bool) error { +func installKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAPIServer, etcdCfg *operatorv1alpha1.Etcd, name, namespace, dnsDomain string, _ map[string]bool) error { apiserverDeploymentBytes, err := util.ParseTemplate(KarmadaApiserverDeployment, struct { DeploymentName, Namespace, Image, ImagePullPolicy string - ServiceSubnet, KarmadaCertsSecret string + ServiceSubnet, KarmadaCertsSecret, DNSDomain string Replicas *int32 }{ DeploymentName: util.KarmadaAPIServerName(name), Namespace: namespace, + DNSDomain: dnsDomain, Image: cfg.Image.Name(), ImagePullPolicy: string(cfg.ImagePullPolicy), ServiceSubnet: *cfg.ServiceSubnet, @@ -73,7 +74,7 @@ func installKarmadaAPIServer(client clientset.Interface, cfg *operatorv1alpha1.K return fmt.Errorf("error when decoding karmadaApiserver deployment: %w", err) } - err = etcd.ConfigureClientCredentials(apiserverDeployment, etcdCfg, name, namespace) + err = etcd.ConfigureClientCredentials(apiserverDeployment, etcdCfg, name, namespace, dnsDomain) if err != nil { return err } @@ -114,7 +115,7 @@ func createKarmadaAPIServerService(client clientset.Interface, cfg *operatorv1al return nil } -func installKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAggregatedAPIServer, etcdCfg *operatorv1alpha1.Etcd, name, namespace string, featureGates map[string]bool) error { +func installKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operatorv1alpha1.KarmadaAggregatedAPIServer, etcdCfg *operatorv1alpha1.Etcd, name, namespace, dnsDomain string, featureGates map[string]bool) error { aggregatedAPIServerDeploymentBytes, err := util.ParseTemplate(KarmadaAggregatedAPIServerDeployment, struct { DeploymentName, Namespace, Image, ImagePullPolicy string KubeconfigSecret, KarmadaCertsSecret string @@ -137,7 +138,7 @@ func installKarmadaAggregatedAPIServer(client clientset.Interface, cfg *operator return fmt.Errorf("err when decoding karmadaApiserver deployment: %w", err) } - err = etcd.ConfigureClientCredentials(aggregatedAPIServerDeployment, etcdCfg, name, namespace) + err = etcd.ConfigureClientCredentials(aggregatedAPIServerDeployment, etcdCfg, name, namespace, dnsDomain) if err != nil { return err } diff --git a/operator/pkg/controlplane/apiserver/apiserver_test.go b/operator/pkg/controlplane/apiserver/apiserver_test.go index 6e1446fa7efe..cce8d73e6fc3 100644 --- a/operator/pkg/controlplane/apiserver/apiserver_test.go +++ b/operator/pkg/controlplane/apiserver/apiserver_test.go @@ -61,7 +61,7 @@ func TestEnsureKarmadaAPIServer(t *testing.T) { fakeClient := fakeclientset.NewSimpleClientset() - err := EnsureKarmadaAPIServer(fakeClient, cfg, name, namespace, map[string]bool{}) + err := EnsureKarmadaAPIServer(fakeClient, cfg, name, namespace, constants.KarmadaDefaultDNSDomain, map[string]bool{}) if err != nil { t.Fatalf("expected no error, but got: %v", err) } @@ -102,7 +102,7 @@ func TestEnsureKarmadaAggregatedAPIServer(t *testing.T) { fakeClient := fakeclientset.NewSimpleClientset() - err := EnsureKarmadaAggregatedAPIServer(fakeClient, cfg, name, namespace, featureGates) + err := EnsureKarmadaAggregatedAPIServer(fakeClient, cfg, name, namespace, constants.KarmadaDefaultDNSDomain, featureGates) if err != nil { t.Fatalf("expected no error, but got: %v", err) } @@ -145,7 +145,7 @@ func TestInstallKarmadaAPIServer(t *testing.T) { featureGates := map[string]bool{"FeatureA": true} // Call the function under test. - err := installKarmadaAPIServer(fakeClient, cfg, etcdCfg, name, namespace, featureGates) + err := installKarmadaAPIServer(fakeClient, cfg, etcdCfg, name, namespace, constants.KarmadaDefaultDNSDomain, featureGates) if err != nil { t.Fatalf("expected no error, but got: %v", err) } @@ -239,7 +239,7 @@ func TestInstallKarmadaAggregatedAPIServer(t *testing.T) { etcdCfg := &operatorv1alpha1.Etcd{ Local: &operatorv1alpha1.LocalEtcd{}, } - err := installKarmadaAggregatedAPIServer(fakeClient, cfg, etcdCfg, name, namespace, featureGates) + err := installKarmadaAggregatedAPIServer(fakeClient, cfg, etcdCfg, name, namespace, constants.KarmadaDefaultDNSDomain, featureGates) if err != nil { t.Fatalf("Failed to install Karmada Aggregated API Server: %v", err) } @@ -385,7 +385,7 @@ func verifyDeploymentDetails(deployment *appsv1.Deployment, replicas *int32, ima } } - etcdServersArg := fmt.Sprintf("https://%s.%s.svc.cluster.local:%d,", util.KarmadaEtcdClientName(name), namespace, constants.EtcdListenClientPort) + etcdServersArg := fmt.Sprintf("https://%s.%s.svc.%s:%d,", util.KarmadaEtcdClientName(name), namespace, constants.KarmadaDefaultDNSDomain, constants.EtcdListenClientPort) etcdServersArg = fmt.Sprintf("--etcd-servers=%s", etcdServersArg[:len(etcdServersArg)-1]) if !contains(container.Command, etcdServersArg) { return fmt.Errorf("etcd servers argument '%s' not found in container command", etcdServersArg) diff --git a/operator/pkg/controlplane/apiserver/mainfests.go b/operator/pkg/controlplane/apiserver/mainfests.go index fb1f99c4e0c4..3456c667d39c 100644 --- a/operator/pkg/controlplane/apiserver/mainfests.go +++ b/operator/pkg/controlplane/apiserver/mainfests.go @@ -52,7 +52,7 @@ spec: - --enable-bootstrap-token-auth=true - --bind-address=0.0.0.0 - --secure-port=5443 - - --service-account-issuer=https://kubernetes.default.svc.cluster.local + - --service-account-issuer=https://kubernetes.default.svc.{{ .DNSDomain }} - --service-account-key-file=/etc/karmada/pki/karmada.key - --service-account-signing-key-file=/etc/karmada/pki/karmada.key - --service-cluster-ip-range={{ .ServiceSubnet }} diff --git a/operator/pkg/controlplane/etcd/etcd.go b/operator/pkg/controlplane/etcd/etcd.go index c4dd16b810ac..4528245ab624 100644 --- a/operator/pkg/controlplane/etcd/etcd.go +++ b/operator/pkg/controlplane/etcd/etcd.go @@ -35,26 +35,27 @@ import ( ) // EnsureKarmadaEtcd creates etcd StatefulSet and service resource. -func EnsureKarmadaEtcd(client clientset.Interface, cfg *operatorv1alpha1.LocalEtcd, name, namespace string) error { - if err := installKarmadaEtcd(client, name, namespace, cfg); err != nil { +func EnsureKarmadaEtcd(client clientset.Interface, cfg *operatorv1alpha1.LocalEtcd, name, namespace, dnsDomain string) error { + if err := installKarmadaEtcd(client, name, namespace, dnsDomain, cfg); err != nil { return err } return createEtcdService(client, name, namespace) } -func installKarmadaEtcd(client clientset.Interface, name, namespace string, cfg *operatorv1alpha1.LocalEtcd) error { +func installKarmadaEtcd(client clientset.Interface, name, namespace, dnsDomain string, cfg *operatorv1alpha1.LocalEtcd) error { // if the number of etcd is greater than one, we need to concatenate the peerURL for each member cluster. // memberName is podName generated by etcd statefulset: ${statefulsetName}-index - // memberPeerURL uses the etcd peer headless service name: ${podName}.${serviceName}.${namespace}.svc.cluster.local:2380 + // memberPeerURL uses the etcd peer headless service name: ${podName}.${serviceName}.${namespace}.svc.${dnsDomain}:2380 initialClusters := make([]string, *cfg.Replicas) for index := range initialClusters { memberName := fmt.Sprintf("%s-%d", util.KarmadaEtcdName(name), index) // build etcd member cluster peer url - memberPeerURL := fmt.Sprintf("http://%s.%s.%s.svc.cluster.local:%v", + memberPeerURL := fmt.Sprintf("http://%s.%s.%s.svc.%s:%v", memberName, util.KarmadaEtcdName(name), namespace, + dnsDomain, constants.EtcdListenPeerPort, ) @@ -62,14 +63,15 @@ func installKarmadaEtcd(client clientset.Interface, name, namespace string, cfg } etcdStatefulSetBytes, err := util.ParseTemplate(KarmadaEtcdStatefulSet, struct { - StatefulSetName, Namespace, Image, ImagePullPolicy, EtcdClientService string - CertsSecretName, EtcdPeerServiceName string - InitialCluster, EtcdDataVolumeName, EtcdCipherSuites string - Replicas, EtcdListenClientPort, EtcdListenPeerPort int32 + StatefulSetName, Namespace, Image, ImagePullPolicy, DNSDomain string + EtcdClientService, CertsSecretName, EtcdPeerServiceName string + InitialCluster, EtcdDataVolumeName, EtcdCipherSuites string + Replicas, EtcdListenClientPort, EtcdListenPeerPort int32 }{ StatefulSetName: util.KarmadaEtcdName(name), Namespace: namespace, Image: cfg.Image.Name(), + DNSDomain: dnsDomain, ImagePullPolicy: string(cfg.ImagePullPolicy), EtcdClientService: util.KarmadaEtcdClientName(name), CertsSecretName: util.EtcdCertSecretName(name), diff --git a/operator/pkg/controlplane/etcd/etcd_test.go b/operator/pkg/controlplane/etcd/etcd_test.go index 4df0d60c7a92..e338aaf582b8 100644 --- a/operator/pkg/controlplane/etcd/etcd_test.go +++ b/operator/pkg/controlplane/etcd/etcd_test.go @@ -60,7 +60,7 @@ func TestEnsureKarmadaEtcd(t *testing.T) { // Create fake clientset. fakeClient := fakeclientset.NewSimpleClientset() - err := EnsureKarmadaEtcd(fakeClient, cfg, name, namespace) + err := EnsureKarmadaEtcd(fakeClient, cfg, name, namespace, constants.KarmadaDefaultDNSDomain) if err != nil { t.Fatalf("expected no error, but got: %v", err) } @@ -98,7 +98,7 @@ func TestInstallKarmadaEtcd(t *testing.T) { // Create fake clientset. fakeClient := fakeclientset.NewSimpleClientset() - err := installKarmadaEtcd(fakeClient, name, namespace, cfg) + err := installKarmadaEtcd(fakeClient, name, namespace, constants.KarmadaDefaultDNSDomain, cfg) if err != nil { t.Fatalf("failed to install karmada etcd, got: %v", err) } @@ -292,7 +292,7 @@ func verifyStatefulSetDetails(statefulSet *appsv1.StatefulSet, replicas int32, i // verifyEtcdServers checks that the container command includes the correct etcd server argument. func verifyEtcdServers(container *corev1.Container, name, namespace string) error { - etcdServersArg := fmt.Sprintf("https://%s.%s.svc.cluster.local:%d,", util.KarmadaEtcdClientName(name), namespace, constants.EtcdListenClientPort) + etcdServersArg := fmt.Sprintf("https://%s.%s.svc.%s:%d,", util.KarmadaEtcdClientName(name), namespace, constants.KarmadaDefaultDNSDomain, constants.EtcdListenClientPort) etcdServersArg = fmt.Sprintf("--advertise-client-urls=%s", etcdServersArg[:len(etcdServersArg)-1]) if !contains(container.Command, etcdServersArg) { return fmt.Errorf("etcd servers argument '%s' not found in container command", etcdServersArg) @@ -338,10 +338,11 @@ func verifyInitialClusters(container *corev1.Container, replicas int32, name, na expectedInitialClusters := make([]string, replicas) for i := 0; i < int(replicas); i++ { memberName := fmt.Sprintf("%s-%d", util.KarmadaEtcdName(name), i) - memberPeerURL := fmt.Sprintf("http://%s.%s.%s.svc.cluster.local:%v", + memberPeerURL := fmt.Sprintf("http://%s.%s.%s.svc.%s:%v", memberName, util.KarmadaEtcdName(name), namespace, + constants.KarmadaDefaultDNSDomain, constants.EtcdListenPeerPort, ) expectedInitialClusters[i] = fmt.Sprintf("%s=%s", memberName, memberPeerURL) diff --git a/operator/pkg/controlplane/etcd/mainfests.go b/operator/pkg/controlplane/etcd/mainfests.go index 91e146bb41aa..2f0724cfbd3d 100644 --- a/operator/pkg/controlplane/etcd/mainfests.go +++ b/operator/pkg/controlplane/etcd/mainfests.go @@ -51,7 +51,7 @@ spec: - --name=$(KARMADA_ETCD_NAME) - --listen-client-urls= https://0.0.0.0:{{ .EtcdListenClientPort }} - --listen-peer-urls=http://0.0.0.0:{{ .EtcdListenPeerPort }} - - --advertise-client-urls=https://{{ .EtcdClientService }}.{{ .Namespace }}.svc.cluster.local:{{ .EtcdListenClientPort }} + - --advertise-client-urls=https://{{ .EtcdClientService }}.{{ .Namespace }}.svc.{{ .DNSDomain }}:{{ .EtcdListenClientPort }} - --initial-cluster={{ .InitialCluster }} - --initial-cluster-state=new - --client-cert-auth=true diff --git a/operator/pkg/controlplane/etcd/util.go b/operator/pkg/controlplane/etcd/util.go index 32b43f642ec8..e7b78cd8129d 100644 --- a/operator/pkg/controlplane/etcd/util.go +++ b/operator/pkg/controlplane/etcd/util.go @@ -30,7 +30,7 @@ import ( ) // ConfigureClientCredentials configures etcd client credentials for Karmada core and aggregated API servers -func ConfigureClientCredentials(apiServerDeployment *appsv1.Deployment, etcdCfg *operatorv1alpha1.Etcd, name, namespace string) error { +func ConfigureClientCredentials(apiServerDeployment *appsv1.Deployment, etcdCfg *operatorv1alpha1.Etcd, name, namespace, dnsDomain string) error { etcdClientServiceName := util.KarmadaEtcdClientName(name) etcdCertSecretName := util.EtcdCertSecretName(name) if etcdCfg.External == nil { @@ -38,7 +38,7 @@ func ConfigureClientCredentials(apiServerDeployment *appsv1.Deployment, etcdCfg fmt.Sprintf("--etcd-cafile=%s/%s.crt", constants.EtcdClientCredentialsMountPath, constants.EtcdCaCertAndKeyName), fmt.Sprintf("--etcd-certfile=%s/%s.crt", constants.EtcdClientCredentialsMountPath, constants.EtcdClientCertAndKeyName), fmt.Sprintf("--etcd-keyfile=%s/%s.key", constants.EtcdClientCredentialsMountPath, constants.EtcdClientCertAndKeyName), - fmt.Sprintf("--etcd-servers=https://%s.%s.svc.cluster.local:%s", etcdClientServiceName, namespace, strconv.Itoa(constants.EtcdListenClientPort)), + fmt.Sprintf("--etcd-servers=https://%s.%s.svc.%s:%s", etcdClientServiceName, namespace, dnsDomain, strconv.Itoa(constants.EtcdListenClientPort)), } apiServerDeployment.Spec.Template.Spec.Containers[0].Command = append(apiServerDeployment.Spec.Template.Spec.Containers[0].Command, etcdClientCredentialsArgs...) diff --git a/operator/pkg/controlplane/search/search.go b/operator/pkg/controlplane/search/search.go index e7251d0c8e4e..4d7d6b68e6be 100644 --- a/operator/pkg/controlplane/search/search.go +++ b/operator/pkg/controlplane/search/search.go @@ -33,15 +33,15 @@ import ( ) // EnsureKarmadaSearch creates karmada search deployment and service resource. -func EnsureKarmadaSearch(client clientset.Interface, cfg *operatorv1alpha1.KarmadaSearch, etcdCfg *operatorv1alpha1.Etcd, name, namespace string, featureGates map[string]bool) error { - if err := installKarmadaSearch(client, cfg, etcdCfg, name, namespace, featureGates); err != nil { +func EnsureKarmadaSearch(client clientset.Interface, cfg *operatorv1alpha1.KarmadaSearch, etcdCfg *operatorv1alpha1.Etcd, name, namespace, dnsDomain string, featureGates map[string]bool) error { + if err := installKarmadaSearch(client, cfg, etcdCfg, name, namespace, dnsDomain, featureGates); err != nil { return err } return createKarmadaSearchService(client, name, namespace) } -func installKarmadaSearch(client clientset.Interface, cfg *operatorv1alpha1.KarmadaSearch, etcdCfg *operatorv1alpha1.Etcd, name, namespace string, _ map[string]bool) error { +func installKarmadaSearch(client clientset.Interface, cfg *operatorv1alpha1.KarmadaSearch, etcdCfg *operatorv1alpha1.Etcd, name, namespace, dnsDomain string, _ map[string]bool) error { searchDeploymentSetBytes, err := util.ParseTemplate(KarmadaSearchDeployment, struct { DeploymentName, Namespace, Image, ImagePullPolicy, KarmadaCertsSecret string KubeconfigSecret string @@ -64,7 +64,7 @@ func installKarmadaSearch(client clientset.Interface, cfg *operatorv1alpha1.Karm return fmt.Errorf("err when decoding KarmadaSearch Deployment: %w", err) } - err = etcd.ConfigureClientCredentials(searchDeployment, etcdCfg, name, namespace) + err = etcd.ConfigureClientCredentials(searchDeployment, etcdCfg, name, namespace, dnsDomain) if err != nil { return err } diff --git a/operator/pkg/controlplane/search/search_test.go b/operator/pkg/controlplane/search/search_test.go index fb9e8f731baf..dc1b3607fa42 100644 --- a/operator/pkg/controlplane/search/search_test.go +++ b/operator/pkg/controlplane/search/search_test.go @@ -58,7 +58,7 @@ func TestEnsureKarmadaSearch(t *testing.T) { etcdCfg := &operatorv1alpha1.Etcd{ Local: &operatorv1alpha1.LocalEtcd{}, } - err := EnsureKarmadaSearch(fakeClient, cfg, etcdCfg, name, namespace, map[string]bool{}) + err := EnsureKarmadaSearch(fakeClient, cfg, etcdCfg, name, namespace, constants.KarmadaDefaultDNSDomain, map[string]bool{}) if err != nil { t.Fatalf("failed to ensure karmada search, but got: %v", err) } @@ -99,7 +99,7 @@ func TestInstallKarmadaSearch(t *testing.T) { etcdCfg := &operatorv1alpha1.Etcd{ Local: &operatorv1alpha1.LocalEtcd{}, } - err := installKarmadaSearch(fakeClient, cfg, etcdCfg, name, namespace, map[string]bool{}) + err := installKarmadaSearch(fakeClient, cfg, etcdCfg, name, namespace, constants.KarmadaDefaultDNSDomain, map[string]bool{}) if err != nil { t.Fatalf("failed to install karmada search: %v", err) } @@ -249,7 +249,7 @@ func verifySecrets(deployment *appsv1.Deployment, name string) error { // verifyEtcdServers checks that the container command includes the correct etcd server argument. func verifyEtcdServers(container *corev1.Container, name, namespace string) error { - etcdServersArg := fmt.Sprintf("https://%s.%s.svc.cluster.local:%d,", util.KarmadaEtcdClientName(name), namespace, constants.EtcdListenClientPort) + etcdServersArg := fmt.Sprintf("https://%s.%s.svc.%s:%d,", util.KarmadaEtcdClientName(name), namespace, constants.KarmadaDefaultDNSDomain, constants.EtcdListenClientPort) etcdServersArg = fmt.Sprintf("--etcd-servers=%s", etcdServersArg[:len(etcdServersArg)-1]) if !contains(container.Command, etcdServersArg) { return fmt.Errorf("etcd servers argument '%s' not found in container command", etcdServersArg) diff --git a/operator/pkg/init.go b/operator/pkg/init.go index 3c91c69b66fc..cfa627d278ce 100644 --- a/operator/pkg/init.go +++ b/operator/pkg/init.go @@ -207,6 +207,10 @@ func (data *initData) GetNamespace() string { return data.namespace } +func (data *initData) GetDNSDomain() string { + return data.dnsDomain +} + func (data *initData) RemoteClient() clientset.Interface { return data.remoteClient } diff --git a/operator/pkg/tasks/init/apiserver.go b/operator/pkg/tasks/init/apiserver.go index 8e8e9dd974fe..b78dc2cc8f0d 100644 --- a/operator/pkg/tasks/init/apiserver.go +++ b/operator/pkg/tasks/init/apiserver.go @@ -104,6 +104,7 @@ func runKarmadaAPIServer(r workflow.RunData) error { cfg, data.GetName(), data.GetNamespace(), + data.GetDNSDomain(), data.FeatureGates()) if err != nil { return fmt.Errorf("failed to install karmada apiserver component, err: %w", err) @@ -147,6 +148,7 @@ func runKarmadaAggregatedAPIServer(r workflow.RunData) error { cfg, data.GetName(), data.GetNamespace(), + data.GetDNSDomain(), data.FeatureGates()) if err != nil { return fmt.Errorf("failed to install karmada aggregated apiserver, err: %w", err) diff --git a/operator/pkg/tasks/init/cert.go b/operator/pkg/tasks/init/cert.go index f48dfe1da72c..5b55efb81656 100644 --- a/operator/pkg/tasks/init/cert.go +++ b/operator/pkg/tasks/init/cert.go @@ -154,6 +154,7 @@ func mutateCertConfig(data InitData, cc *certs.CertConfig) error { Namespace: data.GetNamespace(), Components: data.Components(), ControlplaneAddress: data.ControlplaneAddress(), + DNSDomain: data.GetDNSDomain(), }, cc) if err != nil { diff --git a/operator/pkg/tasks/init/component.go b/operator/pkg/tasks/init/component.go index 7efe777ec415..dbb3efa1dbfa 100644 --- a/operator/pkg/tasks/init/component.go +++ b/operator/pkg/tasks/init/component.go @@ -267,6 +267,7 @@ func runKarmadaSearch(r workflow.RunData) error { cfg.Etcd, data.GetName(), data.GetNamespace(), + data.GetDNSDomain(), data.FeatureGates(), ) if err != nil { diff --git a/operator/pkg/tasks/init/data.go b/operator/pkg/tasks/init/data.go index 12dd256c0a10..dcc71284a511 100644 --- a/operator/pkg/tasks/init/data.go +++ b/operator/pkg/tasks/init/data.go @@ -29,6 +29,7 @@ type InitData interface { certs.CertStore GetName() string GetNamespace() string + GetDNSDomain() string SetControlplaneConfig(config *rest.Config) ControlplaneConfig() *rest.Config ControlplaneAddress() string diff --git a/operator/pkg/tasks/init/etcd.go b/operator/pkg/tasks/init/etcd.go index 757347bdc6a6..f6dad2a1b011 100644 --- a/operator/pkg/tasks/init/etcd.go +++ b/operator/pkg/tasks/init/etcd.go @@ -72,7 +72,7 @@ func runDeployEtcd(r workflow.RunData) error { return errors.New("unexpected empty etcd local configuration") } - err := etcd.EnsureKarmadaEtcd(data.RemoteClient(), cfg.Etcd.Local, data.GetName(), data.GetNamespace()) + err := etcd.EnsureKarmadaEtcd(data.RemoteClient(), cfg.Etcd.Local, data.GetName(), data.GetNamespace(), data.GetDNSDomain()) if err != nil { return fmt.Errorf("failed to install etcd component, err: %w", err) } diff --git a/operator/pkg/tasks/init/test_helpers.go b/operator/pkg/tasks/init/test_helpers.go index e480b5ff8584..67a67dfa10f9 100644 --- a/operator/pkg/tasks/init/test_helpers.go +++ b/operator/pkg/tasks/init/test_helpers.go @@ -58,6 +58,7 @@ type TestInitData struct { KarmadaClientConnector clientset.Interface ControlplaneAddr string Certs []*certs.KarmadaCert + DNSDomain string } // Ensure TestInitData implements InitData interface at compile time. @@ -73,6 +74,11 @@ func (t *TestInitData) GetNamespace() string { return t.Namespace } +// GetDNSDomain returns the dnsDomain of the current Karmada installation. +func (t *TestInitData) GetDNSDomain() string { + return t.DNSDomain +} + // SetControlplaneConfig sets the control plane configuration for Karmada. func (t *TestInitData) SetControlplaneConfig(config *rest.Config) { t.ControlplaneConfigREST = config diff --git a/operator/pkg/tasks/init/upload.go b/operator/pkg/tasks/init/upload.go index 05dfd5e20036..e19c28ec65ca 100644 --- a/operator/pkg/tasks/init/upload.go +++ b/operator/pkg/tasks/init/upload.go @@ -69,7 +69,7 @@ func runUploadAdminKubeconfig(r workflow.RunData) error { switch data.Components().KarmadaAPIServer.ServiceType { case corev1.ServiceTypeClusterIP: apiserverName := util.KarmadaAPIServerName(data.GetName()) - endpoint = fmt.Sprintf("https://%s.%s.svc.cluster.local:%d", apiserverName, data.GetNamespace(), constants.KarmadaAPIserverListenClientPort) + endpoint = fmt.Sprintf("https://%s.%s.svc.%s:%d", apiserverName, data.GetNamespace(), data.GetDNSDomain(), constants.KarmadaAPIserverListenClientPort) case corev1.ServiceTypeNodePort: service, err := apiclient.GetService(data.RemoteClient(), util.KarmadaAPIServerName(data.GetName()), data.GetNamespace())