Skip to content

Commit

Permalink
Introduce a new API named WorkloadRebalancer to support rescheduling.
Browse files Browse the repository at this point in the history
Signed-off-by: chaosi-zju <[email protected]>
  • Loading branch information
chaosi-zju committed Apr 22, 2024
1 parent aded7c0 commit e66a375
Show file tree
Hide file tree
Showing 38 changed files with 2,895 additions and 13 deletions.
1,029 changes: 1,029 additions & 0 deletions api/openapi-spec/swagger.json

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.13.0
name: workloadrebalancers.apps.karmada.io
spec:
group: apps.karmada.io
names:
categories:
- karmada-io
kind: WorkloadRebalancer
listKind: WorkloadRebalancerList
plural: workloadrebalancers
shortNames:
- wr
singular: workloadrebalancer
scope: Cluster
versions:
- name: v1alpha1
schema:
openAPIV3Schema:
description: "WorkloadRebalancer represents the desired behavior and status
of a job which can enforces a rescheduling. \n Notes: make sure the clocks
of controller-manager and scheduler are synchronized when using this API."
properties:
apiVersion:
description: 'APIVersion defines the versioned schema of this representation
of an object. Servers should convert recognized schemas to the latest
internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
type: string
kind:
description: 'Kind is a string value representing the REST resource this
object represents. Servers may infer this from the endpoint the client
submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
type: string
metadata:
type: object
spec:
description: Spec represents the specification of the desired behavior
of WorkloadRebalancer.
properties:
workloads:
description: Workloads used to specify the list of expected resource.
Nil or empty list is not allowed.
items:
description: Workload the expected resource.
properties:
apiVersion:
description: APIVersion represents the API version of the target
resource.
type: string
kind:
description: Kind represents the Kind of the target resource.
type: string
name:
description: Name of the target resource.
type: string
namespace:
description: Namespace of the target resource. Default is empty,
which means it is a non-namespacescoped resource.
type: string
required:
- apiVersion
- kind
- name
type: object
minItems: 1
type: array
required:
- workloads
type: object
status:
description: Status represents the status of WorkloadRebalancer.
properties:
observedWorkloads:
description: ObservedWorkloads contains information about the execution
states and messages of target resources.
items:
description: ObservedWorkload the observed resource.
properties:
apiVersion:
description: APIVersion represents the API version of the target
resource.
type: string
kind:
description: Kind represents the Kind of the target resource.
type: string
name:
description: Name of the target resource.
type: string
namespace:
description: Namespace of the target resource. Default is empty,
which means it is a non-namespacescoped resource.
type: string
reason:
description: Reason represents a machine-readable description
of why this workload failed.
type: string
state:
description: State the observed state of resource.
type: string
required:
- apiVersion
- kind
- name
type: object
type: array
type: object
required:
- spec
type: object
served: true
storage: true
subresources:
status: {}
1 change: 1 addition & 0 deletions charts/karmada/_crds/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ resources:
- bases/autoscaling/autoscaling.karmada.io_federatedhpas.yaml
- bases/autoscaling/autoscaling.karmada.io_cronfederatedhpas.yaml
- bases/remedy/remedy.karmada.io_remedies.yaml
- bases/apps/apps.karmada.io_workloadrebalancers.yaml

patchesStrategicMerge:
- patches/webhook_in_resourcebindings.yaml
Expand Down
3 changes: 3 additions & 0 deletions charts/karmada/templates/post-delete-job.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -97,4 +97,7 @@ spec:
- name: crds-work-base
configMap:
name: {{ $name }}-crds-work-bases-config
- name: crds-apps-base
configMap:
name: {{ $name }}-crds-apps-bases-config
{{- end }}
5 changes: 5 additions & 0 deletions charts/karmada/templates/post-install-job.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ spec:
mountPath: /crds/bases/remedy
- name: {{ $name }}-crds-work-bases
mountPath: /crds/bases/work
- name: {{ $name }}-crds-apps-bases
mountPath: /crds/bases/apps
- name: {{ $name }}-static-resources
mountPath: /static-resources
{{ include "karmada.kubeconfig.volumeMount" . | nindent 10 }}
Expand Down Expand Up @@ -102,6 +104,9 @@ spec:
- name: {{ $name }}-crds-work-bases
configMap:
name: {{ $name }}-crds-work-bases
- name: {{ $name }}-crds-apps-bases
configMap:
name: {{ $name }}-crds-apps-bases
- name: {{ $name }}-static-resources
configMap:
name: {{ $name }}-static-resources
Expand Down
22 changes: 21 additions & 1 deletion charts/karmada/templates/pre-install-job.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,27 @@ data:
{{- (printf "%s: " $name) | nindent 6 }} |-
{{- $.Files.Get $path | nindent 8 }}
{{ end }}


---
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ $name }}-crds-apps-bases
namespace: {{ $namespace }}
annotations:
"helm.sh/hook": pre-install
"helm.sh/hook-weight": "2"
{{- if "karmada.preInstallJob.labels" }}
labels:
{{- include "karmada.preInstallJob.labels" . | nindent 4 }}
{{- end }}
data:
{{ range $path, $bytes := .Files.Glob (printf "_crds/bases/apps/**")}}
{{ $name := base $path }}
{{- (printf "%s: " $name) | nindent 6 }} |-
{{- $.Files.Get $path | nindent 8 }}
{{ end }}

{{- if eq .Values.certs.mode "custom" }}
---
apiVersion: v1
Expand Down
14 changes: 14 additions & 0 deletions cmd/controller-manager/app/controllermanager.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ import (
"github.com/karmada-io/karmada/pkg/controllers/remediation"
"github.com/karmada-io/karmada/pkg/controllers/status"
"github.com/karmada-io/karmada/pkg/controllers/unifiedauth"
"github.com/karmada-io/karmada/pkg/controllers/workloadrebalancer"
"github.com/karmada-io/karmada/pkg/dependenciesdistributor"
"github.com/karmada-io/karmada/pkg/detector"
"github.com/karmada-io/karmada/pkg/features"
Expand Down Expand Up @@ -232,6 +233,7 @@ func init() {
controllers["endpointsliceCollect"] = startEndpointSliceCollectController
controllers["endpointsliceDispatch"] = startEndpointSliceDispatchController
controllers["remedy"] = startRemedyController
controllers["workloadRebalancer"] = startWorkloadRebalancerController
}

func startClusterController(ctx controllerscontext.Context) (enabled bool, err error) {
Expand Down Expand Up @@ -707,6 +709,18 @@ func startRemedyController(ctx controllerscontext.Context) (enabled bool, err er
return true, nil
}

func startWorkloadRebalancerController(ctx controllerscontext.Context) (enabled bool, err error) {
workloadRebalancer := workloadrebalancer.RebalancerController{
Client: ctx.Mgr.GetClient(),
}
err = workloadRebalancer.SetupWithManager(ctx.Mgr)
if err != nil {
return false, err
}

return true, nil
}

// setupControllers initialize controllers and setup one by one.
func setupControllers(mgr controllerruntime.Manager, opts *options.Options, stopChan <-chan struct{}) {
restConfig := mgr.GetConfig()
Expand Down
1 change: 1 addition & 0 deletions hack/.import-aliases
Original file line number Diff line number Diff line change
Expand Up @@ -60,5 +60,6 @@
"github.com/karmada-io/karmada/pkg/apis/work/v1alpha1": "workv1alpha1",
"github.com/karmada-io/karmada/pkg/apis/work/v1alpha2": "workv1alpha2",
"github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1": "autoscalingv1alpha1",
"github.com/karmada-io/karmada/pkg/apis/apps/v1alpha1": "appsv1alpha1",
"github.com/karmada-io/karmada/operator/pkg/apis/operator/v1alpha1": "operatorv1alpha1"
}
6 changes: 6 additions & 0 deletions hack/tools/swagger/generateswagger.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"k8s.io/kube-openapi/pkg/validation/spec"

"github.com/karmada-io/karmada/hack/tools/swagger/lib"
appsv1alpha1 "github.com/karmada-io/karmada/pkg/apis/apps/v1alpha1"
autoscalingv1alpha1 "github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1"
clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1"
configv1alpha1 "github.com/karmada-io/karmada/pkg/apis/config/v1alpha1"
Expand Down Expand Up @@ -112,6 +113,10 @@ func main() {
remedyv1alpha1.SchemeGroupVersion.WithResource(remedyv1alpha1.ResourcePluralRemedy),
remedyv1alpha1.SchemeGroupVersion.WithResource(remedyv1alpha1.ResourceSingularRemedy), meta.RESTScopeRoot)

mapper.AddSpecific(appsv1alpha1.SchemeGroupVersion.WithKind(appsv1alpha1.ResourceKindWorkloadRebalancer),
appsv1alpha1.SchemeGroupVersion.WithResource(appsv1alpha1.ResourcePluralWorkloadRebalancer),
appsv1alpha1.SchemeGroupVersion.WithResource(appsv1alpha1.ResourceSingularWorkloadRebalancer), meta.RESTScopeRoot)

spec, err := lib.RenderOpenAPISpec(lib.Config{
Info: spec.InfoProps{
Title: "Karmada OpenAPI",
Expand Down Expand Up @@ -145,6 +150,7 @@ func main() {
{GVR: autoscalingv1alpha1.SchemeGroupVersion.WithResource(autoscalingv1alpha1.ResourcePluralFederatedHPA), NamespaceScoped: autoscalingv1alpha1.ResourceNamespaceScopedFederatedHPA},
{GVR: autoscalingv1alpha1.SchemeGroupVersion.WithResource(autoscalingv1alpha1.ResourcePluralCronFederatedHPA), NamespaceScoped: autoscalingv1alpha1.ResourceNamespaceScopedCronFederatedHPA},
{GVR: remedyv1alpha1.SchemeGroupVersion.WithResource(remedyv1alpha1.ResourcePluralRemedy), NamespaceScoped: remedyv1alpha1.ResourceNamespaceScopedRemedy},
{GVR: appsv1alpha1.SchemeGroupVersion.WithResource(appsv1alpha1.ResourcePluralWorkloadRebalancer), NamespaceScoped: appsv1alpha1.ResourceNamespaceScopedWorkloadRebalancer},
},
Mapper: mapper,
})
Expand Down
17 changes: 14 additions & 3 deletions hack/update-codegen.sh
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,11 @@ deepcopy-gen \
--input-dirs=github.com/karmada-io/karmada/pkg/apis/remedy/v1alpha1 \
--output-package=github.com/karmada-io/karmada/pkg/apis/remedy/v1alpha1 \
--output-file-base=zz_generated.deepcopy
deepcopy-gen \
--go-header-file hack/boilerplate/boilerplate.go.txt \
--input-dirs=github.com/karmada-io/karmada/pkg/apis/apps/v1alpha1 \
--output-package=github.com/karmada-io/karmada/pkg/apis/apps/v1alpha1 \
--output-file-base=zz_generated.deepcopy

echo "Generating with register-gen"
register-gen \
Expand Down Expand Up @@ -167,6 +172,11 @@ register-gen \
--input-dirs=github.com/karmada-io/karmada/pkg/apis/remedy/v1alpha1 \
--output-package=github.com/karmada-io/karmada/pkg/apis/remedy/v1alpha1 \
--output-file-base=zz_generated.register
register-gen \
--go-header-file hack/boilerplate/boilerplate.go.txt \
--input-dirs=github.com/karmada-io/karmada/pkg/apis/apps/v1alpha1 \
--output-package=github.com/karmada-io/karmada/pkg/apis/apps/v1alpha1 \
--output-file-base=zz_generated.register

echo "Generating with conversion-gen"
conversion-gen \
Expand All @@ -184,7 +194,7 @@ echo "Generating with client-gen"
client-gen \
--go-header-file hack/boilerplate/boilerplate.go.txt \
--input-base="" \
--input=github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1,github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1,github.com/karmada-io/karmada/pkg/apis/work/v1alpha1,github.com/karmada-io/karmada/pkg/apis/work/v1alpha2,github.com/karmada-io/karmada/pkg/apis/config/v1alpha1,github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1,github.com/karmada-io/karmada/pkg/apis/search/v1alpha1,github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1,github.com/karmada-io/karmada/pkg/apis/remedy/v1alpha1 \
--input=github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1,github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1,github.com/karmada-io/karmada/pkg/apis/work/v1alpha1,github.com/karmada-io/karmada/pkg/apis/work/v1alpha2,github.com/karmada-io/karmada/pkg/apis/config/v1alpha1,github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1,github.com/karmada-io/karmada/pkg/apis/search/v1alpha1,github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1,github.com/karmada-io/karmada/pkg/apis/remedy/v1alpha1,github.com/karmada-io/karmada/pkg/apis/apps/v1alpha1 \
--output-package=github.com/karmada-io/karmada/pkg/generated/clientset \
--clientset-name=versioned
client-gen \
Expand All @@ -197,7 +207,7 @@ client-gen \
echo "Generating with lister-gen"
lister-gen \
--go-header-file hack/boilerplate/boilerplate.go.txt \
--input-dirs=github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1,github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1,github.com/karmada-io/karmada/pkg/apis/work/v1alpha1,github.com/karmada-io/karmada/pkg/apis/work/v1alpha2,github.com/karmada-io/karmada/pkg/apis/config/v1alpha1,github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1,github.com/karmada-io/karmada/pkg/apis/search/v1alpha1,github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1,github.com/karmada-io/karmada/pkg/apis/remedy/v1alpha1 \
--input-dirs=github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1,github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1,github.com/karmada-io/karmada/pkg/apis/work/v1alpha1,github.com/karmada-io/karmada/pkg/apis/work/v1alpha2,github.com/karmada-io/karmada/pkg/apis/config/v1alpha1,github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1,github.com/karmada-io/karmada/pkg/apis/search/v1alpha1,github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1,github.com/karmada-io/karmada/pkg/apis/remedy/v1alpha1,github.com/karmada-io/karmada/pkg/apis/apps/v1alpha1 \
--output-package=github.com/karmada-io/karmada/pkg/generated/listers
lister-gen \
--go-header-file hack/boilerplate/boilerplate.go.txt \
Expand All @@ -207,7 +217,7 @@ lister-gen \
echo "Generating with informer-gen"
informer-gen \
--go-header-file hack/boilerplate/boilerplate.go.txt \
--input-dirs=github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1,github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1,github.com/karmada-io/karmada/pkg/apis/work/v1alpha1,github.com/karmada-io/karmada/pkg/apis/work/v1alpha2,github.com/karmada-io/karmada/pkg/apis/config/v1alpha1,github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1,github.com/karmada-io/karmada/pkg/apis/search/v1alpha1,github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1,github.com/karmada-io/karmada/pkg/apis/remedy/v1alpha1 \
--input-dirs=github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1,github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1,github.com/karmada-io/karmada/pkg/apis/work/v1alpha1,github.com/karmada-io/karmada/pkg/apis/work/v1alpha2,github.com/karmada-io/karmada/pkg/apis/config/v1alpha1,github.com/karmada-io/karmada/pkg/apis/networking/v1alpha1,github.com/karmada-io/karmada/pkg/apis/search/v1alpha1,github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1,github.com/karmada-io/karmada/pkg/apis/remedy/v1alpha1,github.com/karmada-io/karmada/pkg/apis/apps/v1alpha1 \
--versioned-clientset-package=github.com/karmada-io/karmada/pkg/generated/clientset/versioned \
--listers-package=github.com/karmada-io/karmada/pkg/generated/listers \
--output-package=github.com/karmada-io/karmada/pkg/generated/informers
Expand All @@ -229,6 +239,7 @@ openapi-gen \
--input-dirs "github.com/karmada-io/karmada/pkg/apis/config/v1alpha1" \
--input-dirs "github.com/karmada-io/karmada/pkg/apis/autoscaling/v1alpha1" \
--input-dirs "github.com/karmada-io/karmada/pkg/apis/remedy/v1alpha1" \
--input-dirs "github.com/karmada-io/karmada/pkg/apis/apps/v1alpha1" \
--input-dirs "k8s.io/api/core/v1,k8s.io/apimachinery/pkg/api/resource" \
--input-dirs "k8s.io/apimachinery/pkg/apis/meta/v1,k8s.io/apimachinery/pkg/runtime,k8s.io/apimachinery/pkg/version" \
--input-dirs "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1,k8s.io/api/admissionregistration/v1,k8s.io/api/networking/v1" \
Expand Down
1 change: 1 addition & 0 deletions hack/update-crdgen.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ controller-gen crd paths=./pkg/apis/networking/... output:crd:dir=./charts/karma
controller-gen crd paths=./pkg/apis/policy/... output:crd:dir=./charts/karmada/_crds/bases/policy
controller-gen crd paths=./pkg/apis/remedy/... output:crd:dir=./charts/karmada/_crds/bases/remedy
controller-gen crd paths=./pkg/apis/work/... output:crd:dir=./charts/karmada/_crds/bases/work
controller-gen crd paths=./pkg/apis/apps/... output:crd:dir=./charts/karmada/_crds/bases/apps
controller-gen crd:generateEmbeddedObjectMeta=true paths=./examples/customresourceinterpreter/apis/... output:crd:dir=./examples/customresourceinterpreter/apis/
controller-gen crd:generateEmbeddedObjectMeta=true paths=./operator/pkg/apis/operator/... output:crd:dir=./charts/karmada-operator/crds
controller-gen crd:generateEmbeddedObjectMeta=true paths=./operator/pkg/apis/operator/... output:crd:dir=./operator/config/crds
21 changes: 21 additions & 0 deletions pkg/apis/apps/v1alpha1/doc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
/*
Copyright 2024 The Karmada Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package v1alpha1 is the v1alpha1 version of the API.
// +k8s:deepcopy-gen=package,register
// +k8s:openapi-gen=true
// +groupName=apps.karmada.io
package v1alpha1
Loading

0 comments on commit e66a375

Please sign in to comment.