diff --git a/README.md b/README.md index 5518f22e..5664b70a 100644 --- a/README.md +++ b/README.md @@ -187,6 +187,8 @@ spec: size: 1 # nameServers is the [ip:port] list of name service nameServers: "" + # Whether enable rocketmq-on-dleger group deploy + enableDLeger: false # replicaPerGroup is the number of each broker cluster replicaPerGroup: 1 # brokerImage is the customized docker image repo of the RocketMQ broker diff --git a/deploy/crds/rocketmq_v1alpha1_broker_crd.yaml b/deploy/crds/rocketmq_v1alpha1_broker_crd.yaml index 35569e04..35f6b696 100644 --- a/deploy/crds/rocketmq_v1alpha1_broker_crd.yaml +++ b/deploy/crds/rocketmq_v1alpha1_broker_crd.yaml @@ -31,14 +31,16 @@ spec: openAPIV3Schema: properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation + description: + "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources" type: string kind: - description: 'Kind is a string value representing the REST resource this + description: + "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds" type: string metadata: type: object @@ -59,6 +61,9 @@ spec: nameServers: description: NameServers defines the name service list e.g. 192.168.1.1:9876;192.168.1.2:9876 type: string + enableDLeger: + description: Whether enable rocketmq-on-dleger group deploy, false in default + type: boolean replicaPerGroup: description: ReplicaPerGroup each broker cluster's replica number format: int64 @@ -70,7 +75,8 @@ spec: description: The name of pod where the metadata from type: string size: - description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + description: + 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster Important: Run "operator-sdk generate k8s" to regenerate code after modifying this file Add custom validation using kubebuilder tags: https://book-v1.book.kubebuilder.io/beyond_basics/generating_crd.html' @@ -84,6 +90,746 @@ spec: items: type: object type: array + affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling + rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule + pods to nodes that satisfy the affinity expressions + specified by this field, but it may choose + a node that violates one or more of the expressions. + The node that is most preferred is the one + with the greatest sum of weights, i.e. for + each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum + by iterating through the elements of this + field and adding "weight" to the sum if the + node matches the corresponding matchExpressions; + the node(s) with the highest sum are the most + preferred. + items: + description: An empty preferred scheduling + term matches all objects with implicit weight + 0 (i.e. it's a no-op). A null preferred + scheduling term matches no objects (i.e. + is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: Represents a key's + relationship to a set of values. + Valid operators are In, NotIn, + Exists, DoesNotExist. Gt, + and Lt. + type: string + values: + description: An array of string + values. If the operator is + In or NotIn, the values array + must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, + the values array must have + a single element, which will + be interpreted as an integer. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: Represents a key's + relationship to a set of values. + Valid operators are In, NotIn, + Exists, DoesNotExist. Gt, + and Lt. + type: string + values: + description: An array of string + values. If the operator is + In or NotIn, the values array + must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, + the values array must have + a single element, which will + be interpreted as an integer. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + description: Weight associated with matching + the corresponding nodeSelectorTerm, + in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified + by this field are not met at scheduling time, + the pod will not be scheduled onto the node. + If the affinity requirements specified by + this field cease to be met at some point during + pod execution (e.g. due to an update), the + system may or may not try to eventually evict + the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector + terms. The terms are ORed. + items: + description: A null or empty node selector + term matches no objects. The requirements + of them are ANDed. The TopologySelectorTerm + type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: Represents a key's + relationship to a set of values. + Valid operators are In, NotIn, + Exists, DoesNotExist. Gt, + and Lt. + type: string + values: + description: An array of string + values. If the operator is + In or NotIn, the values array + must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, + the values array must have + a single element, which will + be interpreted as an integer. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: Represents a key's + relationship to a set of values. + Valid operators are In, NotIn, + Exists, DoesNotExist. Gt, + and Lt. + type: string + values: + description: An array of string + values. If the operator is + In or NotIn, the values array + must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, + the values array must have + a single element, which will + be interpreted as an integer. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + description: Describes pod affinity scheduling rules + (e.g. co-locate this pod in the same node, zone, + etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule + pods to nodes that satisfy the affinity expressions + specified by this field, but it may choose + a node that violates one or more of the expressions. + The node that is most preferred is the one + with the greatest sum of weights, i.e. for + each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum + by iterating through the elements of this + field and adding "weight" to the sum if the + node has pods which matches the corresponding + podAffinityTerm; the node(s) with the highest + sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added + per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: A label query over a + set of resources, in this case pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: A label selector + requirement is a selector + that contains values, a key, + and an operator that relates + the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: operator represents + a key's relationship to + a set of values. Valid + operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an + array of string values. + If the operator is In + or NotIn, the values array + must be non-empty. If + the operator is Exists + or DoesNotExist, the values + array must be empty. This + array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a + map of {key,value} pairs. A + single {key,value} in the matchLabels + map is equivalent to an element + of matchExpressions, whose key + field is "key", the operator + is "In", and the values array + contains only "value". The requirements + are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies + which namespaces the labelSelector + applies to (matches against); null + or empty list means "this pod's + namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where + co-located is defined as running + on a node whose value of the label + with key topologyKey matches that + of any node on which any of the + selected pods is running. Empty + topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching + the corresponding podAffinityTerm, in + the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified + by this field are not met at scheduling time, + the pod will not be scheduled onto the node. + If the affinity requirements specified by + this field cease to be met at some point during + pod execution (e.g. due to a pod label update), + the system may or may not try to eventually + evict the pod from its node. When there are + multiple elements, the lists of nodes corresponding + to each podAffinityTerm are intersected, i.e. + all terms must be satisfied. + items: + description: Defines a set of pods (namely + those matching the labelSelector relative + to the given namespace(s)) that this pod + should be co-located (affinity) or not co-located + (anti-affinity) with, where co-located is + defined as running on a node whose value + of the label with key matches + that of any node on which a pod of the set + of pods is running + properties: + labelSelector: + description: A label query over a set + of resources, in this case pods. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a + set of values. Valid operators + are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values + array must be non-empty. If + the operator is Exists or + DoesNotExist, the values array + must be empty. This array + is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map + of {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which + namespaces the labelSelector applies + to (matches against); null or empty + list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where co-located + is defined as running on a node whose + value of the label with key topologyKey + matches that of any node on which any + of the selected pods is running. Empty + topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling + rules (e.g. avoid putting this pod in the same + node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule + pods to nodes that satisfy the anti-affinity + expressions specified by this field, but it + may choose a node that violates one or more + of the expressions. The node that is most + preferred is the one with the greatest sum + of weights, i.e. for each node that meets + all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity + expressions, etc.), compute a sum by iterating + through the elements of this field and adding + "weight" to the sum if the node has pods which + matches the corresponding podAffinityTerm; + the node(s) with the highest sum are the most + preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added + per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: A label query over a + set of resources, in this case pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: A label selector + requirement is a selector + that contains values, a key, + and an operator that relates + the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: operator represents + a key's relationship to + a set of values. Valid + operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an + array of string values. + If the operator is In + or NotIn, the values array + must be non-empty. If + the operator is Exists + or DoesNotExist, the values + array must be empty. This + array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a + map of {key,value} pairs. A + single {key,value} in the matchLabels + map is equivalent to an element + of matchExpressions, whose key + field is "key", the operator + is "In", and the values array + contains only "value". The requirements + are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies + which namespaces the labelSelector + applies to (matches against); null + or empty list means "this pod's + namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where + co-located is defined as running + on a node whose value of the label + with key topologyKey matches that + of any node on which any of the + selected pods is running. Empty + topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching + the corresponding podAffinityTerm, in + the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the anti-affinity requirements + specified by this field are not met at scheduling + time, the pod will not be scheduled onto the + node. If the anti-affinity requirements specified + by this field cease to be met at some point + during pod execution (e.g. due to a pod label + update), the system may or may not try to + eventually evict the pod from its node. When + there are multiple elements, the lists of + nodes corresponding to each podAffinityTerm + are intersected, i.e. all terms must be satisfied. + items: + description: Defines a set of pods (namely + those matching the labelSelector relative + to the given namespace(s)) that this pod + should be co-located (affinity) or not co-located + (anti-affinity) with, where co-located is + defined as running on a node whose value + of the label with key matches + that of any node on which a pod of the set + of pods is running + properties: + labelSelector: + description: A label query over a set + of resources, in this case pods. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a + set of values. Valid operators + are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values + array must be non-empty. If + the operator is Exists or + DoesNotExist, the values array + must be empty. This array + is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map + of {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which + namespaces the labelSelector applies + to (matches against); null or empty + list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where co-located + is defined as running on a node whose + value of the label with key topologyKey + matches that of any node on which any + of the selected pods is running. Empty + topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + tolerations: + description: If specified, the pod's tolerations. + items: + description: The pod this Toleration is attached to + tolerates any taint that matches the triple + using the matching operator . + properties: + effect: + description: Effect indicates the taint effect + to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, + PreferNoSchedule and NoExecute. + type: string + key: + description: Key is the taint key that the toleration + applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; + this combination means to match all values and + all keys. + type: string + operator: + description: Operator represents a key's relationship + to the value. Valid operators are Exists and + Equal. Defaults to Equal. Exists is equivalent + to wildcard for value, so that a pod can tolerate + all taints of a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the + period of time the toleration (which must be + of effect NoExecute, otherwise this field is + ignored) tolerates the taint. By default, it + is not set, which means tolerate the taint forever + (do not evict). Zero and negative values will + be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration + matches to. If the operator is Exists, the value + should be empty, otherwise just a regular string. + type: string + type: object + type: array volumes: description: volumes defines the broker.conf items: @@ -95,24 +841,26 @@ spec: type: object type: array required: - - size - - replicaPerGroup - - brokerImage - - imagePullPolicy - - nameServers - - allowRestart - - resources - - storageMode - - hostPath - - env - - volumes - - volumeClaimTemplates - - scalePodName + - size + - enableDLeger + - replicaPerGroup + - brokerImage + - imagePullPolicy + - nameServers + - allowRestart + - resources + - storageMode + - hostPath + - env + - volumes + - volumeClaimTemplates + - scalePodName type: object status: properties: nodes: - description: 'INSERT ADDITIONAL STATUS FIELD - define observed state + description: + 'INSERT ADDITIONAL STATUS FIELD - define observed state of cluster Important: Run "operator-sdk generate k8s" to regenerate code after modifying this file Add custom validation using kubebuilder tags: https://book-v1.book.kubebuilder.io/beyond_basics/generating_crd.html' @@ -123,11 +871,11 @@ spec: format: int64 type: integer required: - - nodes - - size + - nodes + - size type: object version: v1alpha1 versions: - - name: v1alpha1 - served: true - storage: true + - name: v1alpha1 + served: true + storage: true diff --git a/deploy/crds/rocketmq_v1alpha1_nameservice_crd.yaml b/deploy/crds/rocketmq_v1alpha1_nameservice_crd.yaml index 3946b039..02584123 100644 --- a/deploy/crds/rocketmq_v1alpha1_nameservice_crd.yaml +++ b/deploy/crds/rocketmq_v1alpha1_nameservice_crd.yaml @@ -31,14 +31,16 @@ spec: openAPIV3Schema: properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation + description: + "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources" type: string kind: - description: 'Kind is a string value representing the REST resource this + description: + "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds" type: string metadata: type: object @@ -62,8 +64,754 @@ spec: resources: description: Resources describes the compute resource requirements type: object + env: + description: env defines custom env, e.g. BROKER_MEM + items: + type: object + type: array + affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling + rules for the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule + pods to nodes that satisfy the affinity expressions + specified by this field, but it may choose + a node that violates one or more of the expressions. + The node that is most preferred is the one + with the greatest sum of weights, i.e. for + each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum + by iterating through the elements of this + field and adding "weight" to the sum if the + node matches the corresponding matchExpressions; + the node(s) with the highest sum are the most + preferred. + items: + description: An empty preferred scheduling + term matches all objects with implicit weight + 0 (i.e. it's a no-op). A null preferred + scheduling term matches no objects (i.e. + is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: Represents a key's + relationship to a set of values. + Valid operators are In, NotIn, + Exists, DoesNotExist. Gt, + and Lt. + type: string + values: + description: An array of string + values. If the operator is + In or NotIn, the values array + must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, + the values array must have + a single element, which will + be interpreted as an integer. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: Represents a key's + relationship to a set of values. + Valid operators are In, NotIn, + Exists, DoesNotExist. Gt, + and Lt. + type: string + values: + description: An array of string + values. If the operator is + In or NotIn, the values array + must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, + the values array must have + a single element, which will + be interpreted as an integer. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + description: Weight associated with matching + the corresponding nodeSelectorTerm, + in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified + by this field are not met at scheduling time, + the pod will not be scheduled onto the node. + If the affinity requirements specified by + this field cease to be met at some point during + pod execution (e.g. due to an update), the + system may or may not try to eventually evict + the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector + terms. The terms are ORed. + items: + description: A null or empty node selector + term matches no objects. The requirements + of them are ANDed. The TopologySelectorTerm + type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector + requirements by node's labels. + items: + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: Represents a key's + relationship to a set of values. + Valid operators are In, NotIn, + Exists, DoesNotExist. Gt, + and Lt. + type: string + values: + description: An array of string + values. If the operator is + In or NotIn, the values array + must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, + the values array must have + a single element, which will + be interpreted as an integer. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector + requirements by node's fields. + items: + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: The label key that + the selector applies to. + type: string + operator: + description: Represents a key's + relationship to a set of values. + Valid operators are In, NotIn, + Exists, DoesNotExist. Gt, + and Lt. + type: string + values: + description: An array of string + values. If the operator is + In or NotIn, the values array + must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, + the values array must have + a single element, which will + be interpreted as an integer. + This array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + description: Describes pod affinity scheduling rules + (e.g. co-locate this pod in the same node, zone, + etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule + pods to nodes that satisfy the affinity expressions + specified by this field, but it may choose + a node that violates one or more of the expressions. + The node that is most preferred is the one + with the greatest sum of weights, i.e. for + each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum + by iterating through the elements of this + field and adding "weight" to the sum if the + node has pods which matches the corresponding + podAffinityTerm; the node(s) with the highest + sum are the most preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added + per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: A label query over a + set of resources, in this case pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: A label selector + requirement is a selector + that contains values, a key, + and an operator that relates + the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: operator represents + a key's relationship to + a set of values. Valid + operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an + array of string values. + If the operator is In + or NotIn, the values array + must be non-empty. If + the operator is Exists + or DoesNotExist, the values + array must be empty. This + array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a + map of {key,value} pairs. A + single {key,value} in the matchLabels + map is equivalent to an element + of matchExpressions, whose key + field is "key", the operator + is "In", and the values array + contains only "value". The requirements + are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies + which namespaces the labelSelector + applies to (matches against); null + or empty list means "this pod's + namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where + co-located is defined as running + on a node whose value of the label + with key topologyKey matches that + of any node on which any of the + selected pods is running. Empty + topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching + the corresponding podAffinityTerm, in + the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified + by this field are not met at scheduling time, + the pod will not be scheduled onto the node. + If the affinity requirements specified by + this field cease to be met at some point during + pod execution (e.g. due to a pod label update), + the system may or may not try to eventually + evict the pod from its node. When there are + multiple elements, the lists of nodes corresponding + to each podAffinityTerm are intersected, i.e. + all terms must be satisfied. + items: + description: Defines a set of pods (namely + those matching the labelSelector relative + to the given namespace(s)) that this pod + should be co-located (affinity) or not co-located + (anti-affinity) with, where co-located is + defined as running on a node whose value + of the label with key matches + that of any node on which a pod of the set + of pods is running + properties: + labelSelector: + description: A label query over a set + of resources, in this case pods. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a + set of values. Valid operators + are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values + array must be non-empty. If + the operator is Exists or + DoesNotExist, the values array + must be empty. This array + is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map + of {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which + namespaces the labelSelector applies + to (matches against); null or empty + list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where co-located + is defined as running on a node whose + value of the label with key topologyKey + matches that of any node on which any + of the selected pods is running. Empty + topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling + rules (e.g. avoid putting this pod in the same + node, zone, etc. as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule + pods to nodes that satisfy the anti-affinity + expressions specified by this field, but it + may choose a node that violates one or more + of the expressions. The node that is most + preferred is the one with the greatest sum + of weights, i.e. for each node that meets + all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity + expressions, etc.), compute a sum by iterating + through the elements of this field and adding + "weight" to the sum if the node has pods which + matches the corresponding podAffinityTerm; + the node(s) with the highest sum are the most + preferred. + items: + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added + per-node to find the most preferred node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity + term, associated with the corresponding + weight. + properties: + labelSelector: + description: A label query over a + set of resources, in this case pods. + properties: + matchExpressions: + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. + items: + description: A label selector + requirement is a selector + that contains values, a key, + and an operator that relates + the key and values. + properties: + key: + description: key is the + label key that the selector + applies to. + type: string + operator: + description: operator represents + a key's relationship to + a set of values. Valid + operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an + array of string values. + If the operator is In + or NotIn, the values array + must be non-empty. If + the operator is Exists + or DoesNotExist, the values + array must be empty. This + array is replaced during + a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a + map of {key,value} pairs. A + single {key,value} in the matchLabels + map is equivalent to an element + of matchExpressions, whose key + field is "key", the operator + is "In", and the values array + contains only "value". The requirements + are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies + which namespaces the labelSelector + applies to (matches against); null + or empty list means "this pod's + namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where + co-located is defined as running + on a node whose value of the label + with key topologyKey matches that + of any node on which any of the + selected pods is running. Empty + topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching + the corresponding podAffinityTerm, in + the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the anti-affinity requirements + specified by this field are not met at scheduling + time, the pod will not be scheduled onto the + node. If the anti-affinity requirements specified + by this field cease to be met at some point + during pod execution (e.g. due to a pod label + update), the system may or may not try to + eventually evict the pod from its node. When + there are multiple elements, the lists of + nodes corresponding to each podAffinityTerm + are intersected, i.e. all terms must be satisfied. + items: + description: Defines a set of pods (namely + those matching the labelSelector relative + to the given namespace(s)) that this pod + should be co-located (affinity) or not co-located + (anti-affinity) with, where co-located is + defined as running on a node whose value + of the label with key matches + that of any node on which a pod of the set + of pods is running + properties: + labelSelector: + description: A label query over a set + of resources, in this case pods. + properties: + matchExpressions: + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label + key that the selector applies + to. + type: string + operator: + description: operator represents + a key's relationship to a + set of values. Valid operators + are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values + array must be non-empty. If + the operator is Exists or + DoesNotExist, the values array + must be empty. This array + is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map + of {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which + namespaces the labelSelector applies + to (matches against); null or empty + list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where co-located + is defined as running on a node whose + value of the label with key topologyKey + matches that of any node on which any + of the selected pods is running. Empty + topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + tolerations: + description: If specified, the pod's tolerations. + items: + description: The pod this Toleration is attached to + tolerates any taint that matches the triple + using the matching operator . + properties: + effect: + description: Effect indicates the taint effect + to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, + PreferNoSchedule and NoExecute. + type: string + key: + description: Key is the taint key that the toleration + applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; + this combination means to match all values and + all keys. + type: string + operator: + description: Operator represents a key's relationship + to the value. Valid operators are Exists and + Equal. Defaults to Equal. Exists is equivalent + to wildcard for value, so that a pod can tolerate + all taints of a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the + period of time the toleration (which must be + of effect NoExecute, otherwise this field is + ignored) tolerates the taint. By default, it + is not set, which means tolerate the taint forever + (do not evict). Zero and negative values will + be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration + matches to. If the operator is Exists, the value + should be empty, otherwise just a regular string. + type: string + type: object + type: array size: - description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster + description: + 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster Important: Run "operator-sdk generate k8s" to regenerate code after modifying this file Add custom validation using kubebuilder tags: https://book-v1.book.kubebuilder.io/beyond_basics/generating_crd.html @@ -79,20 +827,21 @@ spec: type: object type: array required: - - size - - nameServiceImage - - imagePullPolicy - - hostNetwork - - dnsPolicy - - resources - - storageMode - - hostPath - - volumeClaimTemplates + - size + - nameServiceImage + - imagePullPolicy + - hostNetwork + - dnsPolicy + - resources + - storageMode + - hostPath + - volumeClaimTemplates type: object status: properties: nameServers: - description: 'INSERT ADDITIONAL STATUS FIELD - define observed state + description: + 'INSERT ADDITIONAL STATUS FIELD - define observed state of cluster Important: Run "operator-sdk generate k8s" to regenerate code after modifying this file Add custom validation using kubebuilder tags: https://book-v1.book.kubebuilder.io/beyond_basics/generating_crd.html @@ -101,10 +850,10 @@ spec: type: string type: array required: - - nameServers + - nameServers type: object version: v1alpha1 versions: - - name: v1alpha1 - served: true - storage: true + - name: v1alpha1 + served: true + storage: true diff --git a/docs/cn/README.md b/docs/cn/README.md index 2444f18c..a6784ba0 100644 --- a/docs/cn/README.md +++ b/docs/cn/README.md @@ -202,6 +202,8 @@ spec: nameServers: 192.168.130.33:9876 # replicationMode is the broker replica sync mode, can be ASYNC or SYNC replicationMode: ASYNC + # Whether enable rocketmq-on-dleger group deploy + enableDLeger: false # replicaPerGroup is the number of replica broker in each group replicaPerGroup: 1 # brokerImage is the customized docker image repo of the RocketMQ broker diff --git a/example/rocketmq_v1alpha1_broker_cr.yaml b/example/rocketmq_v1alpha1_broker_cr.yaml index e3d9bd3e..4385935b 100644 --- a/example/rocketmq_v1alpha1_broker_cr.yaml +++ b/example/rocketmq_v1alpha1_broker_cr.yaml @@ -38,6 +38,8 @@ spec: size: 1 # nameServers is the [ip:port] list of name service nameServers: "" + # Whether enable rocketmq-on-dleger group deploy + enableDLeger: false # replicaPerGroup is the number of each broker cluster replicaPerGroup: 1 # brokerImage is the customized docker image repo of the RocketMQ broker @@ -67,6 +69,8 @@ spec: configMapKeyRef: name: broker-config key: BROKER_MEM + - name: TZ + value: Asia/Shanghai # volumes defines the broker.conf volumes: - name: broker-config @@ -82,7 +86,7 @@ spec: annotations: volume.beta.kubernetes.io/storage-class: rocketmq-storage spec: - accessModes: [ "ReadWriteOnce" ] + accessModes: ["ReadWriteOnce"] resources: requests: storage: 8Gi diff --git a/example/rocketmq_v1alpha1_console_cr.yaml b/example/rocketmq_v1alpha1_console_cr.yaml index 3761ac99..63af2113 100644 --- a/example/rocketmq_v1alpha1_console_cr.yaml +++ b/example/rocketmq_v1alpha1_console_cr.yaml @@ -27,4 +27,9 @@ spec: - name: console image: apacherocketmq/rocketmq-console:2.0.0 ports: - - containerPort: 8080 \ No newline at end of file + - containerPort: 8080 + env: + - name: TZ + value: Asia/Shanghai + - name: JAVA_OPTS + value: -Drocketmq.namesrv.addr=rocketmq-name-service:9876 -Dcom.rocketmq.sendMessageWithVIPChannel=false diff --git a/example/rocketmq_v1alpha1_nameservice_cr.yaml b/example/rocketmq_v1alpha1_nameservice_cr.yaml index 291d4471..3953494f 100644 --- a/example/rocketmq_v1alpha1_nameservice_cr.yaml +++ b/example/rocketmq_v1alpha1_nameservice_cr.yaml @@ -33,6 +33,19 @@ spec: # To have DNS options set along with hostNetwork, you have to specify DNS policy # explicitly to 'ClusterFirstWithHostNet'. dnsPolicy: ClusterFirstWithHostNet + env: + - name: TZ + value: Asia/Shanghai + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: nameserver + operator: In + values: + - true + tolerations: [] # resources describes the compute resource requirements and limits resources: requests: @@ -52,7 +65,7 @@ spec: annotations: volume.beta.kubernetes.io/storage-class: rocketmq-storage spec: - accessModes: [ "ReadWriteOnce" ] + accessModes: ["ReadWriteOnce"] resources: requests: storage: 1Gi diff --git a/example/rocketmq_v1alpha1_rocketmq_cluster.yaml b/example/rocketmq_v1alpha1_rocketmq_cluster.yaml index a2adb267..baae2a71 100644 --- a/example/rocketmq_v1alpha1_rocketmq_cluster.yaml +++ b/example/rocketmq_v1alpha1_rocketmq_cluster.yaml @@ -41,12 +41,24 @@ spec: size: 1 # nameServers is the [ip:port] list of name service nameServers: "" + # Whether enable rocketmq-on-dleger group deploy + enableDLeger: false # replicaPerGroup is the number of each broker cluster replicaPerGroup: 1 # brokerImage is the customized docker image repo of the RocketMQ broker brokerImage: apacherocketmq/rocketmq-broker:4.5.0-alpine-operator-0.3.0 # imagePullPolicy is the image pull policy imagePullPolicy: Always + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: broker + operator: In + values: + - "true" + tolerations: [] # resources describes the compute resource requirements and limits resources: requests: @@ -70,6 +82,8 @@ spec: configMapKeyRef: name: broker-config key: BROKER_MEM + - name: TZ + value: Asia/Shanghai # volumes defines the broker.conf volumes: - name: broker-config @@ -111,6 +125,19 @@ spec: # To have DNS options set along with hostNetwork, you have to specify DNS policy # explicitly to 'ClusterFirstWithHostNet'. dnsPolicy: ClusterFirstWithHostNet + env: + - name: TZ + value: Asia/Shanghai + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: nameserver + operator: In + values: + - true + tolerations: [] # resources describes the compute resource requirements and limits resources: requests: @@ -165,4 +192,9 @@ spec: - name: console image: apacherocketmq/rocketmq-console:2.0.0 ports: - - containerPort: 8080 \ No newline at end of file + - containerPort: 8080 + env: + - name: TZ + value: Asia/Shanghai + - name: JAVA_OPTS + value: -Drocketmq.namesrv.addr=rocketmq-name-service:9876 -Dcom.rocketmq.sendMessageWithVIPChannel=false diff --git a/images/broker/alpine/brokerGenConfig.sh b/images/broker/alpine/brokerGenConfig.sh index e6819744..02b46b4e 100755 --- a/images/broker/alpine/brokerGenConfig.sh +++ b/images/broker/alpine/brokerGenConfig.sh @@ -30,6 +30,17 @@ function create_config() { if [ $BROKER_ID != 0 ]; then sed -i 's/brokerRole=.*/brokerRole=SLAVE/g' $BROKER_CONFIG_FILE fi + + # Enable RocketMQ-on-DLedger Group + if [ ! -z $ENABLE_DLEGER ]; then + echo "enableDLegerCommitLog=true" >> $BROKER_CONFIG_FILE + echo "dLegerGroup=$BROKER_NAME" >> $BROKER_CONFIG_FILE + echo "dLegerPeers=$DLEGER_PEERS" >> $BROKER_CONFIG_FILE + echo "dLegerSelfId=$DLEGER_SELF_ID" >> $BROKER_CONFIG_FILE + if [ ! -z $DLEGER_THREAD_NUM ]; then + echo "sendMessageThreadPoolNums=$DLEGER_THREAD_NUM" >> $BROKER_CONFIG_FILE + fi + fi } create_config diff --git a/pkg/apis/rocketmq/v1alpha1/broker_types.go b/pkg/apis/rocketmq/v1alpha1/broker_types.go index 1e694ee5..9dd4e93f 100644 --- a/pkg/apis/rocketmq/v1alpha1/broker_types.go +++ b/pkg/apis/rocketmq/v1alpha1/broker_types.go @@ -34,6 +34,8 @@ type BrokerSpec struct { Size int `json:"size"` // NameServers defines the name service list e.g. 192.168.1.1:9876;192.168.1.2:9876 NameServers string `json:"nameServers,omitempty"` + // Whether enable rocketmq-on-dleger group deploy, false in default + EnableDLeger bool `json:"enableDLeger"` // ReplicaPerGroup each broker cluster's replica number ReplicaPerGroup int `json:"replicaPerGroup"` // BaseImage is the broker image to use for the Pods @@ -41,7 +43,9 @@ type BrokerSpec struct { // ImagePullPolicy defines how the image is pulled ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy"` // AllowRestart defines whether allow pod restart - AllowRestart bool `json:"allowRestart"` + AllowRestart bool `json:"allowRestart"` + Affinity corev1.Affinity `json:"affinity"` + Tolerations []corev1.Toleration `json:"tolerations"` // Resources describes the compute resource requirements Resources corev1.ResourceRequirements `json:"resources"` // StorageMode can be EmptyDir, HostPath, StorageClass diff --git a/pkg/apis/rocketmq/v1alpha1/nameservice_types.go b/pkg/apis/rocketmq/v1alpha1/nameservice_types.go index b8900e9b..bf9d133e 100644 --- a/pkg/apis/rocketmq/v1alpha1/nameservice_types.go +++ b/pkg/apis/rocketmq/v1alpha1/nameservice_types.go @@ -40,7 +40,10 @@ type NameServiceSpec struct { // HostNetwork can be true or false HostNetwork bool `json:"hostNetwork"` // dnsPolicy defines how a pod's DNS will be configured - DNSPolicy corev1.DNSPolicy `json:"dnsPolicy"` + DNSPolicy corev1.DNSPolicy `json:"dnsPolicy"` + Env []corev1.EnvVar `json:"env"` + Affinity corev1.Affinity `json:"affinity"` + Tolerations []corev1.Toleration `json:"tolerations"` // Resources describes the compute resource requirements Resources corev1.ResourceRequirements `json:"resources"` // StorageMode can be EmptyDir, HostPath, StorageClass diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 947edcee..c534ba43 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -64,6 +64,18 @@ const ( // EnvBrokerName is the container environment variable name of broker name EnvBrokerName = "BROKER_NAME" + // EnvBrokerIp is the container environment variable name of broker register ip + EnvBrokerIp = "BROKER_IP" + + // EnvEnableDLeger is the container environment variable name of enable dLeger + EnvEnableDLeger = "ENABLE_DLEGER" + + // EnvDLegerPeers is the container environment variable name of dLeger peers + EnvDLegerPeers = "DLEGER_PEERS" + + // EnvDLegerSelfId is the container environment variable name of dLeger self id + EnvDLegerSelfId = "DLEGER_SELF_ID" + // LogMountPath is the directory of RocketMQ log files LogMountPath = DataPath + "/logs" @@ -94,6 +106,12 @@ const ( // BrokerMainContainerPortName is the main port name of broker container BrokerMainContainerPortName = "main" + // BrokerDlegerContainerPort is the dleger port number of broker container + BrokerDlegerContainerPort = 40911 + + // BrokerDlegerContainerPortName is the dleger port name of broker container + BrokerDlegerContainerPortName = "dleger" + // BrokerHighAvailabilityContainerPort is the high availability port number of broker container BrokerHighAvailabilityContainerPort = 10912 diff --git a/pkg/controller/broker/broker_controller.go b/pkg/controller/broker/broker_controller.go index 5b2de774..b9acac9c 100644 --- a/pkg/controller/broker/broker_controller.go +++ b/pkg/controller/broker/broker_controller.go @@ -36,6 +36,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -104,6 +105,43 @@ type ReconcileBroker struct { scheme *runtime.Scheme } +func (r *ReconcileBroker) CreateService(request reconcile.Request, broker *rocketmqv1alpha1.Broker, brokerGroupIndex int, replicaIndex int) { + reqLogger := log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name) + reqLogger.Info("Create a Broker Service...") + + svc := r.getBrokerService(broker, brokerGroupIndex, replicaIndex) + svcObj := &corev1.Service{} + err := r.client.Get(context.TODO(), types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}, svcObj) + if err != nil && errors.IsNotFound(err) { + reqLogger.Info("Creating a Broker Service.", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) + err = r.client.Create(context.TODO(), svc) + if err != nil { + reqLogger.Error(err, "Failed to create new Service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) + } + } else if err != nil { + reqLogger.Error(err, "Failed to get Broker Service.") + } +} + +func (r *ReconcileBroker) DLegerHostIp(request reconcile.Request, broker *rocketmqv1alpha1.Broker, brokerGroupIndex int, replicaIndex int) string { + reqLogger := log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name) + reqLogger.Info("Get DLeger Host IP lists...") + + statefulSetName := getBrokerStatefulSetName(broker, brokerGroupIndex, replicaIndex) + + svcObj := &corev1.Service{} + for { + err := r.client.Get(context.TODO(), types.NamespacedName{Name: statefulSetName, Namespace: broker.Namespace}, svcObj) + if err != nil && errors.IsNotFound(err) { + log.Info("Waiting for broker service created...") + time.Sleep(time.Duration(cons.WaitForNameServerReadyInSecond) * time.Second) + } else { + break + } + } + return svcObj.Spec.ClusterIP +} + // Reconcile reads that state of the cluster for a Broker object and makes changes based on the state read // and what is in the Broker.Spec // TODO(user): Modify this Reconcile function to implement your Controller logic. This example creates @@ -154,9 +192,18 @@ func (r *ReconcileBroker) Reconcile(request reconcile.Request) (reconcile.Result share.BrokerClusterName = broker.Name replicaPerGroup := broker.Spec.ReplicaPerGroup reqLogger.Info("brokerGroupNum=" + strconv.Itoa(share.GroupNum) + ", replicaPerGroup=" + strconv.Itoa(replicaPerGroup)) + for brokerGroupIndex := 0; brokerGroupIndex < share.GroupNum; brokerGroupIndex++ { reqLogger.Info("Check Broker cluster " + strconv.Itoa(brokerGroupIndex+1) + "/" + strconv.Itoa(share.GroupNum)) - dep := r.getBrokerStatefulSet(broker, brokerGroupIndex, 0) + r.CreateService(request, broker, brokerGroupIndex, 0) + brokerDLegerPeers := "n0-" + r.DLegerHostIp(request, broker, brokerGroupIndex, 0) + ":" + strconv.Itoa(cons.BrokerDlegerContainerPort) + for replicaIndex := 1; replicaIndex <= replicaPerGroup; replicaIndex++ { + r.CreateService(request, broker, brokerGroupIndex, replicaIndex) + brokerDLegerPeers = brokerDLegerPeers + ";n" + strconv.Itoa(replicaIndex) + "-" + r.DLegerHostIp(request, broker, brokerGroupIndex, replicaIndex) + ":" + strconv.Itoa(cons.BrokerDlegerContainerPort) + } + + brokerIp := r.DLegerHostIp(request, broker, brokerGroupIndex, 0) + dep := r.getBrokerStatefulSet(broker, brokerGroupIndex, 0, brokerDLegerPeers, brokerIp) // Check if the statefulSet already exists, if not create a new one found := &appsv1.StatefulSet{} err = r.client.Get(context.TODO(), types.NamespacedName{Name: dep.Name, Namespace: dep.Namespace}, found) @@ -172,7 +219,8 @@ func (r *ReconcileBroker) Reconcile(request reconcile.Request) (reconcile.Result for replicaIndex := 1; replicaIndex <= replicaPerGroup; replicaIndex++ { reqLogger.Info("Check Replica Broker of cluster-" + strconv.Itoa(brokerGroupIndex) + " " + strconv.Itoa(replicaIndex) + "/" + strconv.Itoa(replicaPerGroup)) - replicaDep := r.getBrokerStatefulSet(broker, brokerGroupIndex, replicaIndex) + brokerIp = r.DLegerHostIp(request, broker, brokerGroupIndex, replicaIndex) + replicaDep := r.getBrokerStatefulSet(broker, brokerGroupIndex, replicaIndex, brokerDLegerPeers, brokerIp) err = r.client.Get(context.TODO(), types.NamespacedName{Name: replicaDep.Name, Namespace: replicaDep.Namespace}, found) if err != nil && errors.IsNotFound(err) { reqLogger.Info("Creating a new Replica Broker StatefulSet.", "StatefulSet.Namespace", replicaDep.Namespace, "StatefulSet.Name", replicaDep.Name) @@ -193,8 +241,14 @@ func (r *ReconcileBroker) Reconcile(request reconcile.Request) (reconcile.Result for brokerGroupIndex := 0; brokerGroupIndex < broker.Spec.Size; brokerGroupIndex++ { brokerName := getBrokerName(broker, brokerGroupIndex) // Update master broker + brokerDLegerPeers := "n0-" + r.DLegerHostIp(request, broker, brokerGroupIndex, 0) + ":" + strconv.Itoa(cons.BrokerDlegerContainerPort) + for replicaIndex := 1; replicaIndex <= replicaPerGroup; replicaIndex++ { + r.CreateService(request, broker, brokerGroupIndex, replicaIndex) + brokerDLegerPeers = brokerDLegerPeers + ";n" + strconv.Itoa(replicaIndex) + "-" + r.DLegerHostIp(request, broker, brokerGroupIndex, replicaIndex) + ":" + strconv.Itoa(cons.BrokerDlegerContainerPort) + } reqLogger.Info("Update Master Broker NAMESRV_ADDR of " + brokerName) - dep := r.getBrokerStatefulSet(broker, brokerGroupIndex, 0) + brokerIp := r.DLegerHostIp(request, broker, brokerGroupIndex, 0) + dep := r.getBrokerStatefulSet(broker, brokerGroupIndex, 0, brokerDLegerPeers, brokerIp) found := &appsv1.StatefulSet{} err = r.client.Get(context.TODO(), types.NamespacedName{Name: dep.Name, Namespace: dep.Namespace}, found) if err != nil { @@ -212,7 +266,8 @@ func (r *ReconcileBroker) Reconcile(request reconcile.Request) (reconcile.Result // Update replicas brokers for replicaIndex := 1; replicaIndex <= replicaPerGroup; replicaIndex++ { reqLogger.Info("Update Replica Broker NAMESRV_ADDR of " + brokerName + " " + strconv.Itoa(replicaIndex) + "/" + strconv.Itoa(replicaPerGroup)) - replicaDep := r.getBrokerStatefulSet(broker, brokerGroupIndex, replicaIndex) + brokerIp = r.DLegerHostIp(request, broker, brokerGroupIndex, replicaIndex) + replicaDep := r.getBrokerStatefulSet(broker, brokerGroupIndex, replicaIndex, brokerDLegerPeers, brokerIp) replicaFound := &appsv1.StatefulSet{} err = r.client.Get(context.TODO(), types.NamespacedName{Name: replicaDep.Name, Namespace: replicaDep.Namespace}, replicaFound) if err != nil { @@ -373,18 +428,73 @@ func getBrokerName(broker *rocketmqv1alpha1.Broker, brokerGroupIndex int) string return broker.Name + "-" + strconv.Itoa(brokerGroupIndex) } -// getBrokerStatefulSet returns a broker StatefulSet object -func (r *ReconcileBroker) getBrokerStatefulSet(broker *rocketmqv1alpha1.Broker, brokerGroupIndex int, replicaIndex int) *appsv1.StatefulSet { - ls := labelsForBroker(broker.Name) - var a int32 = 1 - var c = &a +func getBrokerStatefulSetServiceName(broker *rocketmqv1alpha1.Broker, brokerGroupIndex int) string { + statefulSetServiceName := "" + if broker.Spec.EnableDLeger { + statefulSetServiceName = broker.Name + "-" + strconv.Itoa(brokerGroupIndex) + } + return statefulSetServiceName +} + +func getBrokerStatefulSetName(broker *rocketmqv1alpha1.Broker, brokerGroupIndex int, replicaIndex int) string { var statefulSetName string - if replicaIndex == 0 { - statefulSetName = broker.Name + "-" + strconv.Itoa(brokerGroupIndex) + "-master" - } else { + if broker.Spec.EnableDLeger { statefulSetName = broker.Name + "-" + strconv.Itoa(brokerGroupIndex) + "-replica-" + strconv.Itoa(replicaIndex) + } else { + if replicaIndex == 0 { + statefulSetName = broker.Name + "-" + strconv.Itoa(brokerGroupIndex) + "-master" + } else { + statefulSetName = broker.Name + "-" + strconv.Itoa(brokerGroupIndex) + "-replica-" + strconv.Itoa(replicaIndex) + } + } + + return statefulSetName +} + +func (r *ReconcileBroker) getBrokerService(broker *rocketmqv1alpha1.Broker, brokerGroupIndex int, replicaIndex int) *corev1.Service { + statefulSetName := getBrokerStatefulSetName(broker, brokerGroupIndex, replicaIndex) + ls := labelsForBrokerInstance(broker.Name, statefulSetName) + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: statefulSetName, + Labels: ls, + Namespace: broker.Namespace, + }, + Spec: corev1.ServiceSpec{ + Selector: ls, + Ports: []corev1.ServicePort{ + { + Name: cons.BrokerDlegerContainerPortName, + Port: cons.BrokerDlegerContainerPort, + TargetPort: intstr.FromInt(cons.BrokerDlegerContainerPort), + }, { + Name: cons.BrokerMainContainerPortName, + Port: cons.BrokerMainContainerPort, + TargetPort: intstr.FromInt(cons.BrokerMainContainerPort), + }, { + Name: cons.BrokerVipContainerPortName, + Port: cons.BrokerVipContainerPort, + TargetPort: intstr.FromInt(cons.BrokerVipContainerPort), + }, + }, + }, } + // Set bind for broker crd and svc + controllerutil.SetControllerReference(broker, svc, r.scheme) + + return svc +} + +// getBrokerStatefulSet returns a broker StatefulSet object +func (r *ReconcileBroker) getBrokerStatefulSet(broker *rocketmqv1alpha1.Broker, brokerGroupIndex int, replicaIndex int, dLegerPeers string, brokerIp string) *appsv1.StatefulSet { + var a int32 = 1 + var c = &a + + statefulSetName := getBrokerStatefulSetName(broker, brokerGroupIndex, replicaIndex) + ls := labelsForBrokerInstance(broker.Name, statefulSetName) + serviceName := getBrokerStatefulSetServiceName(broker, brokerGroupIndex) dep := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ Name: statefulSetName, @@ -398,15 +508,18 @@ func (r *ReconcileBroker) getBrokerStatefulSet(broker *rocketmqv1alpha1.Broker, UpdateStrategy: appsv1.StatefulSetUpdateStrategy{ Type: appsv1.RollingUpdateStatefulSetStrategyType, }, + ServiceName: serviceName, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: ls, }, Spec: corev1.PodSpec{ + Affinity: &broker.Spec.Affinity, + Tolerations: broker.Spec.Tolerations, Containers: []corev1.Container{{ Resources: broker.Spec.Resources, - Image: broker.Spec.BrokerImage, - Name: cons.BrokerContainerName, + Image: broker.Spec.BrokerImage, + Name: cons.BrokerContainerName, Lifecycle: &corev1.Lifecycle{ PostStart: &corev1.Handler{ Exec: &corev1.ExecAction{ @@ -415,8 +528,11 @@ func (r *ReconcileBroker) getBrokerStatefulSet(broker *rocketmqv1alpha1.Broker, }, }, ImagePullPolicy: broker.Spec.ImagePullPolicy, - Env: getENV(broker, replicaIndex, brokerGroupIndex), + Env: getENV(broker, replicaIndex, brokerGroupIndex, dLegerPeers, brokerIp), Ports: []corev1.ContainerPort{{ + ContainerPort: cons.BrokerDlegerContainerPort, + Name: cons.BrokerDlegerContainerPortName, + }, { ContainerPort: cons.BrokerVipContainerPort, Name: cons.BrokerVipContainerPortName, }, { @@ -453,10 +569,10 @@ func (r *ReconcileBroker) getBrokerStatefulSet(broker *rocketmqv1alpha1.Broker, } -func getENV(broker *rocketmqv1alpha1.Broker, replicaIndex int, brokerGroupIndex int) []corev1.EnvVar { +func getENV(broker *rocketmqv1alpha1.Broker, replicaIndex int, brokerGroupIndex int, dLegerPeers string, brokerIp string) []corev1.EnvVar { envs := []corev1.EnvVar{{ Name: cons.EnvNameServiceAddress, - Value: share.NameServersStr, + Value: share.NameServersServiceStr, }, { Name: cons.EnvBrokerId, Value: strconv.Itoa(replicaIndex), @@ -466,7 +582,25 @@ func getENV(broker *rocketmqv1alpha1.Broker, replicaIndex int, brokerGroupIndex }, { Name: cons.EnvBrokerName, Value: broker.Name + "-" + strconv.Itoa(brokerGroupIndex), + }, { + Name: cons.EnvBrokerIp, + Value: brokerIp, }} + + if broker.Spec.EnableDLeger { + dLegerEnvs := []corev1.EnvVar{{ + Name: cons.EnvEnableDLeger, + Value: "true", + }, { + Name: cons.EnvDLegerPeers, + Value: dLegerPeers, + }, { + Name: cons.EnvDLegerSelfId, + Value: "n" + strconv.Itoa(replicaIndex), + }} + envs = append(envs, dLegerEnvs...) + } + envs = append(envs, broker.Spec.Env...) return envs } @@ -522,6 +656,12 @@ func labelsForBroker(name string) map[string]string { return map[string]string{"app": "broker", "broker_cr": name} } +// labelsForBrokerInstance returns the labels for selecting the resources +// belonging to the given broker CR name and broker instance name. +func labelsForBrokerInstance(name string, instanceName string) map[string]string { + return map[string]string{"app": "broker", "broker_cr": name, "name": instanceName} +} + // getPodNames returns the pod names of the array of pods passed in func getPodNames(pods []corev1.Pod) []string { var podNames []string diff --git a/pkg/controller/console/console_controller.go b/pkg/controller/console/console_controller.go index 3044d6ff..a7755dae 100644 --- a/pkg/controller/console/console_controller.go +++ b/pkg/controller/console/console_controller.go @@ -19,7 +19,9 @@ package console import ( "context" - "fmt" + "reflect" + "time" + rocketmqv1alpha1 "github.com/apache/rocketmq-operator/pkg/apis/rocketmq/v1alpha1" cons "github.com/apache/rocketmq-operator/pkg/constants" "github.com/apache/rocketmq-operator/pkg/share" @@ -29,7 +31,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "reflect" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -38,7 +39,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" "sigs.k8s.io/controller-runtime/pkg/source" - "time" ) var log = logf.Log.WithName("controller_console") @@ -179,11 +179,6 @@ func (r *ReconcileConsole) Reconcile(request reconcile.Request) (reconcile.Resul // newDeploymentForCR returns a deployment pod with modifying the ENV func newDeploymentForCR(cr *rocketmqv1alpha1.Console) *appsv1.Deployment { - env := corev1.EnvVar{ - Name: "JAVA_OPTS", - Value: fmt.Sprintf("-Drocketmq.namesrv.addr=%s -Dcom.rocketmq.sendMessageWithVIPChannel=false", share.NameServersStr), - } - dep := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: cr.Name, @@ -200,12 +195,12 @@ func newDeploymentForCR(cr *rocketmqv1alpha1.Console) *appsv1.Deployment { }, Spec: corev1.PodSpec{ Containers: []corev1.Container{{ - Resources: cr.Spec.ConsoleDeployment.Spec.Template.Spec.Containers[0].Resources, - Image: cr.Spec.ConsoleDeployment.Spec.Template.Spec.Containers[0].Image, - Name: cr.Spec.ConsoleDeployment.Spec.Template.Spec.Containers[0].Name, + Resources: cr.Spec.ConsoleDeployment.Spec.Template.Spec.Containers[0].Resources, + Image: cr.Spec.ConsoleDeployment.Spec.Template.Spec.Containers[0].Image, + Name: cr.Spec.ConsoleDeployment.Spec.Template.Spec.Containers[0].Name, ImagePullPolicy: cr.Spec.ConsoleDeployment.Spec.Template.Spec.Containers[0].ImagePullPolicy, - Env: append(cr.Spec.ConsoleDeployment.Spec.Template.Spec.Containers[0].Env, env), - Ports: cr.Spec.ConsoleDeployment.Spec.Template.Spec.Containers[0].Ports, + Env: cr.Spec.ConsoleDeployment.Spec.Template.Spec.Containers[0].Env, + Ports: cr.Spec.ConsoleDeployment.Spec.Template.Spec.Containers[0].Ports, }}, }, }, diff --git a/pkg/controller/nameservice/nameservice_controller.go b/pkg/controller/nameservice/nameservice_controller.go index 410515f8..dc6cfee1 100644 --- a/pkg/controller/nameservice/nameservice_controller.go +++ b/pkg/controller/nameservice/nameservice_controller.go @@ -35,6 +35,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -128,7 +129,7 @@ func (r *ReconcileNameService) Reconcile(request reconcile.Request) (reconcile.R // Check if the statefulSet already exists, if not create a new one found := &appsv1.StatefulSet{} - + r.CreateService(request, instance) dep := r.statefulSetForNameService(instance) err = r.client.Get(context.TODO(), types.NamespacedName{Name: dep.Name, Namespace: dep.Namespace}, found) if err != nil && errors.IsNotFound(err) { @@ -157,9 +158,18 @@ func (r *ReconcileNameService) Reconcile(request reconcile.Request) (reconcile.R return r.updateNameServiceStatus(instance, request, true) } +func getNameServiceName(nameService *rocketmqv1alpha1.NameService) string { + nameserviceName := "" + for replicaIndex := 0; replicaIndex < int(nameService.Spec.Size); replicaIndex++ { + nameserviceName = nameserviceName + nameService.Name + "-" + strconv.Itoa(replicaIndex) + "." + nameService.Name + ":" + strconv.Itoa(cons.NameServiceMainContainerPort) + ";" + } + return nameserviceName[:len(nameserviceName)-1] +} + func (r *ReconcileNameService) updateNameServiceStatus(instance *rocketmqv1alpha1.NameService, request reconcile.Request, requeue bool) (reconcile.Result, error) { reqLogger := log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name) reqLogger.Info("Check the NameServers status") + share.NameServersServiceStr = getNameServiceName(instance) // List the pods for this nameService's statefulSet podList := &corev1.PodList{} labelSelector := labels.SelectorFromSet(labelsForNameService(instance.Name)) @@ -301,7 +311,53 @@ func labelsForNameService(name string) map[string]string { return map[string]string{"app": "name_service", "name_service_cr": name} } +func (r *ReconcileNameService) getNameServiceService(nameService *rocketmqv1alpha1.NameService) *corev1.Service { + statefulSetName := nameService.Name + ls := labelsForNameService(nameService.Name) + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: statefulSetName, + Labels: ls, + Namespace: nameService.Namespace, + }, + Spec: corev1.ServiceSpec{ + Selector: ls, + ClusterIP: "None", + Ports: []corev1.ServicePort{{ + Name: cons.NameServiceMainContainerPortName, + Port: cons.NameServiceMainContainerPort, + TargetPort: intstr.FromInt(cons.NameServiceMainContainerPort), + }}, + }, + } + + // Set bind for broker crd and svc + controllerutil.SetControllerReference(nameService, svc, r.scheme) + + return svc +} + +func (r *ReconcileNameService) CreateService(request reconcile.Request, nameService *rocketmqv1alpha1.NameService) { + reqLogger := log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name) + reqLogger.Info("Create a Name Service...") + + svc := r.getNameServiceService(nameService) + svcObj := &corev1.Service{} + err := r.client.Get(context.TODO(), types.NamespacedName{Name: svc.Name, Namespace: svc.Namespace}, svcObj) + if err != nil && errors.IsNotFound(err) { + reqLogger.Info("Creating a Name Service.", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) + err = r.client.Create(context.TODO(), svc) + if err != nil { + reqLogger.Error(err, "Failed to create new Service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) + } + } else if err != nil { + reqLogger.Error(err, "Failed to get Name Service.") + } +} + func (r *ReconcileNameService) statefulSetForNameService(nameService *rocketmqv1alpha1.NameService) *appsv1.StatefulSet { + serviceName := nameService.Name ls := labelsForNameService(nameService.Name) dep := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ @@ -313,19 +369,23 @@ func (r *ReconcileNameService) statefulSetForNameService(nameService *rocketmqv1 Selector: &metav1.LabelSelector{ MatchLabels: ls, }, + ServiceName: serviceName, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: ls, }, Spec: corev1.PodSpec{ HostNetwork: nameService.Spec.HostNetwork, - DNSPolicy: nameService.Spec.DNSPolicy, + DNSPolicy: nameService.Spec.DNSPolicy, + Affinity: &nameService.Spec.Affinity, + Tolerations: nameService.Spec.Tolerations, Containers: []corev1.Container{{ Resources: nameService.Spec.Resources, - Image: nameService.Spec.NameServiceImage, + Image: nameService.Spec.NameServiceImage, // Name must be lower case ! Name: "name-service", ImagePullPolicy: nameService.Spec.ImagePullPolicy, + Env: nameService.Spec.Env, Ports: []corev1.ContainerPort{{ ContainerPort: cons.NameServiceMainContainerPort, Name: cons.NameServiceMainContainerPortName, diff --git a/pkg/share/share.go b/pkg/share/share.go index bd1304bf..82934725 100644 --- a/pkg/share/share.go +++ b/pkg/share/share.go @@ -25,6 +25,9 @@ var ( // NameServersStr is the name server list NameServersStr = "" + // NameServersStr is the name server list + NameServersServiceStr = "" + // IsNameServersStrUpdated is whether the name server list is updated IsNameServersStrUpdated = false