diff --git a/apis/apps/defaults/v1alpha1.go b/apis/apps/defaults/v1alpha1.go index f4e1a67d9c..37c5a0b85a 100644 --- a/apis/apps/defaults/v1alpha1.go +++ b/apis/apps/defaults/v1alpha1.go @@ -21,6 +21,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" v1 "k8s.io/kubernetes/pkg/apis/core/v1" + utilpointer "k8s.io/utils/pointer" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -175,6 +176,29 @@ func setDefaultContainer(sidecarContainer *v1alpha1.SidecarContainer) { } } +// SetDefaultsConfigMapSet SetDefaults_ConfigMapSet set default values for ConfigMapSet. +func SetDefaultsConfigMapSet(obj *v1alpha1.ConfigMapSet) { + if obj.Spec.UpdateStrategy == nil { + obj.Spec.UpdateStrategy = &v1alpha1.ConfigMapSetUpdateStrategy{} + } + + partitionValue := int32(0) + maxUnavailableValue := intstr.FromInt32(1) + if obj.Spec.UpdateStrategy == nil { + obj.Spec.UpdateStrategy = &v1alpha1.ConfigMapSetUpdateStrategy{} + } + if obj.Spec.UpdateStrategy.Partition == nil { + v := intstr.FromInt32(partitionValue) + obj.Spec.UpdateStrategy.Partition = &v + } + if obj.Spec.UpdateStrategy.MaxUnavailable == nil { + obj.Spec.UpdateStrategy.MaxUnavailable = &maxUnavailableValue + } + if obj.Spec.RevisionHistoryLimit == nil { + obj.Spec.RevisionHistoryLimit = utilpointer.Int32(5) + } +} + // SetDefaults_UnitedDeployment set default values for UnitedDeployment. func SetDefaultsUnitedDeployment(obj *v1alpha1.UnitedDeployment, injectTemplateDefaults bool) { if obj.Spec.RevisionHistoryLimit == nil { diff --git a/apis/apps/v1alpha1/configmapset_types.go b/apis/apps/v1alpha1/configmapset_types.go new file mode 100644 index 0000000000..b073e029f7 --- /dev/null +++ b/apis/apps/v1alpha1/configmapset_types.go @@ -0,0 +1,174 @@ +/* +Copyright 2024 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +// ConfigMapSetSpec defines the desired state of ConfigMapSet +type ConfigMapSetSpec struct { + // CustomVersion alias for the current update revision + // +optional + CustomVersion string `json:"customVersion,omitempty"` + + // Selector is a label query over pods that should be updated + // +optional + Selector *metav1.LabelSelector `json:"selector,omitempty"` + + // Data contains the configuration data to be updated + Data map[string]string `json:"data"` + + // Containers defines the business containers that need to be updated + // +optional + Containers []ConfigMapSetContainer `json:"containers,omitempty"` + + // ReloadSidecarConfig defines the container injected during Pod creation to update configuration files + // +optional + ReloadSidecarConfig *ReloadSidecarConfig `json:"reloadSidecarConfig,omitempty"` + + // EffectPolicy defines how the configuration update takes effect + // +optional + EffectPolicy *EffectPolicy `json:"effectPolicy,omitempty"` + + // RevisionHistoryLimit indicates the maximum quantity of stored revisions about the ConfigMapSet + // +optional + RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"` + + // UpdateStrategy indicates the strategy that the ConfigMapSet controller will use to perform updates + // +optional + UpdateStrategy *ConfigMapSetUpdateStrategy `json:"updateStrategy,omitempty"` +} + +type ConfigMapSetContainer struct { + Name string `json:"name,omitempty"` + NameFrom *SourceContainerNameSource `json:"nameFrom,omitempty"` + MountPath string `json:"mountPath,omitempty"` +} + +type ReloadSidecarConfig struct { + // Type of the reload sidecar config (k8s, sidecarset, custom) + Type ReloadSidecarType `json:"type,omitempty"` + // Config provides the configuration for the chosen type + Config *ReloadSidecarConfigData `json:"config,omitempty"` +} + +type ReloadSidecarType string + +const ( + ReloadSidecarTypeK8s ReloadSidecarType = "k8s" + ReloadSidecarTypeSidecarSet ReloadSidecarType = "sidecarset" + ReloadSidecarTypeCustom ReloadSidecarType = "custom" +) + +type ReloadSidecarConfigData struct { + // Name for k8s type + Name string `json:"name,omitempty"` + // Image for k8s type + Image string `json:"image,omitempty"` + // RestartPolicy for k8s type + RestartPolicy corev1.ContainerRestartPolicy `json:"restartPolicy,omitempty"` + // Command for k8s type + Command []string `json:"command,omitempty"` + + // SidecarSetRef for sidecarset type + SidecarSetRef *SidecarSetRef `json:"sidecarSetRef,omitempty"` + + // ConfigMapRef for custom type + ConfigMapRef *ConfigMapRef `json:"configMapRef,omitempty"` +} + +type SidecarSetRef struct { + Name string `json:"name,omitempty"` + ContainerName string `json:"containerName,omitempty"` +} + +type ConfigMapRef struct { + Name string `json:"name,omitempty"` + Namespace string `json:"namespace,omitempty"` +} + +type EffectPolicy struct { + Type EffectPolicyType `json:"type,omitempty"` + PostHook *PostHookConfig `json:"postHook,omitempty"` +} + +type EffectPolicyType string + +const ( + EffectPolicyTypeReStart EffectPolicyType = "ReStart" + EffectPolicyTypePostHook EffectPolicyType = "PostHook" + EffectPolicyTypeHotUpdate EffectPolicyType = "HotUpdate" +) + +type PostHookConfig struct { + HTTPGet []*corev1.HTTPGetAction `json:"httpGet,omitempty"` + TCPSocket []*corev1.TCPSocketAction `json:"tcpSocket,omitempty"` +} + +type ConfigMapSetUpdateStrategy struct { + Partition *intstr.IntOrString `json:"partition,omitempty"` + MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"` + MatchLabelKeys []string `json:"matchLabelKeys,omitempty"` +} + +// ConfigMapSetStatus defines the observed state of ConfigMapSet +type ConfigMapSetStatus struct { + CurrentCustomVersion string `json:"currentCustomVersion,omitempty"` + CurrentRevision string `json:"currentRevision,omitempty"` + ExpectedUpdatedReplicas int32 `json:"expectedUpdatedReplicas,omitempty"` + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + ReadyReplicas int32 `json:"readyReplicas,omitempty"` + Replicas int32 `json:"replicas,omitempty"` + UpdateCustomVersion string `json:"updateCustomVersion,omitempty"` + UpdateRevision string `json:"updateRevision,omitempty"` + UpdatedReadyReplicas int32 `json:"updatedReadyReplicas,omitempty"` + UpdatedReplicas int32 `json:"updatedReplicas,omitempty"` +} + +// +genclient +// +k8s:openapi-gen=true +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="REPLICAS",type="integer",JSONPath=".status.replicas",description="The number of pods matched." +// +kubebuilder:printcolumn:name="UPDATED",type="integer",JSONPath=".status.updatedReplicas",description="The number of pods matched and updated." +// +kubebuilder:printcolumn:name="READY",type="integer",JSONPath=".status.readyReplicas",description="The number of pods matched and ready." +// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp",description="CreationTimestamp is a timestamp representing the server time when this object was created." + +// ConfigMapSet is the Schema for the configmapsets API +type ConfigMapSet struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ConfigMapSetSpec `json:"spec,omitempty"` + Status ConfigMapSetStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// ConfigMapSetList contains a list of ConfigMapSet +type ConfigMapSetList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ConfigMapSet `json:"items"` +} + +func init() { + SchemeBuilder.Register(&ConfigMapSet{}, &ConfigMapSetList{}) +} diff --git a/apis/apps/v1alpha1/zz_generated.deepcopy.go b/apis/apps/v1alpha1/zz_generated.deepcopy.go index 785dfc00e8..4a73874f57 100644 --- a/apis/apps/v1alpha1/zz_generated.deepcopy.go +++ b/apis/apps/v1alpha1/zz_generated.deepcopy.go @@ -610,6 +610,199 @@ func (in *CompletionPolicy) DeepCopy() *CompletionPolicy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConfigMapRef) DeepCopyInto(out *ConfigMapRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigMapRef. +func (in *ConfigMapRef) DeepCopy() *ConfigMapRef { + if in == nil { + return nil + } + out := new(ConfigMapRef) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConfigMapSet) DeepCopyInto(out *ConfigMapSet) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigMapSet. +func (in *ConfigMapSet) DeepCopy() *ConfigMapSet { + if in == nil { + return nil + } + out := new(ConfigMapSet) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ConfigMapSet) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConfigMapSetContainer) DeepCopyInto(out *ConfigMapSetContainer) { + *out = *in + if in.NameFrom != nil { + in, out := &in.NameFrom, &out.NameFrom + *out = new(SourceContainerNameSource) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigMapSetContainer. +func (in *ConfigMapSetContainer) DeepCopy() *ConfigMapSetContainer { + if in == nil { + return nil + } + out := new(ConfigMapSetContainer) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConfigMapSetList) DeepCopyInto(out *ConfigMapSetList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ConfigMapSet, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigMapSetList. +func (in *ConfigMapSetList) DeepCopy() *ConfigMapSetList { + if in == nil { + return nil + } + out := new(ConfigMapSetList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ConfigMapSetList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConfigMapSetSpec) DeepCopyInto(out *ConfigMapSetSpec) { + *out = *in + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = new(metav1.LabelSelector) + (*in).DeepCopyInto(*out) + } + if in.Data != nil { + in, out := &in.Data, &out.Data + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Containers != nil { + in, out := &in.Containers, &out.Containers + *out = make([]ConfigMapSetContainer, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.ReloadSidecarConfig != nil { + in, out := &in.ReloadSidecarConfig, &out.ReloadSidecarConfig + *out = new(ReloadSidecarConfig) + (*in).DeepCopyInto(*out) + } + if in.EffectPolicy != nil { + in, out := &in.EffectPolicy, &out.EffectPolicy + *out = new(EffectPolicy) + (*in).DeepCopyInto(*out) + } + if in.RevisionHistoryLimit != nil { + in, out := &in.RevisionHistoryLimit, &out.RevisionHistoryLimit + *out = new(int32) + **out = **in + } + if in.UpdateStrategy != nil { + in, out := &in.UpdateStrategy, &out.UpdateStrategy + *out = new(ConfigMapSetUpdateStrategy) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigMapSetSpec. +func (in *ConfigMapSetSpec) DeepCopy() *ConfigMapSetSpec { + if in == nil { + return nil + } + out := new(ConfigMapSetSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConfigMapSetStatus) DeepCopyInto(out *ConfigMapSetStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigMapSetStatus. +func (in *ConfigMapSetStatus) DeepCopy() *ConfigMapSetStatus { + if in == nil { + return nil + } + out := new(ConfigMapSetStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConfigMapSetUpdateStrategy) DeepCopyInto(out *ConfigMapSetUpdateStrategy) { + *out = *in + if in.Partition != nil { + in, out := &in.Partition, &out.Partition + *out = new(intstr.IntOrString) + **out = **in + } + if in.MaxUnavailable != nil { + in, out := &in.MaxUnavailable, &out.MaxUnavailable + *out = new(intstr.IntOrString) + **out = **in + } + if in.MatchLabelKeys != nil { + in, out := &in.MatchLabelKeys, &out.MatchLabelKeys + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigMapSetUpdateStrategy. +func (in *ConfigMapSetUpdateStrategy) DeepCopy() *ConfigMapSetUpdateStrategy { + if in == nil { + return nil + } + out := new(ConfigMapSetUpdateStrategy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ContainerProbe) DeepCopyInto(out *ContainerProbe) { *out = *in @@ -1064,6 +1257,26 @@ func (in *DeploymentTemplateSpec) DeepCopy() *DeploymentTemplateSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EffectPolicy) DeepCopyInto(out *EffectPolicy) { + *out = *in + if in.PostHook != nil { + in, out := &in.PostHook, &out.PostHook + *out = new(PostHookConfig) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EffectPolicy. +func (in *EffectPolicy) DeepCopy() *EffectPolicy { + if in == nil { + return nil + } + out := new(EffectPolicy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *EphemeralContainerTemplateSpec) DeepCopyInto(out *EphemeralContainerTemplateSpec) { *out = *in @@ -2300,6 +2513,43 @@ func (in *PodState) DeepCopy() *PodState { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PostHookConfig) DeepCopyInto(out *PostHookConfig) { + *out = *in + if in.HTTPGet != nil { + in, out := &in.HTTPGet, &out.HTTPGet + *out = make([]*corev1.HTTPGetAction, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(corev1.HTTPGetAction) + (*in).DeepCopyInto(*out) + } + } + } + if in.TCPSocket != nil { + in, out := &in.TCPSocket, &out.TCPSocket + *out = make([]*corev1.TCPSocketAction, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(corev1.TCPSocketAction) + **out = **in + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PostHookConfig. +func (in *PostHookConfig) DeepCopy() *PostHookConfig { + if in == nil { + return nil + } + out := new(PostHookConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PreferredTopologyTerm) DeepCopyInto(out *PreferredTopologyTerm) { *out = *in @@ -2415,6 +2665,56 @@ func (in *ReferenceObject) DeepCopy() *ReferenceObject { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReloadSidecarConfig) DeepCopyInto(out *ReloadSidecarConfig) { + *out = *in + if in.Config != nil { + in, out := &in.Config, &out.Config + *out = new(ReloadSidecarConfigData) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReloadSidecarConfig. +func (in *ReloadSidecarConfig) DeepCopy() *ReloadSidecarConfig { + if in == nil { + return nil + } + out := new(ReloadSidecarConfig) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReloadSidecarConfigData) DeepCopyInto(out *ReloadSidecarConfigData) { + *out = *in + if in.Command != nil { + in, out := &in.Command, &out.Command + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.SidecarSetRef != nil { + in, out := &in.SidecarSetRef, &out.SidecarSetRef + *out = new(SidecarSetRef) + **out = **in + } + if in.ConfigMapRef != nil { + in, out := &in.ConfigMapRef, &out.ConfigMapRef + *out = new(ConfigMapRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReloadSidecarConfigData. +func (in *ReloadSidecarConfigData) DeepCopy() *ReloadSidecarConfigData { + if in == nil { + return nil + } + out := new(ReloadSidecarConfigData) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResourceDistribution) DeepCopyInto(out *ResourceDistribution) { *out = *in @@ -2958,6 +3258,21 @@ func (in *SidecarSetPatchPodMetadata) DeepCopy() *SidecarSetPatchPodMetadata { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SidecarSetRef) DeepCopyInto(out *SidecarSetRef) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SidecarSetRef. +func (in *SidecarSetRef) DeepCopy() *SidecarSetRef { + if in == nil { + return nil + } + out := new(SidecarSetRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SidecarSetSpec) DeepCopyInto(out *SidecarSetSpec) { *out = *in diff --git a/config/crd/bases/apps.kruise.io_configmapsets.yaml b/config/crd/bases/apps.kruise.io_configmapsets.yaml new file mode 100644 index 0000000000..27787031d4 --- /dev/null +++ b/config/crd/bases/apps.kruise.io_configmapsets.yaml @@ -0,0 +1,333 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.3 + name: configmapsets.apps.kruise.io +spec: + group: apps.kruise.io + names: + kind: ConfigMapSet + listKind: ConfigMapSetList + plural: configmapsets + singular: configmapset + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: The number of pods matched. + jsonPath: .status.replicas + name: REPLICAS + type: integer + - description: The number of pods matched and updated. + jsonPath: .status.updatedReplicas + name: UPDATED + type: integer + - description: The number of pods matched and ready. + jsonPath: .status.readyReplicas + name: READY + type: integer + - description: CreationTimestamp is a timestamp representing the server time when + this object was created. + jsonPath: .metadata.creationTimestamp + name: AGE + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: ConfigMapSet is the Schema for the configmapsets API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ConfigMapSetSpec defines the desired state of ConfigMapSet + properties: + containers: + description: Containers defines the business containers that need + to be updated + items: + properties: + mountPath: + type: string + name: + type: string + nameFrom: + properties: + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, + `metadata.labels['''']`, `metadata.annotations['''']`,' + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + type: object + type: object + type: array + customVersion: + description: CustomVersion alias for the current update revision + type: string + data: + additionalProperties: + type: string + description: Data contains the configuration data to be updated + type: object + effectPolicy: + description: EffectPolicy defines how the configuration update takes + effect + properties: + postHook: + properties: + httpGet: + items: + description: HTTPGetAction describes an action based on + HTTP Get requests. + properties: + host: + description: |- + Host name to connect to, defaults to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP + allows repeated headers. + items: + description: HTTPHeader describes a custom header + to be used in HTTP probes + properties: + name: + description: |- + The header field name. + This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: |- + Scheme to use for connecting to the host. + Defaults to HTTP. + type: string + required: + - port + type: object + type: array + tcpSocket: + items: + description: TCPSocketAction describes an action based on + opening a socket + properties: + host: + description: 'Optional: Host name to connect to, defaults + to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: |- + Number or name of the port to access on the container. + Number must be in the range 1 to 65535. + Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + type: object + type: + type: string + type: object + reloadSidecarConfig: + description: ReloadSidecarConfig defines the container injected during + Pod creation to update configuration files + properties: + config: + description: Config provides the configuration for the chosen + type + properties: + command: + description: Command for k8s type + items: + type: string + type: array + configMapRef: + description: ConfigMapRef for custom type + properties: + name: + type: string + namespace: + type: string + type: object + image: + description: Image for k8s type + type: string + name: + description: Name for k8s type + type: string + restartPolicy: + description: RestartPolicy for k8s type + type: string + sidecarSetRef: + description: SidecarSetRef for sidecarset type + properties: + containerName: + type: string + name: + type: string + type: object + type: object + type: + description: Type of the reload sidecar config (k8s, sidecarset, + custom) + type: string + type: object + revisionHistoryLimit: + description: RevisionHistoryLimit indicates the maximum quantity of + stored revisions about the ConfigMapSet + format: int32 + type: integer + selector: + description: Selector is a label query over pods that should be updated + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + updateStrategy: + description: UpdateStrategy indicates the strategy that the ConfigMapSet + controller will use to perform updates + properties: + matchLabelKeys: + items: + type: string + type: array + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + partition: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object + required: + - data + type: object + status: + description: ConfigMapSetStatus defines the observed state of ConfigMapSet + properties: + currentCustomVersion: + type: string + currentRevision: + type: string + expectedUpdatedReplicas: + format: int32 + type: integer + observedGeneration: + format: int64 + type: integer + readyReplicas: + format: int32 + type: integer + replicas: + format: int32 + type: integer + updateCustomVersion: + type: string + updateRevision: + type: string + updatedReadyReplicas: + format: int32 + type: integer + updatedReplicas: + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 0aabd53a6c..8a5ed37fa9 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -127,6 +127,7 @@ rules: - advancedcronjobs - broadcastjobs - clonesets + - configmapsets - containerrecreaterequests - daemonsets - imagelistpulljobs @@ -173,6 +174,7 @@ rules: - advancedcronjobs/status - broadcastjobs/status - clonesets/status + - configmapsets/status - containerrecreaterequests/status - daemonsets/status - ephemeraljobs/finalizers diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml index d05b536304..0f4bbeacab 100644 --- a/config/webhook/manifests.yaml +++ b/config/webhook/manifests.yaml @@ -90,6 +90,27 @@ webhooks: resources: - clonesets sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-apps-kruise-io-v1alpha1-configmapset + failurePolicy: Fail + name: mconfigmapset.kb.io + rules: + - apiGroups: + - apps.kruise.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - configmapsets + sideEffects: None - admissionReviewVersions: - v1 - v1beta1 @@ -439,6 +460,27 @@ webhooks: resources: - clonesets sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-apps-kruise-io-v1alpha1-configmapset + failurePolicy: Fail + name: vconfigmapset.kb.io + rules: + - apiGroups: + - apps.kruise.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - configmapsets + sideEffects: None - admissionReviewVersions: - v1 - v1beta1 diff --git a/go.mod b/go.mod index f2bad031e4..ba28e61d66 100644 --- a/go.mod +++ b/go.mod @@ -61,6 +61,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/nxadm/tail v1.4.8 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect github.com/x448/float16 v0.8.4 // indirect go.etcd.io/etcd/api/v3 v3.5.16 // indirect @@ -76,6 +77,7 @@ require ( google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect k8s.io/controller-manager v0.32.10 // indirect k8s.io/gengo/v2 v2.0.0-20240911193312-2b36238f13e9 // indirect k8s.io/kms v0.32.10 // indirect diff --git a/go.sum b/go.sum index b921d7e75f..6bd319eb32 100644 --- a/go.sum +++ b/go.sum @@ -175,6 +175,7 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= @@ -371,6 +372,7 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/client/clientset/versioned/typed/apps/v1alpha1/apps_client.go b/pkg/client/clientset/versioned/typed/apps/v1alpha1/apps_client.go index a5750c43ea..892dda40e8 100644 --- a/pkg/client/clientset/versioned/typed/apps/v1alpha1/apps_client.go +++ b/pkg/client/clientset/versioned/typed/apps/v1alpha1/apps_client.go @@ -30,6 +30,7 @@ type AppsV1alpha1Interface interface { AdvancedCronJobsGetter BroadcastJobsGetter CloneSetsGetter + ConfigMapSetsGetter ContainerRecreateRequestsGetter DaemonSetsGetter EphemeralJobsGetter @@ -63,6 +64,10 @@ func (c *AppsV1alpha1Client) CloneSets(namespace string) CloneSetInterface { return newCloneSets(c, namespace) } +func (c *AppsV1alpha1Client) ConfigMapSets(namespace string) ConfigMapSetInterface { + return newConfigMapSets(c, namespace) +} + func (c *AppsV1alpha1Client) ContainerRecreateRequests(namespace string) ContainerRecreateRequestInterface { return newContainerRecreateRequests(c, namespace) } diff --git a/pkg/client/clientset/versioned/typed/apps/v1alpha1/configmapset.go b/pkg/client/clientset/versioned/typed/apps/v1alpha1/configmapset.go new file mode 100644 index 0000000000..4338484615 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/apps/v1alpha1/configmapset.go @@ -0,0 +1,69 @@ +/* +Copyright 2025 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + context "context" + + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + scheme "github.com/openkruise/kruise/pkg/client/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// ConfigMapSetsGetter has a method to return a ConfigMapSetInterface. +// A group's client should implement this interface. +type ConfigMapSetsGetter interface { + ConfigMapSets(namespace string) ConfigMapSetInterface +} + +// ConfigMapSetInterface has methods to work with ConfigMapSet resources. +type ConfigMapSetInterface interface { + Create(ctx context.Context, configMapSet *appsv1alpha1.ConfigMapSet, opts v1.CreateOptions) (*appsv1alpha1.ConfigMapSet, error) + Update(ctx context.Context, configMapSet *appsv1alpha1.ConfigMapSet, opts v1.UpdateOptions) (*appsv1alpha1.ConfigMapSet, error) + // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + UpdateStatus(ctx context.Context, configMapSet *appsv1alpha1.ConfigMapSet, opts v1.UpdateOptions) (*appsv1alpha1.ConfigMapSet, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*appsv1alpha1.ConfigMapSet, error) + List(ctx context.Context, opts v1.ListOptions) (*appsv1alpha1.ConfigMapSetList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *appsv1alpha1.ConfigMapSet, err error) + ConfigMapSetExpansion +} + +// configMapSets implements ConfigMapSetInterface +type configMapSets struct { + *gentype.ClientWithList[*appsv1alpha1.ConfigMapSet, *appsv1alpha1.ConfigMapSetList] +} + +// newConfigMapSets returns a ConfigMapSets +func newConfigMapSets(c *AppsV1alpha1Client, namespace string) *configMapSets { + return &configMapSets{ + gentype.NewClientWithList[*appsv1alpha1.ConfigMapSet, *appsv1alpha1.ConfigMapSetList]( + "configmapsets", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *appsv1alpha1.ConfigMapSet { return &appsv1alpha1.ConfigMapSet{} }, + func() *appsv1alpha1.ConfigMapSetList { return &appsv1alpha1.ConfigMapSetList{} }, + ), + } +} diff --git a/pkg/client/clientset/versioned/typed/apps/v1alpha1/fake/fake_apps_client.go b/pkg/client/clientset/versioned/typed/apps/v1alpha1/fake/fake_apps_client.go index cd1c77e756..3281a9603a 100644 --- a/pkg/client/clientset/versioned/typed/apps/v1alpha1/fake/fake_apps_client.go +++ b/pkg/client/clientset/versioned/typed/apps/v1alpha1/fake/fake_apps_client.go @@ -39,6 +39,10 @@ func (c *FakeAppsV1alpha1) CloneSets(namespace string) v1alpha1.CloneSetInterfac return newFakeCloneSets(c, namespace) } +func (c *FakeAppsV1alpha1) ConfigMapSets(namespace string) v1alpha1.ConfigMapSetInterface { + return newFakeConfigMapSets(c, namespace) +} + func (c *FakeAppsV1alpha1) ContainerRecreateRequests(namespace string) v1alpha1.ContainerRecreateRequestInterface { return newFakeContainerRecreateRequests(c, namespace) } diff --git a/pkg/client/clientset/versioned/typed/apps/v1alpha1/fake/fake_configmapset.go b/pkg/client/clientset/versioned/typed/apps/v1alpha1/fake/fake_configmapset.go new file mode 100644 index 0000000000..3beba563f7 --- /dev/null +++ b/pkg/client/clientset/versioned/typed/apps/v1alpha1/fake/fake_configmapset.go @@ -0,0 +1,51 @@ +/* +Copyright 2025 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + appsv1alpha1 "github.com/openkruise/kruise/pkg/client/clientset/versioned/typed/apps/v1alpha1" + gentype "k8s.io/client-go/gentype" +) + +// fakeConfigMapSets implements ConfigMapSetInterface +type fakeConfigMapSets struct { + *gentype.FakeClientWithList[*v1alpha1.ConfigMapSet, *v1alpha1.ConfigMapSetList] + Fake *FakeAppsV1alpha1 +} + +func newFakeConfigMapSets(fake *FakeAppsV1alpha1, namespace string) appsv1alpha1.ConfigMapSetInterface { + return &fakeConfigMapSets{ + gentype.NewFakeClientWithList[*v1alpha1.ConfigMapSet, *v1alpha1.ConfigMapSetList]( + fake.Fake, + namespace, + v1alpha1.SchemeGroupVersion.WithResource("configmapsets"), + v1alpha1.SchemeGroupVersion.WithKind("ConfigMapSet"), + func() *v1alpha1.ConfigMapSet { return &v1alpha1.ConfigMapSet{} }, + func() *v1alpha1.ConfigMapSetList { return &v1alpha1.ConfigMapSetList{} }, + func(dst, src *v1alpha1.ConfigMapSetList) { dst.ListMeta = src.ListMeta }, + func(list *v1alpha1.ConfigMapSetList) []*v1alpha1.ConfigMapSet { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v1alpha1.ConfigMapSetList, items []*v1alpha1.ConfigMapSet) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} diff --git a/pkg/client/clientset/versioned/typed/apps/v1alpha1/generated_expansion.go b/pkg/client/clientset/versioned/typed/apps/v1alpha1/generated_expansion.go index b1f5b406c5..81194962da 100644 --- a/pkg/client/clientset/versioned/typed/apps/v1alpha1/generated_expansion.go +++ b/pkg/client/clientset/versioned/typed/apps/v1alpha1/generated_expansion.go @@ -23,6 +23,8 @@ type BroadcastJobExpansion interface{} type CloneSetExpansion interface{} +type ConfigMapSetExpansion interface{} + type ContainerRecreateRequestExpansion interface{} type DaemonSetExpansion interface{} diff --git a/pkg/client/informers/externalversions/apps/v1alpha1/configmapset.go b/pkg/client/informers/externalversions/apps/v1alpha1/configmapset.go new file mode 100644 index 0000000000..cb55bfe859 --- /dev/null +++ b/pkg/client/informers/externalversions/apps/v1alpha1/configmapset.go @@ -0,0 +1,89 @@ +/* +Copyright 2025 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by informer-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + context "context" + time "time" + + apisappsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + versioned "github.com/openkruise/kruise/pkg/client/clientset/versioned" + internalinterfaces "github.com/openkruise/kruise/pkg/client/informers/externalversions/internalinterfaces" + appsv1alpha1 "github.com/openkruise/kruise/pkg/client/listers/apps/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// ConfigMapSetInformer provides access to a shared informer and lister for +// ConfigMapSets. +type ConfigMapSetInformer interface { + Informer() cache.SharedIndexInformer + Lister() appsv1alpha1.ConfigMapSetLister +} + +type configMapSetInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewConfigMapSetInformer constructs a new informer for ConfigMapSet type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewConfigMapSetInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredConfigMapSetInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredConfigMapSetInformer constructs a new informer for ConfigMapSet type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredConfigMapSetInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AppsV1alpha1().ConfigMapSets(namespace).List(context.TODO(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.AppsV1alpha1().ConfigMapSets(namespace).Watch(context.TODO(), options) + }, + }, + &apisappsv1alpha1.ConfigMapSet{}, + resyncPeriod, + indexers, + ) +} + +func (f *configMapSetInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredConfigMapSetInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *configMapSetInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apisappsv1alpha1.ConfigMapSet{}, f.defaultInformer) +} + +func (f *configMapSetInformer) Lister() appsv1alpha1.ConfigMapSetLister { + return appsv1alpha1.NewConfigMapSetLister(f.Informer().GetIndexer()) +} diff --git a/pkg/client/informers/externalversions/apps/v1alpha1/interface.go b/pkg/client/informers/externalversions/apps/v1alpha1/interface.go index 0a52396825..7c7a51a760 100644 --- a/pkg/client/informers/externalversions/apps/v1alpha1/interface.go +++ b/pkg/client/informers/externalversions/apps/v1alpha1/interface.go @@ -29,6 +29,8 @@ type Interface interface { BroadcastJobs() BroadcastJobInformer // CloneSets returns a CloneSetInformer. CloneSets() CloneSetInformer + // ConfigMapSets returns a ConfigMapSetInformer. + ConfigMapSets() ConfigMapSetInformer // ContainerRecreateRequests returns a ContainerRecreateRequestInformer. ContainerRecreateRequests() ContainerRecreateRequestInformer // DaemonSets returns a DaemonSetInformer. @@ -85,6 +87,11 @@ func (v *version) CloneSets() CloneSetInformer { return &cloneSetInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} } +// ConfigMapSets returns a ConfigMapSetInformer. +func (v *version) ConfigMapSets() ConfigMapSetInformer { + return &configMapSetInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} + // ContainerRecreateRequests returns a ContainerRecreateRequestInformer. func (v *version) ContainerRecreateRequests() ContainerRecreateRequestInformer { return &containerRecreateRequestInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go index aff267bf14..4a3978ff8c 100644 --- a/pkg/client/informers/externalversions/generic.go +++ b/pkg/client/informers/externalversions/generic.go @@ -60,6 +60,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource return &genericInformer{resource: resource.GroupResource(), informer: f.Apps().V1alpha1().BroadcastJobs().Informer()}, nil case v1alpha1.SchemeGroupVersion.WithResource("clonesets"): return &genericInformer{resource: resource.GroupResource(), informer: f.Apps().V1alpha1().CloneSets().Informer()}, nil + case v1alpha1.SchemeGroupVersion.WithResource("configmapsets"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Apps().V1alpha1().ConfigMapSets().Informer()}, nil case v1alpha1.SchemeGroupVersion.WithResource("containerrecreaterequests"): return &genericInformer{resource: resource.GroupResource(), informer: f.Apps().V1alpha1().ContainerRecreateRequests().Informer()}, nil case v1alpha1.SchemeGroupVersion.WithResource("daemonsets"): diff --git a/pkg/client/listers/apps/v1alpha1/configmapset.go b/pkg/client/listers/apps/v1alpha1/configmapset.go new file mode 100644 index 0000000000..172e8d8fc3 --- /dev/null +++ b/pkg/client/listers/apps/v1alpha1/configmapset.go @@ -0,0 +1,69 @@ +/* +Copyright 2025 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by lister-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" +) + +// ConfigMapSetLister helps list ConfigMapSets. +// All objects returned here must be treated as read-only. +type ConfigMapSetLister interface { + // List lists all ConfigMapSets in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*appsv1alpha1.ConfigMapSet, err error) + // ConfigMapSets returns an object that can list and get ConfigMapSets. + ConfigMapSets(namespace string) ConfigMapSetNamespaceLister + ConfigMapSetListerExpansion +} + +// configMapSetLister implements the ConfigMapSetLister interface. +type configMapSetLister struct { + listers.ResourceIndexer[*appsv1alpha1.ConfigMapSet] +} + +// NewConfigMapSetLister returns a new ConfigMapSetLister. +func NewConfigMapSetLister(indexer cache.Indexer) ConfigMapSetLister { + return &configMapSetLister{listers.New[*appsv1alpha1.ConfigMapSet](indexer, appsv1alpha1.Resource("configmapset"))} +} + +// ConfigMapSets returns an object that can list and get ConfigMapSets. +func (s *configMapSetLister) ConfigMapSets(namespace string) ConfigMapSetNamespaceLister { + return configMapSetNamespaceLister{listers.NewNamespaced[*appsv1alpha1.ConfigMapSet](s.ResourceIndexer, namespace)} +} + +// ConfigMapSetNamespaceLister helps list and get ConfigMapSets. +// All objects returned here must be treated as read-only. +type ConfigMapSetNamespaceLister interface { + // List lists all ConfigMapSets in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*appsv1alpha1.ConfigMapSet, err error) + // Get retrieves the ConfigMapSet from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*appsv1alpha1.ConfigMapSet, error) + ConfigMapSetNamespaceListerExpansion +} + +// configMapSetNamespaceLister implements the ConfigMapSetNamespaceLister +// interface. +type configMapSetNamespaceLister struct { + listers.ResourceIndexer[*appsv1alpha1.ConfigMapSet] +} diff --git a/pkg/client/listers/apps/v1alpha1/expansion_generated.go b/pkg/client/listers/apps/v1alpha1/expansion_generated.go index 16cb23c91d..10936c7bd9 100644 --- a/pkg/client/listers/apps/v1alpha1/expansion_generated.go +++ b/pkg/client/listers/apps/v1alpha1/expansion_generated.go @@ -41,6 +41,14 @@ type CloneSetListerExpansion interface{} // CloneSetNamespaceLister. type CloneSetNamespaceListerExpansion interface{} +// ConfigMapSetListerExpansion allows custom methods to be added to +// ConfigMapSetLister. +type ConfigMapSetListerExpansion interface{} + +// ConfigMapSetNamespaceListerExpansion allows custom methods to be added to +// ConfigMapSetNamespaceLister. +type ConfigMapSetNamespaceListerExpansion interface{} + // ContainerRecreateRequestListerExpansion allows custom methods to be added to // ContainerRecreateRequestLister. type ContainerRecreateRequestListerExpansion interface{} diff --git a/pkg/controller/configmapset/configmapset_controller.go b/pkg/controller/configmapset/configmapset_controller.go new file mode 100644 index 0000000000..0a1919c9fe --- /dev/null +++ b/pkg/controller/configmapset/configmapset_controller.go @@ -0,0 +1,1327 @@ +/* +Copyright 2016 The Kubernetes Authors. +Copyright 2019 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package configmapset + +import ( + "context" + "crypto/md5" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "math" + "sort" + "strconv" + "strings" + "time" + + "k8s.io/utils/ptr" + + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/retry" + "k8s.io/klog/v2" + "k8s.io/kubernetes/pkg/controller" + "k8s.io/kubernetes/pkg/fieldpath" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + runController "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + "github.com/openkruise/kruise/pkg/util" + utilclient "github.com/openkruise/kruise/pkg/util/client" + utildiscovery "github.com/openkruise/kruise/pkg/util/discovery" + "github.com/openkruise/kruise/pkg/util/ratelimiter" +) + +func init() { + flag.IntVar(&concurrentReconciles, "configmapset-workers", concurrentReconciles, "Max concurrent workers for ConfigMapSet controller.") +} + +var ( + concurrentReconciles = 3 + controllerKind = appsv1alpha1.SchemeGroupVersion.WithKind("ConfigMapSet") +) + +type RevisionKeys struct { + CurrentRevisionKey string + CurrentRevisionTimeStampKey string + CurrentCustomVersionKey string + UpdateRevisionKey string + UpdateRevisionTimeStampKey string + UpdateCustomVersionKey string +} + +type SpecForHash struct { + CustomVersion string `json:"customVersion,omitempty"` + Selector *metav1.LabelSelector `json:"selector"` + Data map[string]string `json:"data"` + Containers []appsv1alpha1.ConfigMapSetContainer `json:"containers"` + ReloadSidecarConfig *appsv1alpha1.ReloadSidecarConfig `json:"reloadSidecarConfig,omitempty"` + EffectPolicy *appsv1alpha1.EffectPolicy `json:"effectPolicy,omitempty"` + RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"` +} + +// UpdateInfo stores the result of pod update calculation +type UpdateInfo struct { + PodsToUpdate []*corev1.Pod + TargetRevisions []string + TargetCustomVersions []string +} + +const ( + ConfigMapFinalizerName = "finalizer.configmapset.kruise.io" +) + +// RevisionEntry defines the data structure for storing revisions +type RevisionEntry struct { + Hash string `json:"hash"` + CustomVersion string `json:"customVersion"` + TimeStamp *metav1.Time `json:"timeStamp"` + Data map[string]string `json:"data"` +} + +// Add creates a new ConfigMapSet Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller +// and Start it when the Manager is Started. +func Add(mgr manager.Manager) error { + if !utildiscovery.DiscoverGVK(controllerKind) { + return nil + } + return add(mgr, newReconciler(mgr)) +} + +type ReconcileConfigMapSet struct { + client.Client + scheme *runtime.Scheme + recorder record.EventRecorder + config *rest.Config +} + +// newReconciler returns a new reconcile.Reconciler +func newReconciler(mgr manager.Manager) reconcile.Reconciler { + recorder := mgr.GetEventRecorderFor("configmapset-controller") + return &ReconcileConfigMapSet{ + Client: utilclient.NewClientFromManager(mgr, "configmapset-controller"), + scheme: mgr.GetScheme(), + recorder: recorder, + config: mgr.GetConfig(), + } +} + +// add adds a new Controller to mgr with r as the reconcile.Reconciler +func add(mgr manager.Manager, r reconcile.Reconciler) error { + // Create a new controller + c, err := runController.New("configmapset-controller", mgr, runController.Options{ + Reconciler: r, MaxConcurrentReconciles: concurrentReconciles, CacheSyncTimeout: util.GetControllerCacheSyncTimeout(), + RateLimiter: ratelimiter.DefaultControllerRateLimiter[reconcile.Request]()}) + if err != nil { + return err + } + + // Watch for changes to ConfigMapSet + err = c.Watch(source.Kind(mgr.GetCache(), &appsv1alpha1.ConfigMapSet{}, &handler.TypedEnqueueRequestForObject[*appsv1alpha1.ConfigMapSet]{})) + if err != nil { + return err + } + + // Watch for changes to Pod + err = c.Watch(source.Kind(mgr.GetCache(), &v1.Pod{}, &podEventHandler{Reader: mgr.GetCache()})) + if err != nil { + return err + } + + return nil +} + +var _ reconcile.Reconciler = &ReconcileConfigMapSet{} + +// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch +// +kubebuilder:rbac:groups=core,resources=pods/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=apps.kruise.io,resources=configmapsets,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=apps.kruise.io,resources=configmapsets/status,verbs=get;update;patch + +// Reconcile reads that state of the cluster for a ConfigMapSet object and makes changes based on the state read +// and what is in the ConfigMapSet.Spec +func (r *ReconcileConfigMapSet) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + // Fetch the ConfigMapSet instance + cms := &appsv1alpha1.ConfigMapSet{} + err := r.Get(ctx, request.NamespacedName, cms) + if err != nil { + if errors.IsNotFound(err) { + return reconcile.Result{}, nil + } + return reconcile.Result{}, err + } + + // handle delete + if cms.DeletionTimestamp != nil { + if containsString(cms.Finalizers, ConfigMapFinalizerName) { + // 1. check has injected pod exist by webhook + // 2. clean configmap-hub + if err := r.cleanupConfigMap(ctx, cms); err != nil { + return reconcile.Result{}, fmt.Errorf("cleanupConfigMap failed for cms %s/%s: %w", cms.Namespace, cms.Name, err) + } + // 3. remove finalizer + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + latest := &appsv1alpha1.ConfigMapSet{} + if err = r.Get(ctx, types.NamespacedName{Name: cms.Name, Namespace: cms.Namespace}, latest); err != nil { + return err + } + latest.Finalizers = removeString(latest.Finalizers, ConfigMapFinalizerName) + return r.Update(ctx, latest) + }) + if err != nil { + klog.Errorf("RetryOnConflict failed to remove finalizer for %s/%s: %v", cms.Namespace, cms.Name, err) + return reconcile.Result{}, fmt.Errorf("failed to remove finalizer for %s/%s: %w", cms.Namespace, cms.Name, err) + } + } + klog.Infof("ConfigMapSet %s is being deleted", cms.Name) + return reconcile.Result{}, nil + } + + // Add Finalizer (if not present) + if !containsString(cms.Finalizers, ConfigMapFinalizerName) { + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + latest := &appsv1alpha1.ConfigMapSet{} + if err := r.Get(ctx, types.NamespacedName{Name: cms.Name, Namespace: cms.Namespace}, latest); err != nil { + return err + } + if !containsString(latest.Finalizers, ConfigMapFinalizerName) { + latest.Finalizers = append(latest.Finalizers, ConfigMapFinalizerName) + return r.Update(ctx, latest) + } + return nil + }) + if err != nil { + klog.Errorf("RetryOnConflict failed to add finalizer for %s/%s: %v", cms.Namespace, cms.Name, err) + return reconcile.Result{}, fmt.Errorf("failed to add finalizer for %s/%s: %w", cms.Namespace, cms.Name, err) + } + } + + // manage configmap-hub + err = r.syncRevisions(ctx, cms) + if err != nil { + return reconcile.Result{}, fmt.Errorf("manageRevisions failed for cms %s/%s: %w", cms.Namespace, cms.Name, err) + } + + pods, err := GetMatchedPods(ctx, r.Client, cms) + if err != nil { + return reconcile.Result{}, fmt.Errorf("GetMatchedPods failed for cms %s/%s: %w", cms.Namespace, cms.Name, err) + } + + // Sync Pods + requeue, err := r.SyncPods(ctx, cms, pods) + if err != nil { + return reconcile.Result{}, fmt.Errorf("syncPods failed for cms %s/%s: %w", cms.Namespace, cms.Name, err) + } + + if requeue { + klog.Infof("ConfigMapSet %s/%s needs requeue to wait for pod updates", cms.Namespace, cms.Name) + return reconcile.Result{RequeueAfter: 3 * time.Second}, nil + } + + // Update status + err = r.updateStatus(ctx, request, cms) + if err != nil { + return reconcile.Result{}, fmt.Errorf("updateStatus failed for cms %s/%s: %w", cms.Namespace, cms.Name, err) + } + + return reconcile.Result{}, nil +} + +func (r *ReconcileConfigMapSet) cleanupConfigMap(ctx context.Context, cms *appsv1alpha1.ConfigMapSet) error { + cmName := GetConfigMapSetHubName(cms.Name) + cm := &corev1.ConfigMap{} + err := r.Get(ctx, types.NamespacedName{Name: cmName, Namespace: cms.Namespace}, cm) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + return err + } + return r.Delete(ctx, cm) +} + +// CalculateHash computes the hash of an object +func CalculateHash(v interface{}) (string, error) { + if v == nil { + return "", fmt.Errorf("object is nil") + } + // 1. Marshal spec to JSON + specBytes, err := json.Marshal(v) + if err != nil { + return "", fmt.Errorf("failed to marshal spec: %v", err) + } + + // 2. Compute SHA256 hash + hash := sha256.Sum256(specBytes) + + // 3. Return hexadecimal encoded hash string (first 8 bytes for short hash) + return hex.EncodeToString(hash[:8]), nil +} + +// syncRevisions manages version history +func (r *ReconcileConfigMapSet) syncRevisions(ctx context.Context, cms *appsv1alpha1.ConfigMapSet) error { + hash, err := CalculateHash(cms.Spec.Data) + if err != nil { + return fmt.Errorf("failed to compute hash: %v", err) + } + + // ConfigMap name: cms.Name + "-hub" + cmName := GetConfigMapSetHubName(cms.Name) + cmNamespace := cms.Namespace + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + cm := &corev1.ConfigMap{} + if err = r.Get(ctx, types.NamespacedName{Name: cmName, Namespace: cmNamespace}, cm); err != nil { + if errors.IsNotFound(err) { + newCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: cmNamespace, + }, + Data: make(map[string]string), + } + if err = controllerutil.SetControllerReference(cms, newCM, r.scheme); err != nil { + return fmt.Errorf("failed to set owner reference on ConfigMap %s/%s: %v", cmNamespace, cmName, err) + } + if err = r.Create(ctx, newCM); err != nil { + return fmt.Errorf("failed to create ConfigMap %s/%s: %v", cmNamespace, cmName, err) + } + return fmt.Errorf("create ConfigMap %s/%s: %v", cmNamespace, cmName, err) + } + return fmt.Errorf("failed to get ConfigMap: %v", err) + } + + var revisions []RevisionEntry + if revData, exists := cm.Data["revisions"]; exists { + if err := json.Unmarshal([]byte(revData), &revisions); err != nil { + klog.Errorf("Failed to unmarshal revisions from ConfigMap %s: %v, resetting revisions", cmName, err) + return fmt.Errorf("failed to unmarshal revisions from ConfigMap %s: %v", cmName, err) + } + } + + for _, rev := range revisions { + if rev.Hash == hash && rev.CustomVersion == cms.Spec.CustomVersion { + klog.Warningf("Revision %s already exists in ConfigMap %s, skipping update", hash, cmName) + return nil + } + } + revisions = append(revisions, RevisionEntry{ + Hash: hash, + TimeStamp: &metav1.Time{Time: time.Now()}, + Data: cms.Spec.Data, + CustomVersion: cms.Spec.CustomVersion, + }) + + klog.Infof("Updated revisions for ConfigMapSet %s/%s: %v", cms.Namespace, cms.Name, revisions) + revBytes, err := json.Marshal(revisions) + if err != nil { + return fmt.Errorf("failed to marshal revisions: %v", err) + } + cm = cm.DeepCopy() // Avoid modifying the cached object + if cm.Data == nil { + cm.Data = make(map[string]string) + } + cm.Data["revisions"] = string(revBytes) + + // Update ConfigMap + return r.Update(ctx, cm) + }) +} + +func (r *ReconcileConfigMapSet) getHubRevisions(ctx context.Context, cms *appsv1alpha1.ConfigMapSet) ([]RevisionEntry, error) { + cmName := GetConfigMapSetHubName(cms.Name) + cm := &corev1.ConfigMap{} + if err := r.Get(ctx, types.NamespacedName{Name: cmName, Namespace: cms.Namespace}, cm); err != nil { + return nil, err + } + var revisions []RevisionEntry + if revData, exists := cm.Data["revisions"]; exists { + if err := json.Unmarshal([]byte(revData), &revisions); err != nil { + return nil, err + } + } + return revisions, nil +} + +func (r *ReconcileConfigMapSet) SyncPods(ctx context.Context, cms *appsv1alpha1.ConfigMapSet, pods []*corev1.Pod) (bool, error) { + klog.Infof("Syncing pods for ConfigMapSet %s/%s", cms.Namespace, cms.Name) + revisionKeys := &RevisionKeys{ + CurrentRevisionKey: GetConfigMapSetCurrentRevisionKey(cms.Name), + CurrentCustomVersionKey: GetConfigMapSetCurrentCustomVersionKey(cms.Name), + CurrentRevisionTimeStampKey: GetConfigMapSetCurrentRevisionTimeStampKey(cms.Name), + UpdateRevisionKey: GetConfigMapSetUpdateRevisionKey(cms.Name), + UpdateCustomVersionKey: GetConfigMapSetUpdateCustomVersionKey(cms.Name), + UpdateRevisionTimeStampKey: GetConfigMapSetUpdateRevisionTimeStampKey(cms.Name), + } + // Load the version list from the hub ConfigMap for Pod selection priority sorting + hubRevisions, err := r.getHubRevisions(ctx, cms) + if err != nil { + klog.Errorf("Failed to load hub revisions for ConfigMapSet %s/%s: %v, pod selection priority rules 2&3 will be degraded", cms.Namespace, cms.Name, err) + return false, err + } + + // Select Pods to update based on the update strategy + updateStrategy := cms.Spec.UpdateStrategy + + requeue := false + + podGroups := GroupPodsByMatchLabelKeys(pods, updateStrategy.MatchLabelKeys) + + var allErrs []error + for _, group := range podGroups { + distributions, err := getDistributionByPartition(cms, group) + if err != nil { + klog.Errorf("failed to transform partition to distribution: %v", err) + allErrs = append(allErrs, fmt.Errorf("failed to transform partition to distribution: %v", err)) + continue + } + groupRequeue, err := r.UpdateByDistribution(ctx, cms, distributions, group, revisionKeys, hubRevisions) + if groupRequeue { + requeue = true + } + if err != nil { + allErrs = append(allErrs, err) + } + } + + if len(allErrs) > 0 { + return requeue, fmt.Errorf("errors occurred during syncPods: %v", allErrs) + } + return requeue, nil +} + +// Distribution is used internally in controller now +type Distribution struct { + Revision string `json:"revision,omitempty"` + CustomVersion string `json:"customVersion,omitempty"` + Reserved int32 `json:"reserved"` +} + +func getDistributionByPartition(cms *appsv1alpha1.ConfigMapSet, pods []*corev1.Pod) ([]Distribution, error) { + // Total replicas = number of passed pods + replicas := len(pods) + // Parse partition + partition := cms.Spec.UpdateStrategy.Partition + + // Calculate the latest updateRevision here since syncPods is called before updateStatus + updateRevision, err := CalculateHash(cms.Spec.Data) // Calculate current version + if err != nil { + return nil, fmt.Errorf("failed to compute hash for cms %s/%s: %w", cms.Namespace, cms.Name, err) + } + updateCustomVersion := cms.Spec.CustomVersion + currentRevision := cms.Status.CurrentRevision + currentCustomVersion := cms.Status.CurrentCustomVersion + + newReplicas := replicas // New replicas count + if partition != nil { + partitionStr := partition.String() + if strings.HasSuffix(partitionStr, "%") { + // Handle percentage partition + percentStr := strings.TrimSuffix(partitionStr, "%") + percent, err := strconv.ParseInt(percentStr, 10, 32) + if err != nil || percent < 0 || percent > 100 { + return nil, fmt.Errorf("invalid partition percentage: %s", partitionStr) + } + newReplicas = replicas - int(math.Ceil(float64(percent)/100.0*float64(replicas))) + } else if count, err := strconv.ParseInt(partitionStr, 10, 32); err == nil && count >= 0 { + // Handle integer partition + newReplicas = replicas - int(count) + if newReplicas < 0 { // Cannot be negative + newReplicas = 0 + } + } else { + return nil, fmt.Errorf("invalid partition number: %s", partitionStr) + } + } + // Construct distribution + distributions := make([]Distribution, 2) + distributions[0] = Distribution{ + Revision: updateRevision, + Reserved: int32(newReplicas), + CustomVersion: updateCustomVersion, + } + distributions[1] = Distribution{ + Revision: currentRevision, + Reserved: int32(replicas - newReplicas), + CustomVersion: currentCustomVersion, + } + + klog.Infof("distributions after transform: %v", distributions) + return distributions, nil +} + +func getUpdateInfoByDistribution(cms *appsv1alpha1.ConfigMapSet, distributions []Distribution, pods []*corev1.Pod, revisionKeys *RevisionKeys, hubRevisions []RevisionEntry) (*UpdateInfo, error) { + // Collect pods to update (excess version -> target version) + var podsToUpdate []*corev1.Pod + targetRevisions := make([]string, 0) + targetCustomVersions := make([]string, 0) + + podsToUpdate, targetRevisions, targetCustomVersions = getUpdatePodsByDistributions(cms, distributions, pods, revisionKeys, hubRevisions) + return &UpdateInfo{ + PodsToUpdate: podsToUpdate, + TargetRevisions: targetRevisions, + TargetCustomVersions: targetCustomVersions, + }, nil +} + +func getUpdatePodsByDistributions(cms *appsv1alpha1.ConfigMapSet, distributions []Distribution, pods []*v1.Pod, revisionKeys *RevisionKeys, hubRevisions []RevisionEntry) ([]*v1.Pod, []string, []string) { + updateDistribution := distributions[0] + currentDistribution := distributions[1] + + var validPods []*v1.Pod + var podsToUpdate []*v1.Pod + var targetRevisions []string + var targetCustomVersions []string + + // Filter pods that don't have the enabled annotation and count unavailable pods + var currentUnavailableCount int32 = 0 + for _, pod := range pods { + if pod.Annotations != nil && pod.Annotations[GetConfigMapSetEnabledKey()] == "true" { + validPods = append(validPods, pod) + if !IsPodReady(pod) { + currentUnavailableCount++ + } + } + } + + // Calculate maxUnavailable quota + var maxUnavailableQuota int32 = math.MaxInt32 // Default to unlimited if not set + if cms.Spec.UpdateStrategy.MaxUnavailable != nil { + totalValidPods := len(validPods) + maxUnavailableStr := cms.Spec.UpdateStrategy.MaxUnavailable.String() + if strings.HasSuffix(maxUnavailableStr, "%") { + percentStr := strings.TrimSuffix(maxUnavailableStr, "%") + if percent, err := strconv.ParseInt(percentStr, 10, 32); err == nil && percent >= 0 && percent <= 100 { + maxUnavailableQuota = int32(math.Ceil(float64(percent) / 100.0 * float64(totalValidPods))) + } + } else if count, err := strconv.ParseInt(maxUnavailableStr, 10, 32); err == nil && count >= 0 { + maxUnavailableQuota = int32(count) + } + } + + // The actual quota allowed for this reconcile round + allowedUpdateQuota := maxUnavailableQuota - currentUnavailableCount + if allowedUpdateQuota <= 0 && maxUnavailableQuota != math.MaxInt32 { + klog.Infof("ConfigMapSet %s/%s maxUnavailable limit reached (current unavailable: %d, max allowed: %d), no more pods will be updated in this round", cms.Namespace, cms.Name, currentUnavailableCount, maxUnavailableQuota) + return podsToUpdate, targetRevisions, targetCustomVersions + } + + // Calculate how many pods are currently at the update revision + var currentUpdateRevisionCount int32 = 0 + for _, pod := range pods { + isUpdated := false + if pod.Annotations == nil { + isUpdated = true + } else { + hasCurrentRev := pod.Annotations[revisionKeys.CurrentRevisionKey] != "" + hasUpdateRev := pod.Annotations[revisionKeys.UpdateRevisionKey] != "" + // If it does not have currentVersion and UpdateVersion annotations, we treat it as updated + if !hasCurrentRev && !hasUpdateRev { + isUpdated = true + } else if pod.Annotations[revisionKeys.CurrentRevisionKey] == updateDistribution.Revision { + isUpdated = true + } + } + if isUpdated { + currentUpdateRevisionCount++ + } + } + + // Calculate how many pods we need to update + needToUpdate := updateDistribution.Reserved - currentUpdateRevisionCount + if needToUpdate <= 0 { + return podsToUpdate, targetRevisions, targetCustomVersions + } + + // Build hub revision index: hash -> position in hub (smaller is older) + hubRevisionIndex := make(map[string]int, len(hubRevisions)) + for idx, rev := range hubRevisions { + hubRevisionIndex[rev.Hash] = idx + } + + // Sort valid pods based on the priority rules + sort.Slice(validPods, func(i, j int) bool { + podI := validPods[i] + podJ := validPods[j] + + revI := "" + revJ := "" + if podI.Annotations != nil { + revI = podI.Annotations[revisionKeys.CurrentRevisionKey] + } + if podJ.Annotations != nil { + revJ = podJ.Annotations[revisionKeys.CurrentRevisionKey] + } + + // 1. Non-currentRevision > currentRevision + isICurrent := revI == currentDistribution.Revision + isJCurrent := revJ == currentDistribution.Revision + if !isICurrent && isJCurrent { + return true + } + if isICurrent && !isJCurrent { + return false + } + + // For non-currentRevision pods, apply rules 2 and 3 + if !isICurrent && !isJCurrent { + idxI, inRMCI := hubRevisionIndex[revI] + idxJ, inRMCJ := hubRevisionIndex[revJ] + + // 3. Not in RMC > In RMC + if !inRMCI && inRMCJ { + return true + } + if inRMCI && !inRMCJ { + return false + } + + // 2. Smaller index (older) > Larger index (newer) + if inRMCI && inRMCJ && idxI != idxJ { + return idxI < idxJ + } + } + + if podI.CreationTimestamp.Equal(&podJ.CreationTimestamp) { + return podI.Name < podJ.Name + } + + return podI.CreationTimestamp.Before(&podJ.CreationTimestamp) + }) + + // Select pods to update + for _, pod := range validPods { + if needToUpdate <= 0 { + break + } + + rev := "" + if pod.Annotations != nil { + rev = pod.Annotations[revisionKeys.CurrentRevisionKey] + } + + // Skip pods already at the update revision + if rev == updateDistribution.Revision { + continue + } + + podsToUpdate = append(podsToUpdate, pod) + targetRevisions = append(targetRevisions, updateDistribution.Revision) + targetCustomVersions = append(targetCustomVersions, updateDistribution.CustomVersion) + needToUpdate-- + } + + return podsToUpdate, targetRevisions, targetCustomVersions +} + +func (r *ReconcileConfigMapSet) getReloadSidecarName(ctx context.Context, cms *appsv1alpha1.ConfigMapSet) string { + expectedSidecarName := GetConfigMapSetDefaultSidecarName(cms.Name) + if cms.Spec.ReloadSidecarConfig != nil { + if cms.Spec.ReloadSidecarConfig.Type == appsv1alpha1.ReloadSidecarTypeK8s && cms.Spec.ReloadSidecarConfig.Config != nil && cms.Spec.ReloadSidecarConfig.Config.Name != "" { + expectedSidecarName = cms.Spec.ReloadSidecarConfig.Config.Name + } else if cms.Spec.ReloadSidecarConfig.Type == appsv1alpha1.ReloadSidecarTypeSidecarSet && cms.Spec.ReloadSidecarConfig.Config != nil && cms.Spec.ReloadSidecarConfig.Config.SidecarSetRef != nil { + expectedSidecarName = cms.Spec.ReloadSidecarConfig.Config.SidecarSetRef.ContainerName + } else if cms.Spec.ReloadSidecarConfig.Type == appsv1alpha1.ReloadSidecarTypeCustom { + if cms.Spec.ReloadSidecarConfig.Config != nil && cms.Spec.ReloadSidecarConfig.Config.ConfigMapRef != nil { + cmRef := cms.Spec.ReloadSidecarConfig.Config.ConfigMapRef + customerCM := &corev1.ConfigMap{} + if err := r.Get(ctx, types.NamespacedName{Name: cmRef.Name, Namespace: cmRef.Namespace}, customerCM); err == nil { + for _, v := range customerCM.Data { + var reloadSidecar corev1.Container + if unmarshalErr := json.Unmarshal([]byte(v), &reloadSidecar); unmarshalErr == nil { + if reloadSidecar.Name != "" { + expectedSidecarName = reloadSidecar.Name + } + break + } + } + } else { + klog.Errorf("failed to get customer sidecar configmap %s/%s: %v", cmRef.Namespace, cmRef.Name, err) + } + } + } + } + return expectedSidecarName +} + +// UpdateByDistribution updates pods based on the distribution +func (r *ReconcileConfigMapSet) UpdateByDistribution(ctx context.Context, cms *appsv1alpha1.ConfigMapSet, distributions []Distribution, pods []*corev1.Pod, revisionKeys *RevisionKeys, hubRevisions []RevisionEntry) (bool, error) { + // Step 1: Calculate the current version distribution and the update version queue + updateInfo, err := getUpdateInfoByDistribution(cms, distributions, pods, revisionKeys, hubRevisions) + if err != nil { + klog.Errorf("failed to fetch update info: %v", err) + return false, err + } + + // Step 2: Trigger updates + requeue := false + for i, pod := range updateInfo.PodsToUpdate { + if i >= len(updateInfo.TargetRevisions) || i >= len(updateInfo.TargetCustomVersions) { + break + } + if pod == nil || !controller.IsPodActive(pod) { + continue + } + targetRevision := updateInfo.TargetRevisions[i] + targetCustomVersion := updateInfo.TargetCustomVersions[i] + + klog.Infof("Updating Pod %s/%s to revision %s", pod.Namespace, pod.Name, targetRevision) + + // 1. Update Pod Annotation (TargetRevision and TargetCustomVersion) + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + latestPod := &corev1.Pod{} + if err := r.Get(ctx, types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, latestPod); err != nil { + return err + } + + if latestPod.Annotations == nil { + latestPod.Annotations = make(map[string]string) + } + + // If already updated CurrentRevision, skip + if latestPod.Annotations[revisionKeys.CurrentRevisionKey] == targetRevision && + latestPod.Annotations[revisionKeys.CurrentCustomVersionKey] == targetCustomVersion { + return nil + } + + // If TargetRevision is not yet updated, update it + if latestPod.Annotations[revisionKeys.UpdateRevisionKey] != targetRevision || + latestPod.Annotations[revisionKeys.UpdateCustomVersionKey] != targetCustomVersion { + + latestPod.Annotations[revisionKeys.UpdateRevisionKey] = targetRevision + latestPod.Annotations[revisionKeys.UpdateCustomVersionKey] = targetCustomVersion + + klog.Infof("Updating TargetRevision for Pod %s/%s to %s to trigger reload-sidecar update", latestPod.Namespace, latestPod.Name, targetRevision) + err = r.Update(ctx, latestPod) + if err == nil { + requeue = true + } + return err + } + + // 2. Execute EffectPolicy logic + if cms.Spec.EffectPolicy != nil { + switch cms.Spec.EffectPolicy.Type { + case appsv1alpha1.EffectPolicyTypeReStart: + if latestPod.Annotations == nil { + latestPod.Annotations = make(map[string]string) + } + + reloadSidecarName := r.getReloadSidecarName(ctx, cms) + needRestart := false + reloadSidecarRestartKey := GetConfigMapSetReloadSidecarRestartKey(cms.Name) + expectHash := GetContainerHash(latestPod, targetRevision) + if latestPod.Annotations[reloadSidecarRestartKey] != expectHash { + latestPod.Annotations[reloadSidecarRestartKey] = expectHash + needRestart = true + } + + if needRestart { + klog.Infof("Triggering reload-sidecar container inplace-update for Pod %s/%s to revision %s", latestPod.Namespace, latestPod.Name, targetRevision) + for _, status := range latestPod.Status.ContainerStatuses { + if status.Name == reloadSidecarName && status.State.Running != nil { + latestPod.Annotations[GetConfigMapSetContainerStartedAtKey(reloadSidecarName)] = status.State.Running.StartedAt.Format(time.RFC3339) + } + } + err = r.rebootSidecarByCrr(latestPod, reloadSidecarName, expectHash) + if err != nil { + return err + } + err = r.Update(ctx, latestPod) + if err == nil { + requeue = true + } + return err + } + + // wait reload-sidecar reboot success + err = r.waitSidecarRebootByCrrSuccess(ctx, latestPod, reloadSidecarName, expectHash) + if err != nil { + klog.Errorf("Pod %s/%s reload-sidecar (%s) is not rebooted yet because of %s, waiting...", latestPod.Namespace, latestPod.Name, reloadSidecarName, err.Error()) + return err + } + + time.Sleep(time.Second * 3) + + // wait reload-sidecar is ready + isReloadSidecarReady := false + for _, cs := range latestPod.Status.ContainerStatuses { + if cs.Name == reloadSidecarName { + if cs.Ready { + isReloadSidecarReady = true + } + break + } + } + if !isReloadSidecarReady { + klog.Infof("Pod %s/%s reload-sidecar (%s) is not ready yet, waiting...", latestPod.Namespace, latestPod.Name, reloadSidecarName) + requeue = true + return nil + } + + allContainerNames := []string{} + rebootContainerNames := []string{} + for _, c := range cms.Spec.Containers { + cName := GetContainerName(latestPod, c) + if cName == "" { + klog.Warningf("Pod %s/%s cannot determine container name from spec %v", latestPod.Namespace, latestPod.Name, c) + continue + } + allContainerNames = append(allContainerNames, cName) + annotationKey := GetConfigMapSetContainerRestartKey(cms.Name, cName) + if latestPod.Annotations[annotationKey] != expectHash { + latestPod.Annotations[annotationKey] = expectHash + rebootContainerNames = append(rebootContainerNames, cName) + needRestart = true + } + } + + if needRestart { + klog.Infof("Triggering business container inplace-update for Pod %s/%s to revision %s", latestPod.Namespace, latestPod.Name, targetRevision) + for _, name := range rebootContainerNames { + for _, status := range latestPod.Status.ContainerStatuses { + if status.Name == name && status.State.Running != nil { + latestPod.Annotations[GetConfigMapSetContainerStartedAtKey(name)] = status.State.Running.StartedAt.Format(time.RFC3339) + } + } + err = r.rebootSidecarByCrr(latestPod, name, expectHash) + if err != nil { + return err + } + } + + err = r.Update(ctx, latestPod) + if err == nil { + requeue = true + } + return err + } + + // wait business-sidecar reboot success + for _, containerName := range allContainerNames { + err = r.waitSidecarRebootByCrrSuccess(ctx, latestPod, containerName, expectHash) + if err != nil { + klog.Infof("Pod %s/%s business-sidecars (%v) is not rebooted yet, waiting...", latestPod.Namespace, latestPod.Name, rebootContainerNames) + requeue = true + return err + } + } + case appsv1alpha1.EffectPolicyTypeHotUpdate: + // wait reload-sidecar is ready + isReloadSidecarReady := false + expectedSidecarName := r.getReloadSidecarName(ctx, cms) + for _, cs := range latestPod.Status.ContainerStatuses { + if cs.Name == expectedSidecarName { + if cs.Ready { + isReloadSidecarReady = true + } + break + } + } + if !isReloadSidecarReady { + klog.Infof("Pod %s/%s reload-sidecar (%s) is not ready yet, waiting...", latestPod.Namespace, latestPod.Name, expectedSidecarName) + requeue = true + return nil + } + case appsv1alpha1.EffectPolicyTypePostHook: + if latestPod.Labels == nil { + latestPod.Labels = make(map[string]string) + } + + podNameForHash := latestPod.Name + if podNameForHash == "" { + podNameForHash = "unnamed" + } + hashBytes := md5.Sum([]byte(podNameForHash + targetRevision)) + hashStr := hex.EncodeToString(hashBytes[:]) + + reloadSidecarName := r.getReloadSidecarName(ctx, cms) + needRestart := false + restartKey := GetConfigMapSetReloadSidecarRestartKey(cms.Name) + if latestPod.Annotations[restartKey] != hashStr { + latestPod.Annotations[restartKey] = hashStr + needRestart = true + } + + if needRestart { + klog.Infof("Triggering reload-sidecar container inplace-update for Pod %s/%s to revision %s", latestPod.Namespace, latestPod.Name, targetRevision) + for _, status := range latestPod.Status.ContainerStatuses { + if status.Name == reloadSidecarName && status.State.Running != nil { + latestPod.Annotations[GetConfigMapSetContainerStartedAtKey(reloadSidecarName)] = status.State.Running.StartedAt.Format(time.RFC3339) + } + } + err = r.rebootSidecarByCrr(latestPod, reloadSidecarName, hashStr) + if err != nil { + return err + } + err = r.Update(ctx, latestPod) + if err == nil { + requeue = true + } + return err + } + + // wait business-sidecar reboot success + err = r.waitSidecarRebootByCrrSuccess(ctx, latestPod, reloadSidecarName, hashStr) + if err != nil { + klog.Infof("Pod %s/%s reload-sidecar (%s) is not rebooted yet, waiting...", latestPod.Namespace, latestPod.Name, reloadSidecarName) + requeue = true + return nil + } + + // wait reload-sidecar is ready + isReloadSidecarReady := false + for _, cs := range latestPod.Status.ContainerStatuses { + if cs.Name == reloadSidecarName { + if cs.Ready { + isReloadSidecarReady = true + } + break + } + } + if !isReloadSidecarReady { + klog.Infof("Pod %s/%s reload-sidecar (%s) is not ready yet, waiting...", latestPod.Namespace, latestPod.Name, reloadSidecarName) + requeue = true + return nil + } + } + } + + // Update CurrentRevisionKey and CurrentCustomVersionKey + latestPod.Annotations[revisionKeys.CurrentRevisionKey] = targetRevision + latestPod.Annotations[revisionKeys.CurrentCustomVersionKey] = targetCustomVersion + + klog.Infof("Successfully executed EffectPolicy and updated CurrentRevision for Pod %s/%s to %s", latestPod.Namespace, latestPod.Name, targetRevision) + return r.Update(ctx, latestPod) + }) + + if err != nil { + klog.Errorf("Failed to update pod annotations for %s/%s: %v", pod.Namespace, pod.Name, err) + return requeue, err + } + } + + return requeue, nil +} + +// updateStatus updates the status field of cms +func (r *ReconcileConfigMapSet) updateStatus(ctx context.Context, request reconcile.Request, cms *appsv1alpha1.ConfigMapSet) error { + klog.Infof("Updating status for ConfigMapSet %s/%s", cms.Namespace, cms.Name) + // Get matched Pod list + pods, err := GetMatchedPods(ctx, r.Client, cms) + if err != nil { + return err + } + var updateRevision string + var updatedPodsNum, readyPodsNum, updatedReadyPodsNum int32 + // apps.kruise.io/configmapset.configMapSetName.revision + targetRevisionKey := GetConfigMapSetUpdateRevisionKey(cms.Name) + currentRevisionKey := GetConfigMapSetCurrentRevisionKey(cms.Name) + + updateRevision, err = CalculateHash(cms.Spec.Data) // Calculate current version + if err != nil { + return fmt.Errorf("failed to compute hash for cms %s/%s: %w", cms.Namespace, cms.Name, err) + } + + // Calculate status metrics + for _, pod := range pods { + isUpdated := false + if pod.Annotations == nil { + isUpdated = true + } else { + hasCurrentRev := pod.Annotations[currentRevisionKey] != "" + hasUpdateRev := pod.Annotations[targetRevisionKey] != "" + if !hasCurrentRev && !hasUpdateRev { + isUpdated = true + } else if pod.Annotations[targetRevisionKey] == updateRevision && pod.Annotations[currentRevisionKey] == updateRevision { + isUpdated = true + } + } + + if isUpdated { + updatedPodsNum++ + if IsPodReady(pod) { + klog.Infof("Pod %s/%s is ready", pod.Namespace, pod.Name) + readyPodsNum++ + updatedReadyPodsNum++ + } + } else if IsPodReady(pod) { + klog.Infof("Pod %s/%s is ready", pod.Namespace, pod.Name) + readyPodsNum++ + } + } + + klog.Infof("cms %s matched pods: %d, ready pods: %d, updated pods: %d, updated ready pods: %d", cms.Name, len(pods), readyPodsNum, updatedPodsNum, updatedReadyPodsNum) + + if cms.Status.ObservedGeneration == cms.Generation && + cms.Status.UpdateRevision == updateRevision && + cms.Status.ReadyReplicas == readyPodsNum && + cms.Status.Replicas == int32(len(pods)) && + cms.Status.UpdatedReplicas == updatedPodsNum && + cms.Status.UpdatedReadyReplicas == updatedReadyPodsNum { + klog.Infof("No change in status for ConfigMapSet %s/%s, skipping update", cms.Namespace, cms.Name) + return nil + } + + // Update ConfigMapSet status + // currentRevision can only be updated after all pods are at the latest version + // If it is the first release of cms, use the current version + if updatedReadyPodsNum == int32(len(pods)) || cms.Status.CurrentRevision == "" { + cms.Status.CurrentRevision = updateRevision + cms.Status.CurrentCustomVersion = cms.Spec.CustomVersion + // Manage version history only after all pods are updated to a version + err = r.cleanHistoryRevision(ctx, cms) + } + cms.Status.ObservedGeneration = cms.Generation + cms.Status.UpdateRevision = updateRevision + cms.Status.UpdateCustomVersion = cms.Spec.CustomVersion + cms.Status.Replicas = int32(len(pods)) + cms.Status.ReadyReplicas = readyPodsNum + cms.Status.UpdatedReplicas = updatedPodsNum + cms.Status.UpdatedReadyReplicas = updatedReadyPodsNum + // Calculate ExpectedUpdatedReplicas based on update strategy + expected := int32(len(pods)) + if cms.Spec.UpdateStrategy.Partition != nil { + partitionStr := cms.Spec.UpdateStrategy.Partition.String() + if strings.HasSuffix(partitionStr, "%") { + percentStr := strings.TrimSuffix(partitionStr, "%") + if percent, err := strconv.ParseInt(percentStr, 10, 32); err == nil && percent >= 0 && percent <= 100 { + expected = int32(len(pods)) - int32(math.Ceil(float64(percent)/100.0*float64(len(pods)))) + } + } else if count, err := strconv.ParseInt(partitionStr, 10, 32); err == nil && count >= 0 { + expected = int32(len(pods)) - int32(count) + } + if expected < 0 { + expected = 0 + } + } + cms.Status.ExpectedUpdatedReplicas = expected + + // Sync status to etcd + klog.Infof("Updating cms %s status %#v", cms.Name, cms.Status) + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + // Fetch the latest object + latest := &appsv1alpha1.ConfigMapSet{} + if err := r.Get(context.TODO(), request.NamespacedName, latest); err != nil { + utilruntime.HandleError(fmt.Errorf("error getting latest configmapset %s/%s: %v", cms.Namespace, cms.Name, err)) + return err + } + + // Update Status fields + latest.Status = cms.Status + + klog.Info("Before update, RV:", latest.ResourceVersion) + + // Execute Status update + if err := r.Status().Update(context.TODO(), latest); err != nil { + utilruntime.HandleError(fmt.Errorf("error updating configmapset status %s/%s: %v", cms.Namespace, cms.Name, err)) + return err + } + + // Log the updated ResourceVersion + klog.Infof("Updated cms %s status successfully, new ResourceVersion: %s", latest.Name, latest.ResourceVersion) + return nil + }) +} + +func (r *ReconcileConfigMapSet) cleanHistoryRevision(ctx context.Context, cms *appsv1alpha1.ConfigMapSet) error { + cmName := GetConfigMapSetHubName(cms.Name) + cmNamespace := cms.Namespace + + var revisions []RevisionEntry + + // Define update logic + updateFunc := func() error { + // Fetch the latest ConfigMap again to avoid concurrent conflicts + cm := &corev1.ConfigMap{} + if err := r.Get(ctx, types.NamespacedName{Name: cmName, Namespace: cmNamespace}, cm); err != nil { + if errors.IsNotFound(err) { + // Create if ConfigMap does not exist + newCM := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmName, + Namespace: cmNamespace, + }, + Data: make(map[string]string), + } + return r.Create(ctx, newCM) + } + return fmt.Errorf("failed to get ConfigMap: %v", err) + } + + // Parse existing revisions from ConfigMap + if revData, exists := cm.Data["revisions"]; exists { + if err := json.Unmarshal([]byte(revData), &revisions); err != nil { + klog.Errorf("Failed to unmarshal revisions from ConfigMap %s: %v, resetting revisions", cmName, err) + return fmt.Errorf("failed to unmarshal revisions from ConfigMap %s: %v", cmName, err) + } + } + + // If the number of revisions reaches the limit, remove the oldest one that is not used by any Pod + if cms.Spec.RevisionHistoryLimit != nil && int32(len(revisions)) >= *cms.Spec.RevisionHistoryLimit { + pods, podErr := GetMatchedPods(ctx, r.Client, cms) + if podErr != nil { + return podErr + } + revisionsInUse := make(map[string]bool) + currentRevisionKey := GetConfigMapSetCurrentRevisionKey(cms.Name) + for _, pod := range pods { + if pod.Annotations != nil && pod.Annotations[currentRevisionKey] != "" { + revisionsInUse[pod.Annotations[currentRevisionKey]] = true + } + } + // Find the oldest revision that is not in use + oldestUnusedIdx := -1 + for i, rev := range revisions { + if !revisionsInUse[rev.Hash] { + oldestUnusedIdx = i + break + } + } + + // If all revisions are in use, do not remove any + if oldestUnusedIdx == -1 { + klog.Warningf("All revisions are currently in use, skipping cleanup for ConfigMapSet %s/%s", cms.Namespace, cms.Name) + return nil + } + + // Remove the oldest unused revision + revToRemove := revisions[oldestUnusedIdx] + klog.Infof("Removing oldest unused revision %s (customVersion=%s) from hub to maintain limit %d", revToRemove.Hash, revToRemove.CustomVersion, *cms.Spec.RevisionHistoryLimit) + revisions = append(revisions[:oldestUnusedIdx], revisions[oldestUnusedIdx+1:]...) + } + + klog.Infof("Updated revisions for ConfigMapSet %s/%s: %v", cms.Namespace, cms.Name, revisions) + + // Update ConfigMap Data + revBytes, err := json.Marshal(revisions) + if err != nil { + return fmt.Errorf("failed to marshal revisions: %v", err) + } + cm = cm.DeepCopy() // Avoid modifying the cached object + if cm.Data == nil { + cm.Data = make(map[string]string) + } + cm.Data["revisions"] = string(revBytes) + + // Update ConfigMap + return r.Update(ctx, cm) + } + + // Use `retry.RetryOnConflict` to handle update conflicts + err := retry.RetryOnConflict(retry.DefaultRetry, updateFunc) + if err != nil { + return fmt.Errorf("failed to update ConfigMap %s: %v", cmName, err) + } + return nil +} + +func GetContainerName(pod *corev1.Pod, containerSpec appsv1alpha1.ConfigMapSetContainer) string { + if containerSpec.Name != "" { + return containerSpec.Name + } + if containerSpec.NameFrom != nil && containerSpec.NameFrom.FieldRef.FieldPath != "" { + path, subscript, ok := fieldpath.SplitMaybeSubscriptedPath(containerSpec.NameFrom.FieldRef.FieldPath) + if ok { + switch path { + case "metadata.annotations": + if pod.Annotations != nil { + return pod.Annotations[subscript] + } + case "metadata.labels": + if pod.Labels != nil { + return pod.Labels[subscript] + } + } + } + } + return "" +} + +func (r *ReconcileConfigMapSet) genContainerRecreateRequestName(pod *corev1.Pod, containerName string, hashStr string) string { + // Combine hashStr and sortedContainerNames to form a unique hash for this specific set of containers and targetRevision + sum := md5.Sum([]byte(fmt.Sprintf("%s-%s", hashStr, containerName))) + combinedHashStr := hex.EncodeToString(sum[:])[:10] + return fmt.Sprintf("%s-%s", pod.Name, combinedHashStr) +} + +func (r *ReconcileConfigMapSet) rebootSidecarByCrr(pod *corev1.Pod, containerName string, hashStr string) error { + if len(containerName) == 0 { + return nil + } + crrName := r.genContainerRecreateRequestName(pod, containerName, hashStr) + // Check if CRR already exists first + existingCrr := &appsv1alpha1.ContainerRecreateRequest{} + err := r.Get(context.TODO(), types.NamespacedName{Namespace: pod.Namespace, Name: crrName}, existingCrr) + if err == nil { + klog.Infof("CRR %s already exists for pod %s/%s to reboot container %s", crrName, pod.Namespace, pod.Name, containerName) + return nil + } else if !errors.IsNotFound(err) { + klog.Errorf("Failed to get CRR %s for pod %s/%s: %v", crrName, pod.Namespace, pod.Name, err) + return err + } + + // Create a new CRR + isController := true + crr := &appsv1alpha1.ContainerRecreateRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: crrName, + Namespace: pod.Namespace, + Labels: map[string]string{ + GetConfigMapSetCrrKey(): "true", + }, + Annotations: make(map[string]string), + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "v1", + Kind: "Pod", + Name: pod.Name, + UID: pod.UID, + Controller: &isController, + }, + }, + }, + Spec: appsv1alpha1.ContainerRecreateRequestSpec{ + PodName: pod.Name, + Strategy: &appsv1alpha1.ContainerRecreateRequestStrategy{ + ForceRecreate: true, + FailurePolicy: appsv1alpha1.ContainerRecreateRequestFailurePolicyFail, + OrderedRecreate: true, + UnreadyGracePeriodSeconds: ptr.To[int64](5), + MinStartedSeconds: int32(3), + }, + ActiveDeadlineSeconds: pointer.Int64(14400), + TTLSecondsAfterFinished: pointer.Int32(3), + }, + } + + crr.Spec.Containers = []appsv1alpha1.ContainerRecreateRequestContainer{{Name: containerName}} + for _, status := range pod.Status.ContainerStatuses { + if status.Name == containerName && status.State.Running != nil { + crr.Annotations[GetConfigMapSetContainerStartedAtKey(containerName)] = status.State.Running.StartedAt.Format(time.RFC3339) + } + } + + err = r.Create(context.TODO(), crr) + if err != nil { + if errors.IsAlreadyExists(err) { + return nil + } + klog.Errorf("Failed to create CRR for pod %s/%s container %s: %v", pod.Namespace, pod.Name, containerName, err) + return err + } + + klog.Infof("Created CRR %s for pod %s/%s to reboot container %s", crr.Name, pod.Namespace, pod.Name, containerName) + return nil +} + +func (r *ReconcileConfigMapSet) waitSidecarRebootByCrrSuccess(ctx context.Context, pod *corev1.Pod, containerName string, hashStr string) error { + if len(containerName) == 0 { + return nil + } + + // Combine hashStr and sortedContainerNames to form a unique hash for this specific set of containers and targetRevision + crrName := r.genContainerRecreateRequestName(pod, containerName, hashStr) + + crr := &appsv1alpha1.ContainerRecreateRequest{} + err := r.Get(ctx, types.NamespacedName{Namespace: pod.Namespace, Name: crrName}, crr) + if err != nil { + if errors.IsNotFound(err) { + klog.Infof("CRR %s not found in waitSidecarRebootByCrrSuccess", crrName) + // Fallback to check if the pod's container has actually restarted + recordedTimeStr := pod.Annotations[GetConfigMapSetContainerStartedAtKey(containerName)] + if recordedTimeStr == "" { + return fmt.Errorf("CRR %s not found startAt annotation", crrName) + } + recordedTime, parseErr := time.Parse(time.RFC3339, recordedTimeStr) + if parseErr != nil { + return parseErr + } + for _, status := range pod.Status.ContainerStatuses { + if status.Name == containerName { + if status.State.Running == nil { + return fmt.Errorf("CRR %s not found and container %s is not running", crrName, containerName) + } + if !status.State.Running.StartedAt.After(recordedTime) { + return fmt.Errorf("CRR %s not found and container %s has not restarted yet", crrName, containerName) + } + } + } + return nil + } + return fmt.Errorf("failed to get CRR %s: %v", crrName, err) + } + + if crr.Status.Phase != appsv1alpha1.ContainerRecreateRequestCompleted { + return fmt.Errorf("CRR %s is in phase %s", crrName, crr.Status.Phase) + } + + // For Completed CRR, verify that all target containers were recreated successfully + for _, cState := range crr.Status.ContainerRecreateStates { + if cState.Phase != appsv1alpha1.ContainerRecreateRequestSucceeded { + return fmt.Errorf("container %s in CRR %s is in phase %s, message: %s", cState.Name, crrName, cState.Phase, cState.Message) + } + } + + // Verify that the StartedAt time of all containers is strictly greater than the time recorded in the CRR + recordedTimeStr := crr.Annotations[GetConfigMapSetContainerStartedAtKey(containerName)] + if recordedTimeStr != "" { + recordedTime, err := time.Parse(time.RFC3339, recordedTimeStr) + if err == nil { + found := false + for _, status := range pod.Status.ContainerStatuses { + if status.Name == containerName { + found = true + if status.State.Running == nil { + return fmt.Errorf("container %s is not running yet", containerName) + } + if !status.State.Running.StartedAt.After(recordedTime) { + return fmt.Errorf("container %s has not been restarted yet (StartedAt <= recorded time)", containerName) + } + } + } + if !found { + return fmt.Errorf("container %s not found in pod status", containerName) + } + } + } + + // Clean up the CRR since we have successfully rebooted + err = r.Delete(ctx, crr) + if err != nil && !errors.IsNotFound(err) { + klog.Errorf("Failed to delete completed CRR %s for pod %s/%s: %v", crrName, pod.Namespace, pod.Name, err) + return err + } + + return nil +} diff --git a/pkg/controller/configmapset/configmapset_controller_test.go b/pkg/controller/configmapset/configmapset_controller_test.go new file mode 100644 index 0000000000..4d88ce18af --- /dev/null +++ b/pkg/controller/configmapset/configmapset_controller_test.go @@ -0,0 +1,572 @@ +package configmapset + +import ( + "context" + "encoding/json" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" +) + +func init() { + _ = appsv1alpha1.AddToScheme(scheme.Scheme) +} + +func TestCleanHistoryRevision(t *testing.T) { + limit := int32(1) + cms := &appsv1alpha1.ConfigMapSet{ + ObjectMeta: metav1.ObjectMeta{Name: "test-cms", Namespace: "default"}, + Spec: appsv1alpha1.ConfigMapSetSpec{ + RevisionHistoryLimit: &limit, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + UpdateStrategy: &appsv1alpha1.ConfigMapSetUpdateStrategy{}, + }, + } + + revisions := []RevisionEntry{ + {Hash: "hash1", CustomVersion: "v1"}, + {Hash: "hash2", CustomVersion: "v2"}, + } + revBytes, _ := json.Marshal(revisions) + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: GetConfigMapSetHubName("test-cms"), Namespace: "default"}, + Data: map[string]string{"revisions": string(revBytes)}, + } + + fakeClient := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(cms, cm).Build() + r := &ReconcileConfigMapSet{Client: fakeClient, scheme: scheme.Scheme} + + err := r.cleanHistoryRevision(context.TODO(), cms) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + updatedCm := &corev1.ConfigMap{} + err = fakeClient.Get(context.TODO(), types.NamespacedName{Name: GetConfigMapSetHubName("test-cms"), Namespace: "default"}, updatedCm) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + var updatedRevisions []RevisionEntry + json.Unmarshal([]byte(updatedCm.Data["revisions"]), &updatedRevisions) + + if len(updatedRevisions) != 1 { + t.Fatalf("expected 1 revision, got %d", len(updatedRevisions)) + } + if updatedRevisions[0].Hash != "hash2" { + t.Errorf("expected hash2, got %s", updatedRevisions[0].Hash) + } +} + +func TestCrrOperations(t *testing.T) { + startTime := metav1.Time{Time: time.Now().Add(-1 * time.Hour).Truncate(time.Second)} + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "test-pod", Namespace: "default"}, + Status: corev1.PodStatus{ + ContainerStatuses: []corev1.ContainerStatus{ + { + Name: "container1", + State: corev1.ContainerState{ + Running: &corev1.ContainerStateRunning{ + StartedAt: startTime, + }, + }, + }, + }, + }, + } + fakeClient := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(pod).Build() + r := &ReconcileConfigMapSet{Client: fakeClient, scheme: scheme.Scheme} + + err := r.rebootSidecarByCrr(pod, "container1", "hash1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Verify CRR is created + crrList := &appsv1alpha1.ContainerRecreateRequestList{} + fakeClient.List(context.TODO(), crrList) + if len(crrList.Items) != 1 { + t.Fatalf("expected 1 CRR, got %d", len(crrList.Items)) + } + + crr := &crrList.Items[0] + if crr.Spec.Containers[0].Name != "container1" { + t.Errorf("expected container1, got %s", crr.Spec.Containers[0].Name) + } + + // Verify annotation is set + if crr.Annotations[GetConfigMapSetContainerStartedAtKey("container1")] == "" { + t.Errorf("expected container1 startedAt annotation to be set") + } else { + t.Logf("Annotation is set: %s", crr.Annotations[GetConfigMapSetContainerStartedAtKey("container1")]) + } + + // Mark CRR as completed + crr.Status.Phase = appsv1alpha1.ContainerRecreateRequestCompleted + crr.Status.ContainerRecreateStates = []appsv1alpha1.ContainerRecreateRequestContainerRecreateState{ + {Name: "container1", Phase: appsv1alpha1.ContainerRecreateRequestSucceeded}, + } + err = fakeClient.Update(context.TODO(), crr) + if err != nil { + t.Fatalf("failed to update CRR: %v", err) + } + + // wait should fail because StartedAt hasn't advanced + err = r.waitSidecarRebootByCrrSuccess(context.TODO(), pod, "container1", "hash1") + if err == nil { + t.Fatalf("expected error due to StartedAt not advancing, got nil") + } + + // advance StartedAt + pod.Status.ContainerStatuses[0].State.Running.StartedAt = metav1.Time{Time: time.Now()} + err = r.waitSidecarRebootByCrrSuccess(context.TODO(), pod, "container1", "hash1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestReconcile(t *testing.T) { + cms := &appsv1alpha1.ConfigMapSet{ + ObjectMeta: metav1.ObjectMeta{Name: "test-cms", Namespace: "default"}, + Spec: appsv1alpha1.ConfigMapSetSpec{ + Data: map[string]string{"key": "val"}, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + UpdateStrategy: &appsv1alpha1.ConfigMapSetUpdateStrategy{}, + }, + } + fakeClient := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(cms).WithStatusSubresource(&appsv1alpha1.ConfigMapSet{}).Build() + r := &ReconcileConfigMapSet{Client: fakeClient, scheme: scheme.Scheme} + + req := reconcile.Request{ + NamespacedName: types.NamespacedName{Name: "test-cms", Namespace: "default"}, + } + res, err := r.Reconcile(context.TODO(), req) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if res.Requeue { + t.Errorf("expected no requeue") + } + + updatedCms := &appsv1alpha1.ConfigMapSet{} + err = fakeClient.Get(context.TODO(), req.NamespacedName, updatedCms) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(updatedCms.Finalizers) == 0 || updatedCms.Finalizers[0] != ConfigMapFinalizerName { + t.Errorf("expected finalizer to be added") + } + + hubName := GetConfigMapSetHubName("test-cms") + cm := &corev1.ConfigMap{} + err = fakeClient.Get(context.TODO(), types.NamespacedName{Name: hubName, Namespace: "default"}, cm) + if err != nil { + t.Fatalf("expected hub ConfigMap to be created, got error: %v", err) + } +} + +func TestCalculateHash(t *testing.T) { + data1 := map[string]string{"key1": "val1", "key2": "val2"} + data2 := map[string]string{"key2": "val2", "key1": "val1"} // Same data, different order + + hash1, err := CalculateHash(data1) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + hash2, err := CalculateHash(data2) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if hash1 != hash2 { + t.Errorf("expected hashes to be identical for same data, got %s and %s", hash1, hash2) + } + + data3 := map[string]string{"key1": "val3"} + hash3, err := CalculateHash(data3) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if hash1 == hash3 { + t.Errorf("expected hashes to be different for different data, got %s", hash1) + } +} + +func TestGetDistributionByPartition(t *testing.T) { + cases := []struct { + name string + replicas int + partition *intstr.IntOrString + expectedNewReplicas int32 + expectedOldReplicas int32 + }{ + { + name: "nil partition", + replicas: 10, + partition: nil, + expectedNewReplicas: 10, + expectedOldReplicas: 0, + }, + { + name: "int partition 0", + replicas: 10, + partition: &intstr.IntOrString{Type: intstr.Int, IntVal: 0}, + expectedNewReplicas: 10, + expectedOldReplicas: 0, + }, + { + name: "int partition 3", + replicas: 10, + partition: &intstr.IntOrString{Type: intstr.Int, IntVal: 3}, + expectedNewReplicas: 7, + expectedOldReplicas: 3, + }, + { + name: "int partition greater than replicas", + replicas: 10, + partition: &intstr.IntOrString{Type: intstr.Int, IntVal: 15}, + expectedNewReplicas: 0, + expectedOldReplicas: 10, + }, + { + name: "string percent partition 30%", + replicas: 10, + partition: &intstr.IntOrString{Type: intstr.String, StrVal: "30%"}, + expectedNewReplicas: 7, + expectedOldReplicas: 3, + }, + { + name: "string percent partition 33% ceil", + replicas: 10, + partition: &intstr.IntOrString{Type: intstr.String, StrVal: "33%"}, // 3.3 -> ceil(3.3) = 4 old replicas -> 6 new + expectedNewReplicas: 6, + expectedOldReplicas: 4, + }, + { + name: "string int partition 3", + replicas: 10, + partition: &intstr.IntOrString{Type: intstr.String, StrVal: "3"}, + expectedNewReplicas: 7, + expectedOldReplicas: 3, + }, + { + name: "string percent partition 100%", + replicas: 10, + partition: &intstr.IntOrString{Type: intstr.String, StrVal: "100%"}, + expectedNewReplicas: 0, + expectedOldReplicas: 10, + }, + { + name: "string percent partition 0%", + replicas: 10, + partition: &intstr.IntOrString{Type: intstr.String, StrVal: "0%"}, + expectedNewReplicas: 10, + expectedOldReplicas: 0, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cms := &appsv1alpha1.ConfigMapSet{ + Spec: appsv1alpha1.ConfigMapSetSpec{ + Data: map[string]string{"key": "value"}, + UpdateStrategy: &appsv1alpha1.ConfigMapSetUpdateStrategy{ + Partition: tc.partition, + }, + }, + Status: appsv1alpha1.ConfigMapSetStatus{ + CurrentRevision: "old-rev", + }, + } + + pods := make([]*corev1.Pod, tc.replicas) + for i := 0; i < tc.replicas; i++ { + pods[i] = &corev1.Pod{} + } + + distributions, err := getDistributionByPartition(cms, pods) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(distributions) != 2 { + t.Fatalf("expected 2 distributions, got %d", len(distributions)) + } + + if distributions[0].Reserved != tc.expectedNewReplicas { + t.Errorf("expected new replicas %d, got %d", tc.expectedNewReplicas, distributions[0].Reserved) + } + + if distributions[1].Reserved != tc.expectedOldReplicas { + t.Errorf("expected old replicas %d, got %d", tc.expectedOldReplicas, distributions[1].Reserved) + } + }) + } +} + +func TestGetUpdatePodsByDistributions(t *testing.T) { + revisionKeys := &RevisionKeys{ + CurrentRevisionKey: "current-rev", + } + + now := metav1.Now() + later := metav1.Time{Time: now.Add(time.Hour)} + + // Revisions + updateRev := "rev-update" + currentRev := "rev-current" + olderRev := "rev-older" + oldestRev := "rev-oldest" + unknownRev := "rev-unknown" + + hubRevisions := []RevisionEntry{ + {Hash: oldestRev}, + {Hash: olderRev}, + {Hash: currentRev}, + {Hash: updateRev}, + } + + // 1. Revisions in RMC: oldestRev, olderRev, currentRev, updateRev + // 2. Revisions not in RMC: unknownRev + + // Construct Pods with different states + createPod := func(name, rev string, creationTime metav1.Time, ready bool) *corev1.Pod { + status := corev1.ConditionFalse + if ready { + status = corev1.ConditionTrue + } + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Annotations: map[string]string{ + GetConfigMapSetEnabledKey(): "true", + revisionKeys.CurrentRevisionKey: rev, + }, + CreationTimestamp: creationTime, + }, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + {Type: corev1.PodReady, Status: status}, + }, + }, + } + } + + // pods list + pCurrent := createPod("p-current", currentRev, now, true) + pOldest := createPod("p-oldest", oldestRev, now, true) + pOlder := createPod("p-older", olderRev, now, true) + pUnknown := createPod("p-unknown", unknownRev, now, true) + pUpdate := createPod("p-update", updateRev, now, true) + + pOlderLater := createPod("p-older-later", olderRev, later, true) + pOlderAlpha1 := createPod("a-older", olderRev, now, true) + pOlderAlpha2 := createPod("b-older", olderRev, now, true) + + pCurrentUnready := createPod("p-current-unready", currentRev, now, false) + + cases := []struct { + name string + pods []*corev1.Pod + distributions []Distribution + maxUnavailable *intstr.IntOrString + expectedUpdateCount int + expectedFirstPodName string + }{ + { + name: "Priority Rule 1: Non-currentRevision > currentRevision", + pods: []*corev1.Pod{pCurrent, pOlder}, + distributions: []Distribution{ + {Revision: updateRev, Reserved: 2}, // Want to update all + {Revision: currentRev, Reserved: 0}, + }, + expectedUpdateCount: 2, + expectedFirstPodName: "p-older", // pOlder is not current, pCurrent is + }, + { + name: "Priority Rule 2: Older Revision > Newer Revision", + pods: []*corev1.Pod{pOlder, pOldest}, + distributions: []Distribution{ + {Revision: updateRev, Reserved: 1}, // Want to update 1 + {Revision: currentRev, Reserved: 1}, + }, + expectedUpdateCount: 1, + expectedFirstPodName: "p-oldest", // pOldest is older than pOlder in RMC + }, + { + name: "Priority Rule 3: Unknown Revision > Known Revision", + pods: []*corev1.Pod{pOldest, pUnknown}, + distributions: []Distribution{ + {Revision: updateRev, Reserved: 1}, // Want to update 1 + {Revision: currentRev, Reserved: 1}, + }, + expectedUpdateCount: 1, + expectedFirstPodName: "p-unknown", // pUnknown is not in RMC, pOldest is + }, + { + name: "Priority Rule 4: Older CreationTimestamp > Newer CreationTimestamp", + pods: []*corev1.Pod{pOlderLater, pOlder}, + distributions: []Distribution{ + {Revision: updateRev, Reserved: 1}, + {Revision: currentRev, Reserved: 1}, + }, + expectedUpdateCount: 1, + expectedFirstPodName: "p-older", // pOlder created before pOlderLater + }, + { + name: "Priority Rule 5: Alphabetical Name", + pods: []*corev1.Pod{pOlderAlpha2, pOlderAlpha1}, + distributions: []Distribution{ + {Revision: updateRev, Reserved: 1}, + {Revision: currentRev, Reserved: 1}, + }, + expectedUpdateCount: 1, + expectedFirstPodName: "a-older", // a-older < b-older + }, + { + name: "MaxUnavailable limit reached", + pods: []*corev1.Pod{ + createPod("p1", currentRev, now, false), // 1 unavailable + createPod("p2", currentRev, now, true), + }, + distributions: []Distribution{ + {Revision: updateRev, Reserved: 2}, + {Revision: currentRev, Reserved: 0}, + }, + maxUnavailable: &intstr.IntOrString{Type: intstr.Int, IntVal: 1}, + expectedUpdateCount: 0, // 1 unavailable >= maxUnavailable(1), cannot update more + }, + { + name: "Skip already updated pods", + pods: []*corev1.Pod{pUpdate, pCurrent}, + distributions: []Distribution{ + {Revision: updateRev, Reserved: 2}, + {Revision: currentRev, Reserved: 0}, + }, + expectedUpdateCount: 1, + expectedFirstPodName: "p-current", // pUpdate is already updated + }, + { + name: "MaxUnavailable limit reached exactly", + pods: []*corev1.Pod{ + pCurrentUnready, + pCurrent, + }, + distributions: []Distribution{ + {Revision: updateRev, Reserved: 2}, + {Revision: currentRev, Reserved: 0}, + }, + maxUnavailable: &intstr.IntOrString{Type: intstr.Int, IntVal: 1}, + expectedUpdateCount: 0, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cms := &appsv1alpha1.ConfigMapSet{ + Spec: appsv1alpha1.ConfigMapSetSpec{ + UpdateStrategy: &appsv1alpha1.ConfigMapSetUpdateStrategy{ + MaxUnavailable: tc.maxUnavailable, + }, + }, + } + + podsToUpdate, _, _ := getUpdatePodsByDistributions(cms, tc.distributions, tc.pods, revisionKeys, hubRevisions) + + if len(podsToUpdate) != tc.expectedUpdateCount { + t.Fatalf("expected %d pods to update, got %d", tc.expectedUpdateCount, len(podsToUpdate)) + } + + if tc.expectedUpdateCount > 0 && podsToUpdate[0].Name != tc.expectedFirstPodName { + t.Errorf("expected first pod to be %s, got %s", tc.expectedFirstPodName, podsToUpdate[0].Name) + } + }) + } +} + +func TestGetContainerName(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "container-name-from-annotation": "annot-container", + }, + Labels: map[string]string{ + "container-name-from-label": "label-container", + }, + }, + } + + cases := []struct { + name string + containerSpec appsv1alpha1.ConfigMapSetContainer + expectedName string + }{ + { + name: "direct name", + containerSpec: appsv1alpha1.ConfigMapSetContainer{ + Name: "direct-container", + }, + expectedName: "direct-container", + }, + { + name: "name from annotation", + containerSpec: appsv1alpha1.ConfigMapSetContainer{ + NameFrom: &appsv1alpha1.SourceContainerNameSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.annotations['container-name-from-annotation']", + }, + }, + }, + expectedName: "annot-container", + }, + { + name: "name from label", + containerSpec: appsv1alpha1.ConfigMapSetContainer{ + NameFrom: &appsv1alpha1.SourceContainerNameSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.labels['container-name-from-label']", + }, + }, + }, + expectedName: "label-container", + }, + { + name: "name from missing label", + containerSpec: appsv1alpha1.ConfigMapSetContainer{ + NameFrom: &appsv1alpha1.SourceContainerNameSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.labels['missing-label']", + }, + }, + }, + expectedName: "", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + name := GetContainerName(pod, tc.containerSpec) + if name != tc.expectedName { + t.Errorf("expected container name %s, got %s", tc.expectedName, name) + } + }) + } +} diff --git a/pkg/controller/configmapset/configmapset_event_handler.go b/pkg/controller/configmapset/configmapset_event_handler.go new file mode 100644 index 0000000000..d5cece92e6 --- /dev/null +++ b/pkg/controller/configmapset/configmapset_event_handler.go @@ -0,0 +1,124 @@ +package configmapset + +import ( + "context" + "reflect" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" +) + +// ConfigMapSet event handler +type configMapSetEventHandler struct { + client.Reader +} + +var _ handler.TypedEventHandler[*appsv1alpha1.ConfigMapSet, reconcile.Request] = &configMapSetEventHandler{} + +func (e *configMapSetEventHandler) Create(ctx context.Context, evt event.TypedCreateEvent[*appsv1alpha1.ConfigMapSet], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + obj := evt.Object + e.handle(obj, q) +} + +func (e *configMapSetEventHandler) Update(ctx context.Context, evt event.TypedUpdateEvent[*appsv1alpha1.ConfigMapSet], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + obj := evt.ObjectNew + oldObj := evt.ObjectOld + if obj.DeletionTimestamp != nil { + e.handle(obj, q) + } else { + e.handleUpdate(obj, oldObj, q) + } +} + +func (e *configMapSetEventHandler) Delete(ctx context.Context, evt event.TypedDeleteEvent[*appsv1alpha1.ConfigMapSet], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + obj := evt.Object + e.handle(obj, q) +} + +func (e *configMapSetEventHandler) Generic(ctx context.Context, evt event.TypedGenericEvent[*appsv1alpha1.ConfigMapSet], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { +} + +func (e *configMapSetEventHandler) handle(cms *appsv1alpha1.ConfigMapSet, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + pods, err := getMatchedPods(e.Reader, cms) + if err != nil { + klog.Errorf("Failed to get Pods for appsv1alpha1.ConfigMapSet %s: %v", cms.Name, err) + return + } + for _, pod := range pods.Items { + q.Add(reconcile.Request{NamespacedName: types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}}) + } +} + +func (e *configMapSetEventHandler) handleUpdate(cms, oldCms *appsv1alpha1.ConfigMapSet, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + if reflect.DeepEqual(cms.Spec, oldCms.Spec) { + return + } + e.handle(cms, q) +} + +// Pod event handler +type podEventHandler struct { + client.Reader +} + +var _ handler.TypedEventHandler[*v1.Pod, reconcile.Request] = &podEventHandler{} + +func (p *podEventHandler) Create(ctx context.Context, evt event.TypedCreateEvent[*v1.Pod], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + pod := evt.Object + if pod.DeletionTimestamp != nil { + p.Delete(ctx, event.TypedDeleteEvent[*v1.Pod]{Object: evt.Object}, q) + return + } + p.handle(pod, q) +} +func (p *podEventHandler) Update(ctx context.Context, evt event.TypedUpdateEvent[*v1.Pod], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + obj := evt.ObjectNew + // Do not process terminating pods + if obj.DeletionTimestamp != nil { + klog.Infof("Pod %s/%s is terminating, skip", obj.Namespace, obj.Name) + return + } + p.handle(obj, q) +} + +func (p *podEventHandler) Delete(ctx context.Context, evt event.TypedDeleteEvent[*v1.Pod], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + //obj := evt.Object.(*corev1.Pod) + //e.handle(obj, q) +} + +func (p *podEventHandler) Generic(ctx context.Context, evt event.TypedGenericEvent[*v1.Pod], q workqueue.TypedRateLimitingInterface[reconcile.Request]) { +} + +func (p *podEventHandler) handleUpdate(pod, oldPod *v1.Pod, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + if pod.Spec.NodeName == "" { + return + } + p.handle(pod, q) +} + +func (p *podEventHandler) handle(pod *v1.Pod, q workqueue.TypedRateLimitingInterface[reconcile.Request]) { + if len(pod.Labels) == 0 { + klog.Infof("Pod %s/%s has no labels, skip", pod.Namespace, pod.Name) + return + } + cmsList, err := GetMatchConfigMapSets(p.Reader, pod) + if err != nil { + klog.Errorf("Failed to get ConfigMapSet for Pod %s/%s: %v", pod.Namespace, pod.Name, err) + return + } + if len(cmsList) > 0 { + // Add to queue for each one + for _, cms := range cmsList { + klog.Infof("Pod %s/%s update ConfigMapSet %s", pod.Namespace, pod.Name, cms.Name) + q.Add(reconcile.Request{NamespacedName: types.NamespacedName{Namespace: cms.Namespace, Name: cms.Name}}) + } + } +} diff --git a/pkg/controller/configmapset/configmapset_utils.go b/pkg/controller/configmapset/configmapset_utils.go new file mode 100644 index 0000000000..d5d6020bae --- /dev/null +++ b/pkg/controller/configmapset/configmapset_utils.go @@ -0,0 +1,441 @@ +/* +Copyright 2019 The Kruise Authors. +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package configmapset + +import ( + "context" + "crypto/md5" + "encoding/hex" + "flag" + "fmt" + "k8s.io/apimachinery/pkg/types" + "sort" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/klog/v2" + kubecontroller "k8s.io/kubernetes/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" +) + +var ( + defaultReloadConfigMap string + defaultReloadSidecarSet string +) + +func init() { + flag.StringVar(&defaultReloadConfigMap, "default-reload-configmap", "kruise-system/default-cms-reloadsidecar-cm", "Default reload sidecar image to use if not explicitly specified in ConfigMapSet.") + flag.StringVar(&defaultReloadSidecarSet, "default-reload-sidecarset", "default-cms-reloadsidecar-sidecarset/reload-sidecar", "Default reload sidecar image to use if not explicitly specified in ConfigMapSet.") + +} + +const ( + ConfigMapSetAnnotationPrefix = "apps.kruise.io/configmapset." + ConfigMapSetAnnotationEnabled = "apps.kruise.io/configmapset-enabled" + ConfigMapSetAnnotationCrr = "apps.kruise.io/configmapset-crr" + ConfigMapSetAnnotationUpdateRevisionSuffix = ".updateRevision" + ConfigMapSetAnnotationCurrentRevisionSuffix = ".currentRevision" + ConfigMapSetAnnotationUpdateRevisionTimeStampSuffix = ".updateRevisionTimestamp" + ConfigMapSetAnnotationCurrentRevisionTimeStampSuffix = ".currentRevisionTimestamp" + ConfigMapSetAnnotationUpdateCustomVersionSuffix = ".updateCustomVersion" + ConfigMapSetAnnotationCurrentCustomVersionSuffix = ".currentCustomVersion" +) + +func GetConfigMapSetEnabledKey() string { + return ConfigMapSetAnnotationEnabled +} + +func GetConfigMapSetCrrKey() string { + return ConfigMapSetAnnotationCrr +} + +func GetConfigMapSetUpdateRevisionKey(cmsName string) string { + return ConfigMapSetAnnotationPrefix + cmsName + ConfigMapSetAnnotationUpdateRevisionSuffix +} + +func GetConfigMapSetCurrentRevisionKey(cmsName string) string { + return ConfigMapSetAnnotationPrefix + cmsName + ConfigMapSetAnnotationCurrentRevisionSuffix +} + +func GetConfigMapSetUpdateRevisionTimeStampKey(cmsName string) string { + return ConfigMapSetAnnotationPrefix + cmsName + ConfigMapSetAnnotationUpdateRevisionTimeStampSuffix +} + +func GetConfigMapSetCurrentRevisionTimeStampKey(cmsName string) string { + return ConfigMapSetAnnotationPrefix + cmsName + ConfigMapSetAnnotationCurrentRevisionTimeStampSuffix +} + +func GetConfigMapSetUpdateCustomVersionKey(cmsName string) string { + return ConfigMapSetAnnotationPrefix + cmsName + ConfigMapSetAnnotationUpdateCustomVersionSuffix +} + +func GetConfigMapSetCurrentCustomVersionKey(cmsName string) string { + return ConfigMapSetAnnotationPrefix + cmsName + ConfigMapSetAnnotationCurrentCustomVersionSuffix +} + +func GetConfigMapSetReloadSidecarRestartKey(cmsName string) string { + return fmt.Sprintf("apps.kruise.io/configmapset-%s-reload-sidecar-restart", cmsName) +} + +func GetContainerHash(pod *corev1.Pod, revision string) string { + podNameForHash := pod.Name + hashBytes := md5.Sum([]byte(podNameForHash + revision)) + return hex.EncodeToString(hashBytes[:]) +} + +func GetConfigMapSetContainerRestartKey(cmsName, containerName string) string { + return fmt.Sprintf("apps.kruise.io/configmapset-%s-%s-restart", cmsName, containerName) +} + +func GetConfigMapSetPostHookConfigKey(cmsName string) string { + return fmt.Sprintf("apps.kruise.io/configmapset-%s-post-hook-config", cmsName) +} + +// GetConfigMapSetContainerStartedAtKey gets the container StartedAt annotation key for CRR +func GetConfigMapSetContainerStartedAtKey(containerName string) string { + return fmt.Sprintf("configmapset.kruise.io/started-at-%s", containerName) +} + +func GenerateDerivedName(name, suffix string) string { + fullName := fmt.Sprintf("%s-%s", name, suffix) + if len(fullName) <= 63 { + return fullName + } + hash := md5.Sum([]byte(name)) + hashStr := hex.EncodeToString(hash[:])[:8] + maxNameLen := 53 - len(suffix) + truncatedName := name + if maxNameLen > 0 { + truncatedName = name[:maxNameLen] + // Trim trailing hyphens to avoid invalid DNS names like `xxx---hash-suffix` + truncatedName = strings.TrimRight(truncatedName, "-") + } else { + truncatedName = "" + } + if truncatedName != "" { + return fmt.Sprintf("%s-%s-%s", truncatedName, hashStr, suffix) + } + return fmt.Sprintf("%s-%s", hashStr, suffix) +} + +func GetConfigMapSetHubName(cmsName string) string { + return GenerateDerivedName(strings.ToLower(cmsName), "hub") +} + +func GetConfigMapSetDefaultSidecarName(cmsName string) string { + return GenerateDerivedName(strings.ToLower(cmsName), "reload-sidecar") +} + +func GetConfigMapSetVolumeName(cmsName string) string { + return GenerateDerivedName(strings.ToLower(cmsName), "empty") +} + +func GetConfigMapSetHubVolumeName(cmsName string) string { + return GenerateDerivedName(strings.ToLower(cmsName), "hub-volume") +} + +func GetConfigMapSetDownwardAPIVolumeName(cmsName string) string { + return GenerateDerivedName(fmt.Sprintf("cms-%s", strings.ToLower(cmsName)), "config") +} + +func GetDefaultCmsConfigMap() types.NamespacedName { + defaultReloadConfigMapArray := strings.Split(defaultReloadConfigMap, "/") + return types.NamespacedName{Namespace: defaultReloadConfigMapArray[0], Name: defaultReloadConfigMapArray[1]} +} + +func GetDefaultCmsSidecarSet() (string, string) { + defaultReloadSidecarSetArray := strings.Split(defaultReloadSidecarSet, "/") + return defaultReloadSidecarSetArray[0], defaultReloadSidecarSetArray[1] +} + +func GetConfigMapSetEnvConfigPathName() string { + return "CMS_CONFIG_PATH" +} + +func GetConfigMapSetEnvSharePathName() string { + return "CMS_SHARE_PATH" +} + +func GetConfigMapSetConfigMountPath(cmsName string) string { + return fmt.Sprintf("/etc/config/%s", strings.ToLower(cmsName)) +} + +func GetConfigMapSetConfigMapMountPath(cmsName string) string { + return fmt.Sprintf("/etc/cms/%s", strings.ToLower(cmsName)) +} + +func GetConfigMapSetEnvFieldPath(restartKey string) string { + return fmt.Sprintf("metadata.annotations['%s']", restartKey) +} + +// GetReloadSidecarHealthCheckScript returns the health check script for the reload-sidecar +func GetReloadSidecarHealthCheckScript(path string) string { + return fmt.Sprintf(` +TARGET_REVISION=$(cat /etc/cms_config/target_revision 2>/dev/null || true) +if [ -z "$TARGET_REVISION" ]; then exit 0; fi + +# 1. Verify target revision matches local identity (requires sidecar to write .current_revision) +# Retaining compatibility for content hash check here. Because the hash algorithm (json.marshal) +# differs across languages and OS, the standard practice is: +# After reload-sidecar copies/updates the files from configmap-hub to the shared directory, +# it generates a hidden file '.current_revision' in the shared directory to record the updated Hash. +if [ -f %[1]s/.current_revision ]; then + CURRENT_REVISION=$(cat %[1]s/.current_revision) + if [ "$TARGET_REVISION" != "$CURRENT_REVISION" ]; then + exit 1 + fi +else + exit 1 +fi + +# 2. If PostHook is configured, check for success marker +POST_HOOK_CONFIG=$(cat /etc/cms_config/post_hook_config 2>/dev/null || true) +if [ -n "$POST_HOOK_CONFIG" ] && [ "$POST_HOOK_CONFIG" != "null" ]; then + if [ ! -f "%[1]s/.post_hook_success_${TARGET_REVISION}" ]; then + exit 1 + fi +fi + +exit 0 +`, path) +} + +func containsString(slice []string, str string) bool { + for _, s := range slice { + if s == str { + return true + } + } + return false +} + +func removeString(slice []string, str string) []string { + result := make([]string, 0) + for _, s := range slice { + if s != str { + result = append(result, s) + } + } + return result +} + +func GetMatchedPods(ctx context.Context, reader client.Reader, cms *appsv1alpha1.ConfigMapSet) ([]*corev1.Pod, error) { + // Get related Pods by spec.selector + // Get matched pods + podList := &corev1.PodList{} + if cms.Spec.Selector == nil { + return nil, fmt.Errorf("ConfigMapSet %s has no selector", cms.Name) + } + labelSelector, err := metav1.LabelSelectorAsSelector(cms.Spec.Selector) + if err != nil { + return nil, fmt.Errorf("invalid label selector: %v", err) + } + opts := &client.ListOptions{ + Namespace: cms.Namespace, + LabelSelector: labelSelector, // Use converted LabelSelector + } + err = reader.List(ctx, podList, opts) + if err != nil { + if errors.IsNotFound(err) { + return nil, nil + } + return nil, fmt.Errorf("failed to list pods: %v", err) + } + + // Filter inactive pods, newly started pods will not be filtered + matchedPods := make([]*corev1.Pod, 0) + klog.Infof("total pods: %d", len(podList.Items)) + for i, pod := range podList.Items { + if !kubecontroller.IsPodActive(&pod) { + klog.V(4).Infof("Pod %s/%s is not active, skipping", pod.Namespace, pod.Name) + continue + } + if pod.DeletionTimestamp != nil { + klog.V(4).Infof("Pod %s/%s is being deleted, skipping", pod.Namespace, pod.Name) + continue + } + matchedPods = append(matchedPods, &podList.Items[i]) + } + + return matchedPods, nil +} + +func IsPodReady(pod *corev1.Pod) bool { + for _, cond := range pod.Status.Conditions { + if cond.Type == corev1.PodReady && cond.Status == corev1.ConditionTrue { + return true + } + } + return false +} + +func GetMatchConfigMapSets(reader client.Reader, pod *corev1.Pod) ([]*appsv1alpha1.ConfigMapSet, error) { + res := make([]*appsv1alpha1.ConfigMapSet, 0) + + // Query ConfigMapSet objects + configMapSets := &appsv1alpha1.ConfigMapSetList{} + err := reader.List(context.Background(), configMapSets, &client.ListOptions{ + Namespace: pod.Namespace, + }) + if err != nil { + if errors.IsNotFound(err) { + return res, nil // Some pods may not have cms, do not return error + } + return nil, fmt.Errorf("failed to list ConfigMapSets in namespace %s: %v", pod.Namespace, err) + } + + // Find ConfigMapSets that match the Pod's labels + // There may be multiple + for i := range configMapSets.Items { + cms := configMapSets.Items[i] // Get actual element before taking address + if cms.DeletionTimestamp != nil { + continue + } + ls, err := metav1.LabelSelectorAsSelector(cms.Spec.Selector) + if err != nil { + return nil, fmt.Errorf("invalid label selector for ConfigMapSet %s: %v", cms.Name, err) + } + if ls.Matches(labels.Set(pod.Labels)) { + res = append(res, &cms) // The cms here is now independent + } + } + + return res, nil +} + +// GroupPodsByMatchLabelKeys groups pods by matchLabelKeys values. +// Missing key and empty value are treated equivalently. +// Pods form a group if and only if they have the same value for ALL keys. +// Returns groups in deterministic order. +func GroupPodsByMatchLabelKeys(pods []*corev1.Pod, matchLabelKeys []string) [][]*corev1.Pod { + if len(matchLabelKeys) == 0 { + return [][]*corev1.Pod{pods} + } + groupMap := make(map[string][]*corev1.Pod) + for _, pod := range pods { + groupKey := podGroupKey(pod, matchLabelKeys) + groupMap[groupKey] = append(groupMap[groupKey], pod) + } + keys := make([]string, 0, len(groupMap)) + for k := range groupMap { + keys = append(keys, k) + } + sort.Strings(keys) + groups := make([][]*corev1.Pod, 0, len(keys)) + for _, k := range keys { + groups = append(groups, groupMap[k]) + } + return groups +} + +// GetPodGroupByMatchLabelKeys returns the subset of pods that belong to the same group as targetPod. +func GetPodGroupByMatchLabelKeys(pods []*corev1.Pod, targetPod *corev1.Pod, matchLabelKeys []string) []*corev1.Pod { + if len(matchLabelKeys) == 0 { + return pods + } + var group []*corev1.Pod + for _, p := range pods { + allMatch := true + for _, key := range matchLabelKeys { + if p.Labels[key] != targetPod.Labels[key] { + allMatch = false + break + } + } + if allMatch { + group = append(group, p) + } + } + return group +} + +func podGroupKey(pod *corev1.Pod, matchLabelKeys []string) string { + keyVals := make([]string, len(matchLabelKeys)) + for i, key := range matchLabelKeys { + keyVals[i] = fmt.Sprintf("%d=%s", len(pod.Labels[key]), pod.Labels[key]) + } + return strings.Join(keyVals, "\x00") +} + +func getMatchedPods(reader client.Reader, cms *appsv1alpha1.ConfigMapSet) (*corev1.PodList, error) { + // Get related Pods by spec.selector + // Get matched pods + pods := &corev1.PodList{} + labelSelector, err := metav1.LabelSelectorAsSelector(cms.Spec.Selector) + if err != nil { + return nil, fmt.Errorf("invalid label selector: %v", err) + } + err = reader.List(context.Background(), pods, &client.ListOptions{ + Namespace: cms.Namespace, + LabelSelector: labelSelector, // Use converted LabelSelector + }) + if err != nil { + return nil, fmt.Errorf("failed to list pods: %v", err) + } + return pods, nil +} + +// GetReloadSidecarReadinessProbe returns the readiness probe for the reload sidecar. +func GetReloadSidecarReadinessProbe() *corev1.Probe { + return &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + Exec: &corev1.ExecAction{ + Command: []string{ + "sh", + "-c", + ` +TARGET_REVISION=$(cat /etc/cms_config/target_revision 2>/dev/null || true) +if [ -z "$TARGET_REVISION" ]; then exit 0; fi + +# 1. Verify target revision matches local identity (requires sidecar to write .current_revision) +# Retaining compatibility for content hash check here. Because the hash algorithm (json.marshal) +# differs across languages and OS, the standard practice is: +# After reload-sidecar copies/updates the files from configmap-hub to the shared directory, +# it generates a hidden file '.current_revision' in the shared directory to record the updated Hash. +if [ -f /etc/config/.current_revision ]; then + CURRENT_REVISION=$(cat /etc/config/.current_revision) + if [ "$TARGET_REVISION" != "$CURRENT_REVISION" ]; then + exit 1 + fi +else + exit 1 +fi + +# 2. If PostHook is configured, check for success marker +POST_HOOK_CONFIG=$(cat /etc/cms_config/post_hook_config 2>/dev/null || true) +if [ -n "$POST_HOOK_CONFIG" ] && [ "$POST_HOOK_CONFIG" != "null" ]; then + if [ ! -f "/etc/config/.post_hook_success_${TARGET_REVISION}" ]; then + exit 1 + fi +fi + +exit 0 +`, + }, + }, + }, + InitialDelaySeconds: 1, + PeriodSeconds: 3, + TimeoutSeconds: 5, + } +} diff --git a/pkg/controller/configmapset/configmapset_utils_test.go b/pkg/controller/configmapset/configmapset_utils_test.go new file mode 100644 index 0000000000..230a9cf69e --- /dev/null +++ b/pkg/controller/configmapset/configmapset_utils_test.go @@ -0,0 +1,364 @@ +package configmapset + +import ( + "context" + "encoding/json" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" +) + +func init() { + _ = appsv1alpha1.AddToScheme(scheme.Scheme) +} + +func TestGetMatchedPods(t *testing.T) { + now := metav1.Now() + + cms := &appsv1alpha1.ConfigMapSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cms", + Namespace: "default", + }, + Spec: appsv1alpha1.ConfigMapSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + }, + } + + podActive := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-active", + Namespace: "default", + Labels: map[string]string{"app": "test"}, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + } + + podPending := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-pending", + Namespace: "default", + Labels: map[string]string{"app": "test"}, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + }, + } + + podInactive := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-inactive", + Namespace: "default", + Labels: map[string]string{"app": "test"}, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodSucceeded, // Will be filtered out by IsPodActive + }, + } + + podDeleting := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-deleting", + Namespace: "default", + Labels: map[string]string{"app": "test"}, + DeletionTimestamp: &now, + Finalizers: []string{"test-finalizer"}, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, // Will be filtered out because of DeletionTimestamp != nil + }, + } + + podNotMatch := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-not-match", + Namespace: "default", + Labels: map[string]string{"app": "other"}, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + } + + fakeClient := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(podActive, podPending, podInactive, podDeleting, podNotMatch).Build() + + matchedPods, err := GetMatchedPods(context.TODO(), fakeClient, cms) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(matchedPods) != 2 { + t.Fatalf("expected 2 matched pod, got %d", len(matchedPods)) + } + + if matchedPods[0].Name != "pod-active" { + t.Errorf("expected pod-active, got %s", matchedPods[0].Name) + } +} + +func TestGetMatchConfigMapSets(t *testing.T) { + now := metav1.Now() + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Labels: map[string]string{"app": "test"}, + }, + } + + cmsMatched := &appsv1alpha1.ConfigMapSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cms-matched", + Namespace: "default", + }, + Spec: appsv1alpha1.ConfigMapSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + }, + } + + cmsMatched2 := &appsv1alpha1.ConfigMapSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cms-matched-2", + Namespace: "default", + }, + Spec: appsv1alpha1.ConfigMapSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + }, + } + + cmsNotMatched := &appsv1alpha1.ConfigMapSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cms-not-matched", + Namespace: "default", + }, + Spec: appsv1alpha1.ConfigMapSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "other"}, + }, + }, + } + + cmsDeleting := &appsv1alpha1.ConfigMapSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cms-deleting", + Namespace: "default", + DeletionTimestamp: &now, // Should be ignored based on our new logic + Finalizers: []string{"test-finalizer"}, + }, + Spec: appsv1alpha1.ConfigMapSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + }, + } + + fakeClient := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(cmsMatched, cmsMatched2, cmsNotMatched, cmsDeleting).Build() + + res, err := GetMatchConfigMapSets(fakeClient, pod) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(res) != 2 { + t.Fatalf("expected 2 matched ConfigMapSet, got %d", len(res)) + } + + if res[0].Name != "cms-matched" { + t.Errorf("expected cms-matched, got %s", res[0].Name) + } +} + +func TestIsPodReady(t *testing.T) { + cases := []struct { + name string + pod *corev1.Pod + expected bool + }{ + { + name: "pod ready", + pod: &corev1.Pod{ + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + {Type: corev1.PodReady, Status: corev1.ConditionTrue}, + }, + }, + }, + expected: true, + }, + { + name: "pod not ready", + pod: &corev1.Pod{ + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + {Type: corev1.PodReady, Status: corev1.ConditionFalse}, + }, + }, + }, + expected: false, + }, + { + name: "no ready condition", + pod: &corev1.Pod{ + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + {Type: corev1.PodScheduled, Status: corev1.ConditionTrue}, + }, + }, + }, + expected: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + result := IsPodReady(tc.pod) + if result != tc.expected { + t.Errorf("expected %v, got %v", tc.expected, result) + } + }) + } +} + +func TestCleanHistoryRevision(t *testing.T) { + limit := int32(1) + cms := &appsv1alpha1.ConfigMapSet{ + ObjectMeta: metav1.ObjectMeta{Name: "test-cms", Namespace: "default"}, + Spec: appsv1alpha1.ConfigMapSetSpec{ + RevisionHistoryLimit: &limit, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + }, + } + + revisions := []RevisionEntry{ + {Hash: "hash1", CustomVersion: "v1"}, + {Hash: "hash2", CustomVersion: "v2"}, + {Hash: "hash3", CustomVersion: "v3"}, + } + revBytes, _ := json.Marshal(revisions) + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: GetConfigMapSetHubName("test-cms"), Namespace: "default"}, + Data: map[string]string{"revisions": string(revBytes)}, + } + + fakeClient := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(cms, cm).Build() + r := &ReconcileConfigMapSet{Client: fakeClient, scheme: scheme.Scheme} + + err := r.cleanHistoryRevision(context.TODO(), cms) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + updatedCm := &corev1.ConfigMap{} + err = fakeClient.Get(context.TODO(), types.NamespacedName{Name: GetConfigMapSetHubName("test-cms"), Namespace: "default"}, updatedCm) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + var updatedRevisions []RevisionEntry + json.Unmarshal([]byte(updatedCm.Data["revisions"]), &updatedRevisions) + + if len(updatedRevisions) != 1 { + t.Fatalf("expected 1 revision, got %d", len(updatedRevisions)) + } + if updatedRevisions[0].Hash != "hash3" { + t.Errorf("expected hash3, got %s", updatedRevisions[0].Hash) + } +} + +func TestCrrOperations(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "test-pod", Namespace: "default"}, + } + fakeClient := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(pod).Build() + r := &ReconcileConfigMapSet{Client: fakeClient, scheme: scheme.Scheme} + + err := r.rebootSidecarsByCrr(pod, []string{"container1"}, "hash1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + // Verify CRR is created + crrList := &appsv1alpha1.ContainerRecreateRequestList{} + fakeClient.List(context.TODO(), crrList) + if len(crrList.Items) != 1 { + t.Fatalf("expected 1 CRR, got %d", len(crrList.Items)) + } + + crr := &crrList.Items[0] + if crr.Spec.Containers[0].Name != "container1" { + t.Errorf("expected container1, got %s", crr.Spec.Containers[0].Name) + } + + // Mark CRR as completed + crr.Status.Phase = appsv1alpha1.ContainerRecreateRequestCompleted + crr.Status.ContainerRecreateStates = []appsv1alpha1.ContainerRecreateRequestContainerRecreateState{ + {Name: "container1", Phase: appsv1alpha1.ContainerRecreateRequestSucceeded}, + } + fakeClient.Update(context.TODO(), crr) + + err = r.waitSidecarsRebootByCrrSuccess(context.TODO(), pod, []string{"container1"}, "hash1") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestReconcile(t *testing.T) { + cms := &appsv1alpha1.ConfigMapSet{ + ObjectMeta: metav1.ObjectMeta{Name: "test-cms", Namespace: "default"}, + Spec: appsv1alpha1.ConfigMapSetSpec{ + Data: map[string]string{"key": "val"}, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + }, + } + fakeClient := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(cms).Build() + r := &ReconcileConfigMapSet{Client: fakeClient, scheme: scheme.Scheme} + + req := reconcile.Request{ + NamespacedName: types.NamespacedName{Name: "test-cms", Namespace: "default"}, + } + res, err := r.Reconcile(context.TODO(), req) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if res.Requeue { + t.Errorf("expected no requeue") + } + + updatedCms := &appsv1alpha1.ConfigMapSet{} + err = fakeClient.Get(context.TODO(), req.NamespacedName, updatedCms) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(updatedCms.Finalizers) == 0 || updatedCms.Finalizers[0] != ConfigMapFinalizerName { + t.Errorf("expected finalizer to be added") + } + + hubName := GetConfigMapSetHubName("test-cms") + cm := &corev1.ConfigMap{} + err = fakeClient.Get(context.TODO(), types.NamespacedName{Name: hubName, Namespace: "default"}, cm) + if err != nil { + t.Fatalf("expected hub ConfigMap to be created, got error: %v", err) + } +} diff --git a/pkg/controller/controllers.go b/pkg/controller/controllers.go index 091434f535..ccd6f5cfb9 100644 --- a/pkg/controller/controllers.go +++ b/pkg/controller/controllers.go @@ -24,6 +24,7 @@ import ( "github.com/openkruise/kruise/pkg/controller/advancedcronjob" "github.com/openkruise/kruise/pkg/controller/broadcastjob" "github.com/openkruise/kruise/pkg/controller/cloneset" + "github.com/openkruise/kruise/pkg/controller/configmapset" containerlauchpriority "github.com/openkruise/kruise/pkg/controller/containerlaunchpriority" "github.com/openkruise/kruise/pkg/controller/containerrecreaterequest" "github.com/openkruise/kruise/pkg/controller/daemonset" @@ -50,6 +51,7 @@ func init() { controllerAddFuncs = append(controllerAddFuncs, advancedcronjob.Add) controllerAddFuncs = append(controllerAddFuncs, broadcastjob.Add) controllerAddFuncs = append(controllerAddFuncs, cloneset.Add) + controllerAddFuncs = append(controllerAddFuncs, configmapset.Add) controllerAddFuncs = append(controllerAddFuncs, containerrecreaterequest.Add) controllerAddFuncs = append(controllerAddFuncs, daemonset.Add) controllerAddFuncs = append(controllerAddFuncs, nodeimage.Add) diff --git a/pkg/webhook/add_configmapset.go b/pkg/webhook/add_configmapset.go new file mode 100644 index 0000000000..116b2c7ec3 --- /dev/null +++ b/pkg/webhook/add_configmapset.go @@ -0,0 +1,27 @@ +/* +Copyright 2020 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhook + +import ( + "github.com/openkruise/kruise/pkg/webhook/configmapset/mutating" + "github.com/openkruise/kruise/pkg/webhook/configmapset/validating" +) + +func init() { + addHandlers(mutating.HandlerGetterMap) + addHandlers(validating.HandlerGetterMap) +} diff --git a/pkg/webhook/configmapset/mutating/configmapset_create_update_handler.go b/pkg/webhook/configmapset/mutating/configmapset_create_update_handler.go new file mode 100644 index 0000000000..b80d288de4 --- /dev/null +++ b/pkg/webhook/configmapset/mutating/configmapset_create_update_handler.go @@ -0,0 +1,65 @@ +/* +Copyright 2019 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mutating + +import ( + "context" + "encoding/json" + admissionv1 "k8s.io/api/admission/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/klog/v2" + "net/http" + "reflect" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/openkruise/kruise/apis/apps/defaults" + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + "github.com/openkruise/kruise/pkg/util" +) + +// ConfigMapSetCreateUpdateHandler handles ConfigMapSet +type ConfigMapSetCreateUpdateHandler struct { + // Decoder decodes objects + Decoder admission.Decoder +} + +var _ admission.Handler = &ConfigMapSetCreateUpdateHandler{} + +// Handle handles admission requests. +func (h *ConfigMapSetCreateUpdateHandler) Handle(ctx context.Context, req admission.Request) admission.Response { + klog.Infof("configmapset %s/%s action %v", req.Namespace, req.Name, req.AdmissionRequest.Operation) + obj := &appsv1alpha1.ConfigMapSet{} + err := h.Decoder.Decode(req, obj) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + var copyObj runtime.Object = obj.DeepCopy() + switch req.AdmissionRequest.Operation { + case admissionv1.Create, admissionv1.Update: + klog.Infof("setting default for configmapset %s/%s", obj.Namespace, obj.Name) + defaults.SetDefaultsConfigMapSet(obj) + } + klog.V(4).Infof("configmapset after mutating: %v", util.DumpJSON(obj)) + if reflect.DeepEqual(obj, copyObj) { + return admission.Allowed("") + } + marshalled, err := json.Marshal(obj) + if err != nil { + return admission.Errored(http.StatusInternalServerError, err) + } + return admission.PatchResponseFromRaw(req.AdmissionRequest.Object.Raw, marshalled) +} diff --git a/pkg/webhook/configmapset/mutating/webhooks.go b/pkg/webhook/configmapset/mutating/webhooks.go new file mode 100644 index 0000000000..6feab7cd99 --- /dev/null +++ b/pkg/webhook/configmapset/mutating/webhooks.go @@ -0,0 +1,35 @@ +/* +Copyright 2019 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mutating + +import ( + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/openkruise/kruise/pkg/webhook/types" +) + +// +kubebuilder:webhook:path=/mutate-apps-kruise-io-v1alpha1-configmapset,mutating=true,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups=apps.kruise.io,resources=configmapsets,verbs=create;update,versions=v1alpha1,name=mconfigmapset.kb.io + +var ( + // HandlerGetterMap contains admission webhook handlers + HandlerGetterMap = map[string]types.HandlerGetter{ + "mutate-apps-kruise-io-v1alpha1-configmapset": func(mgr manager.Manager) admission.Handler { + return &ConfigMapSetCreateUpdateHandler{Decoder: admission.NewDecoder(mgr.GetScheme())} + }, + } +) diff --git a/pkg/webhook/configmapset/validating/configmapset_conflicts.go b/pkg/webhook/configmapset/validating/configmapset_conflicts.go new file mode 100644 index 0000000000..827215304c --- /dev/null +++ b/pkg/webhook/configmapset/validating/configmapset_conflicts.go @@ -0,0 +1,103 @@ +package validating + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + "github.com/openkruise/kruise/pkg/controller/configmapset" + "github.com/openkruise/kruise/pkg/util" +) + +func (h *ConfigMapSetCreateUpdateHandler) checkConflictsWithExistingConfigMapSets(ctx context.Context, obj *appsv1alpha1.ConfigMapSet) error { + cmsList := &appsv1alpha1.ConfigMapSetList{} + if err := h.Client.List(ctx, cmsList, client.InNamespace(obj.Namespace)); err != nil { + return fmt.Errorf("failed to list existing ConfigMapSets: %v", err) + } + + objSidecarName, err := h.getReloadSidecarName(ctx, obj) + if err != nil { + return fmt.Errorf("failed to get reload-sidecar name for current ConfigMapSet: %v", err) + } + + for _, existingCMS := range cmsList.Items { + if existingCMS.Name == obj.Name { + continue + } + + if obj.Spec.Selector != nil && existingCMS.Spec.Selector != nil { + if !util.IsSelectorOverlapping(obj.Spec.Selector, existingCMS.Spec.Selector) { + continue + } + } + + // Check for reload-sidecar name conflict + existingSidecarName, err := h.getReloadSidecarName(ctx, &existingCMS) + if err == nil && objSidecarName != "" && existingSidecarName != "" && objSidecarName == existingSidecarName { + return fmt.Errorf("reload-sidecar name conflict: both %s and %s use the same sidecar name '%s' and their selectors overlap", obj.Name, existingCMS.Name, objSidecarName) + } + + // Check for container mount path conflict + for _, objC := range obj.Spec.Containers { + for _, existC := range existingCMS.Spec.Containers { + if objC.MountPath == existC.MountPath { + if objC.Name != "" && objC.Name == existC.Name { + return fmt.Errorf("container mount path conflict: both %s and %s mount to '%s' on container '%s'", obj.Name, existingCMS.Name, objC.MountPath, objC.Name) + } + if objC.NameFrom != nil && existC.NameFrom != nil && reflect.DeepEqual(objC.NameFrom, existC.NameFrom) { + return fmt.Errorf("container mount path conflict: both %s and %s mount to '%s' using the same nameFrom reference", obj.Name, existingCMS.Name, objC.MountPath) + } + } + } + } + } + + return nil +} + +func (h *ConfigMapSetCreateUpdateHandler) getReloadSidecarName(ctx context.Context, cms *appsv1alpha1.ConfigMapSet) (string, error) { + if cms.Spec.ReloadSidecarConfig == nil { + return configmapset.GetConfigMapSetDefaultSidecarName(cms.Name), nil + } + config := cms.Spec.ReloadSidecarConfig.Config + if config == nil { + return configmapset.GetConfigMapSetDefaultSidecarName(cms.Name), nil + } + + switch cms.Spec.ReloadSidecarConfig.Type { + case appsv1alpha1.ReloadSidecarTypeK8s: + return config.Name, nil + case appsv1alpha1.ReloadSidecarTypeSidecarSet: + if config.SidecarSetRef != nil { + return config.SidecarSetRef.ContainerName, nil + } + case appsv1alpha1.ReloadSidecarTypeCustom: + if config.ConfigMapRef != nil { + cmNamespace := config.ConfigMapRef.Namespace + if cmNamespace == "" { + cmNamespace = cms.Namespace + } + customerCM := &corev1.ConfigMap{} + err := h.Client.Get(ctx, types.NamespacedName{Name: config.ConfigMapRef.Name, Namespace: cmNamespace}, customerCM) + if err != nil { + return "", err + } + if containerData, exists := customerCM.Data["reload-sidecar"]; exists { + var container corev1.Container + if err := json.Unmarshal([]byte(containerData), &container); err == nil { + if container.Name != "" { + return container.Name, nil + } + return "reload-sidecar", nil + } + } + } + } + return "", nil +} diff --git a/pkg/webhook/configmapset/validating/configmapset_create_update_handler.go b/pkg/webhook/configmapset/validating/configmapset_create_update_handler.go new file mode 100644 index 0000000000..b3b83f2eca --- /dev/null +++ b/pkg/webhook/configmapset/validating/configmapset_create_update_handler.go @@ -0,0 +1,462 @@ +/* +Copyright 2019 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validating + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "reflect" + "regexp" + "strconv" + "strings" + + "k8s.io/kubernetes/pkg/controller" + + admissionv1 "k8s.io/api/admission/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + genericvalidation "k8s.io/apimachinery/pkg/api/validation" + validationutil "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/apimachinery/pkg/util/validation/field" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/openkruise/kruise/pkg/controller/configmapset" + "github.com/openkruise/kruise/pkg/util" + + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" +) + +const ( + configMapSetNameMaxLen = 63 +) + +var ( + validateConfigMapSetNameMsg = "ConfigMapSet name must consist of alphanumeric characters or '-'" + validateConfigMapSetNameRegex = regexp.MustCompile(validConfigMapSetNameFmt) + validConfigMapSetNameFmt = `^[a-zA-Z0-9\-]+$` +) + +// ConfigMapSetCreateUpdateHandler handles ConfigMapSet +type ConfigMapSetCreateUpdateHandler struct { + Client client.Client + // Decoder decodes objects + Decoder admission.Decoder +} + +func (h *ConfigMapSetCreateUpdateHandler) validateConfigMapSetSpec(ctx context.Context, name, namespace string, spec *appsv1alpha1.ConfigMapSetSpec, fldPath *field.Path) *field.Error { + if spec == nil { + return field.Required(fldPath.Child("spec"), "") + } + if spec.Selector == nil { + return field.Required(fldPath.Child("selector"), "selector is required") + } + + if len(spec.Selector.MatchLabels) == 0 { + return field.Required(fldPath.Child("selector", "matchLabels"), "") + } + + if spec.Data == nil { + return field.Required(fldPath.Child("data"), "data is required") + } + + if len(spec.UpdateStrategy.MatchLabelKeys) > 0 { + for _, key := range spec.UpdateStrategy.MatchLabelKeys { + if _, exists := spec.Selector.MatchLabels[key]; exists { + return field.Invalid(fldPath.Child("updateStrategy", "matchLabelKeys"), spec.UpdateStrategy.MatchLabelKeys, fmt.Sprintf("matchLabelKeys cannot intersect with selector matchLabels, conflict key: %s", key)) + } + } + } + + containerNames := make(map[string]struct{}) + for i, c := range spec.Containers { + if c.Name != "" { + if _, exists := containerNames[c.Name]; exists { + return field.Duplicate(fldPath.Child("containers").Index(i).Child("name"), c.Name) + } + containerNames[c.Name] = struct{}{} + } + } + + // Calculate Hash of current Spec.Data + hash, err := configmapset.CalculateHash(spec.Data) + if err != nil { + return field.InternalError(fldPath.Child("data"), fmt.Errorf("failed to compute hash: %v", err)) + } + // Check if the same revision / customVersion already exists + // Re-fetch the latest ConfigMap to avoid concurrent conflicts + cmName := configmapset.GetConfigMapSetHubName(name) + cmNamespace := namespace + cm := &corev1.ConfigMap{} + if err = h.Client.Get(context.TODO(), types.NamespacedName{Name: cmName, Namespace: cmNamespace}, cm); err != nil { + if !errors.IsNotFound(err) { + return field.InternalError(fldPath.Child("data"), fmt.Errorf("failed to get ConfigMap: %v", err)) + } + cm = nil + } + + if cm != nil { + var revisions []configmapset.RevisionEntry + if revData, exists := cm.Data["revisions"]; exists { + if err := json.Unmarshal([]byte(revData), &revisions); err != nil { + klog.Errorf("Failed to unmarshal revisions from ConfigMap %s: %v, resetting revisions", cmName, err) + return field.InternalError(fldPath.Child("data"), fmt.Errorf("failed to unmarshal revisions from ConfigMap: %v", err)) + } + } + + for _, rev := range revisions { + // hash must same if customVersion same + if rev.CustomVersion == spec.CustomVersion && rev.Hash != hash { + return field.Invalid(fldPath.Child("customVersion"), spec.CustomVersion, "configmapset already exists with hash "+rev.Hash+" for customVersion "+spec.CustomVersion) + } + } + + // RevisionHistoryLimit validation: if new hash is not in existing revisions, check if limit exceeded + isExistingRevision := false + for _, rev := range revisions { + if rev.Hash == hash { + isExistingRevision = true + break + } + } + if !isExistingRevision && spec.RevisionHistoryLimit != nil && int32(len(revisions)) >= *spec.RevisionHistoryLimit { + // Need to evict the oldest revision, check if any Pod is using it + tempCMS := &appsv1alpha1.ConfigMapSet{Spec: *spec} + tempCMS.Name = name + tempCMS.Namespace = namespace + pods, podErr := configmapset.GetMatchedPods(ctx, h.Client, tempCMS) + if podErr != nil { + // Construct temporary object for querying matched Pods + return field.InternalError(fldPath.Child("revisionHistoryLimit"), fmt.Errorf("failed to get matched pods: %v", podErr)) + } + revisionsInUse := make(map[string]bool) + currentRevisionKey := configmapset.GetConfigMapSetCurrentRevisionKey(name) + for _, pod := range pods { + if pod.Annotations != nil && pod.Annotations[currentRevisionKey] != "" { + revisionsInUse[pod.Annotations[currentRevisionKey]] = true + } + } + // Check from the oldest whether it can be evicted + keep := int(*spec.RevisionHistoryLimit) + excessCount := len(revisions) - keep + 1 // +1 because we're adding a new one + removable := 0 + for _, rev := range revisions { + if !revisionsInUse[rev.Hash] { + removable++ + } + } + if removable < excessCount { + return field.Forbidden(fldPath.Child("revisionHistoryLimit"), fmt.Sprintf("cannot add new revision: revisionHistoryLimit is %d, current revisions count is %d, and oldest revisions are still in use by pods", *spec.RevisionHistoryLimit, len(revisions))) + } + } + } + + strategy := spec.UpdateStrategy + if strategy.Partition != nil && !validatePartition(strategy.Partition) { + return field.Invalid(fldPath.Child("updateStrategy"), + spec.UpdateStrategy, + "invalid partition value") + } + + if strategy.MaxUnavailable != nil && !validateMaxUnavailable(strategy.MaxUnavailable) { + return field.Invalid(fldPath.Child("updateStrategy"), + spec.UpdateStrategy, + "invalid maxUnavailable value") + } + + // Validate EffectPolicy + if spec.EffectPolicy != nil { + effectPath := fldPath.Child("effectPolicy") + switch spec.EffectPolicy.Type { + case appsv1alpha1.EffectPolicyTypeReStart, appsv1alpha1.EffectPolicyTypeHotUpdate: + // Valid types that don't strictly require PostHook config + case appsv1alpha1.EffectPolicyTypePostHook: + if spec.EffectPolicy.PostHook == nil { + return field.Required(effectPath.Child("postHook"), "postHook must be specified when type is PostHook") + } + if len(spec.EffectPolicy.PostHook.HTTPGet) == 0 && len(spec.EffectPolicy.PostHook.TCPSocket) == 0 { + return field.Required(effectPath.Child("postHook"), "either httpGet or tcpSocket must be specified for PostHook") + } + if len(spec.EffectPolicy.PostHook.HTTPGet) > 0 && len(spec.EffectPolicy.PostHook.TCPSocket) > 0 { + return field.Invalid(effectPath.Child("postHook"), "", "cannot specify both httpGet and tcpSocket") + } + default: + return field.Invalid(effectPath.Child("type"), spec.EffectPolicy.Type, "invalid effect policy type") + } + } + + // validate ReloadSidecarConfig + if spec.ReloadSidecarConfig != nil { + reloadConfigPath := fldPath.Child("reloadSidecarConfig") + if spec.ReloadSidecarConfig.Type == appsv1alpha1.ReloadSidecarTypeSidecarSet { + var sidecarName string + var sidecarContainerName string + if spec.ReloadSidecarConfig.Config == nil || spec.ReloadSidecarConfig.Config.SidecarSetRef == nil { + sidecarName, sidecarContainerName = configmapset.GetDefaultCmsSidecarSet() + } else { + sidecarName = spec.ReloadSidecarConfig.Config.SidecarSetRef.Name + sidecarContainerName = spec.ReloadSidecarConfig.Config.SidecarSetRef.ContainerName + } + sidecarSet := &appsv1alpha1.SidecarSet{} + err = h.Client.Get(ctx, types.NamespacedName{Name: sidecarName}, sidecarSet) + if err != nil { + if errors.IsNotFound(err) { + return field.Invalid(reloadConfigPath.Child("config", "sidecarSetRef", "name"), sidecarName, "SidecarSet not found") + } + return field.InternalError(reloadConfigPath.Child("config", "sidecarSetRef", "name"), fmt.Errorf("failed to get SidecarSet: %v", err)) + } + containerFound := false + for _, c := range sidecarSet.Spec.Containers { + if c.Name == sidecarContainerName && c.Image != "" { + containerFound = true + break + } + } + if !containerFound { + return field.Invalid(reloadConfigPath.Child("config", "sidecarSetRef", "containerName"), sidecarContainerName, "container not found in SidecarSet") + } + + // Validate if SidecarSet includes the current ConfigMapSet's Pods + if sidecarSet.Spec.Namespace != "" && sidecarSet.Spec.Namespace != namespace { + return field.Invalid(reloadConfigPath.Child("config", "sidecarSetRef", "name"), sidecarName, fmt.Sprintf("SidecarSet targets namespace %s which does not match ConfigMapSet namespace %s", sidecarSet.Spec.Namespace, namespace)) + } + + // Validate NamespaceSelector + if sidecarSet.Spec.NamespaceSelector != nil { + nsObj := &corev1.Namespace{} + err = h.Client.Get(ctx, types.NamespacedName{Name: namespace}, nsObj) + if err != nil { + if errors.IsNotFound(err) { + return field.Invalid(field.NewPath("metadata", "namespace"), namespace, "Namespace not found") + } + return field.InternalError(field.NewPath("metadata", "namespace"), fmt.Errorf("failed to get namespace: %v", err)) + } + selector, err := util.ValidatedLabelSelectorAsSelector(sidecarSet.Spec.NamespaceSelector) + if err != nil { + return field.Invalid(reloadConfigPath.Child("config", "sidecarSetRef", "name"), sidecarName, fmt.Sprintf("invalid NamespaceSelector in SidecarSet: %v", err)) + } + if !selector.Matches(labels.Set(nsObj.Labels)) { + return field.Invalid(reloadConfigPath.Child("config", "sidecarSetRef", "name"), sidecarName, fmt.Sprintf("SidecarSet NamespaceSelector does not match ConfigMapSet namespace %s", namespace)) + } + } + } else if spec.ReloadSidecarConfig.Type == appsv1alpha1.ReloadSidecarTypeCustom { + var namespacedName types.NamespacedName + if spec.ReloadSidecarConfig.Config == nil || spec.ReloadSidecarConfig.Config.ConfigMapRef == nil { + namespacedName = configmapset.GetDefaultCmsConfigMap() + } else { + namespacedName = types.NamespacedName{ + Name: spec.ReloadSidecarConfig.Config.ConfigMapRef.Name, + Namespace: spec.ReloadSidecarConfig.Config.ConfigMapRef.Namespace, + } + } + customerCM := &corev1.ConfigMap{} + // Use the ConfigMapSet's namespace if the ConfigMapRef namespace is not specified + cmRefNamespace := namespacedName.Namespace + if cmRefNamespace == "" { + cmRefNamespace = namespace + } + err = h.Client.Get(ctx, types.NamespacedName{Name: namespacedName.Name, Namespace: cmRefNamespace}, customerCM) + if err != nil { + if errors.IsNotFound(err) { + return field.Invalid(reloadConfigPath.Child("config", "configMapRef", "name"), namespacedName.Name, "custom sidecar ConfigMap not found") + } + return field.InternalError(reloadConfigPath.Child("config", "configMapRef", "name"), fmt.Errorf("failed to get custom sidecar ConfigMap: %v", err)) + } + + // Validate if the specific key "reload-sidecar" exists and can be unmarshaled into a valid Container + containerData, exists := customerCM.Data["reload-sidecar"] + if !exists { + return field.Invalid(reloadConfigPath.Child("config", "configMapRef", "name"), namespacedName.Name, "custom sidecar ConfigMap must contain key 'reload-sidecar'") + } + + var reloadSidecar corev1.Container + if err = json.Unmarshal([]byte(containerData), &reloadSidecar); err != nil { + return field.Invalid(reloadConfigPath.Child("config", "configMapRef", "name"), namespacedName.Name, fmt.Sprintf("failed to unmarshal 'reload-sidecar' data to Container: %v", err)) + } + } + } + return nil +} + +var percentagePattern = regexp.MustCompile(`^\d+%$`) + +func validatePartition(partition *intstr.IntOrString) bool { + if partition == nil { + return false + } + + switch partition.Type { + case intstr.Int: + return partition.IntVal >= 0 + + case intstr.String: + if !percentagePattern.MatchString(partition.StrVal) { + return false + } + // Remove % and verify if it is an integer between 0-100 + valStr := partition.StrVal[:len(partition.StrVal)-1] + val, err := strconv.Atoi(valStr) + return err == nil && val >= 0 && val <= 100 + default: + return false + } +} + +// Verify if maxUnavailable is a percentage between 1~100, or a positive integer (>0) +func validateMaxUnavailable(maxUnavailable *intstr.IntOrString) bool { + if maxUnavailable == nil { + return false + } + + switch maxUnavailable.Type { + case intstr.Int: + if maxUnavailable.IntVal > 0 { + return true + } + case intstr.String: + s := strings.TrimSpace(maxUnavailable.StrVal) + if strings.HasSuffix(s, "%") { + numStr := strings.TrimSuffix(s, "%") + num, err := strconv.Atoi(numStr) + if err != nil || num <= 0 || num > 100 { + return false + } + return true + } + // Non-percentage case, parse as integer + num, err := strconv.Atoi(s) + if err != nil || num <= 0 { + return false + } + return true + } + return false +} + +func validateConfigMapSetName(name string, prefix bool) (allErrs []string) { + if !validateConfigMapSetNameRegex.MatchString(name) { + allErrs = append(allErrs, validationutil.RegexError(validateConfigMapSetNameMsg, validConfigMapSetNameFmt, "example-com")) + } + if len(name) > configMapSetNameMaxLen { + allErrs = append(allErrs, validationutil.MaxLenError(configMapSetNameMaxLen)) + } + return allErrs +} + +var _ admission.Handler = &ConfigMapSetCreateUpdateHandler{} + +// Handle handles admission requests. +func (h *ConfigMapSetCreateUpdateHandler) validateConfigMapSet(ctx context.Context, configMapSet *appsv1alpha1.ConfigMapSet) field.ErrorList { + metaErrs := genericvalidation.ValidateObjectMeta(&configMapSet.ObjectMeta, true, validateConfigMapSetName, field.NewPath("metadata")) + if len(metaErrs) > 0 { + return metaErrs + } + specErr := h.validateConfigMapSetSpec(ctx, configMapSet.Name, configMapSet.Namespace, &configMapSet.Spec, field.NewPath("spec")) + if specErr != nil { + return field.ErrorList{specErr} + } + return nil +} + +// Handle handles admission requests. +func (h *ConfigMapSetCreateUpdateHandler) Handle(ctx context.Context, req admission.Request) admission.Response { + obj := &appsv1alpha1.ConfigMapSet{} + oldObj := &appsv1alpha1.ConfigMapSet{} + + switch req.AdmissionRequest.Operation { + case admissionv1.Create, admissionv1.Update: + err := h.Decoder.Decode(req, obj) + if err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + if req.AdmissionRequest.Operation == admissionv1.Update { + if len(req.OldObject.Raw) > 0 { + if err := h.Decoder.DecodeRaw(req.AdmissionRequest.OldObject, oldObj); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + if oldObj.Spec.Selector != nil && obj.Spec.Selector != nil { + if !reflect.DeepEqual(oldObj.Spec.Selector.MatchLabels, obj.Spec.Selector.MatchLabels) { + errList := field.ErrorList{field.Forbidden(field.NewPath("spec", "selector", "matchLabels"), "field is immutable")} + return admission.Errored(http.StatusUnprocessableEntity, errList.ToAggregate()) + } + } + } + } + + if allErrs := h.validateConfigMapSet(ctx, obj); len(allErrs) > 0 { + return admission.Errored(http.StatusUnprocessableEntity, allErrs.ToAggregate()) + } + + // Layer 2: Defensive check against existing ConfigMapSets + if err := h.checkConflictsWithExistingConfigMapSets(ctx, obj); err != nil { + return admission.Errored(http.StatusConflict, err) + } + case admissionv1.Delete: + if len(req.OldObject.Raw) == 0 { + klog.InfoS("Skip to validate CloneSet %s/%s deletion for no old object, maybe because of Kubernetes version < 1.16", "namespace", req.Namespace, "name", req.Name) + return admission.ValidationResponse(true, "") + } + if err := h.Decoder.DecodeRaw(req.AdmissionRequest.OldObject, oldObj); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + err := h.validateDeleteConfigMapSet(ctx, oldObj) + if err != nil { + return admission.Errored(http.StatusForbidden, err) + } + } + + return admission.ValidationResponse(true, "") +} + +func (h *ConfigMapSetCreateUpdateHandler) validateDeleteConfigMapSet(ctx context.Context, cms *appsv1alpha1.ConfigMapSet) error { + selector := cms.Spec.Selector + if selector == nil { + return nil + } + matchLabels := selector.MatchLabels + podList := &corev1.PodList{} + err := h.Client.List(ctx, podList, client.InNamespace(cms.Namespace), client.MatchingLabels(matchLabels)) + if err != nil { + return err + } + if len(podList.Items) == 0 { + return nil + } + for _, pod := range podList.Items { + if !controller.IsPodActive(&pod) { + continue + } + + if pod.Annotations == nil { + continue + } + + if pod.Annotations[configmapset.GetConfigMapSetEnabledKey()] == "true" { + return fmt.Errorf("pod %s/%s is still used configmapSet:%s/%s", pod.Namespace, pod.Name, cms.Namespace, cms.Name) + } + } + return nil +} diff --git a/pkg/webhook/configmapset/validating/configmapset_create_update_handler_test.go b/pkg/webhook/configmapset/validating/configmapset_create_update_handler_test.go new file mode 100644 index 0000000000..06cb6eadf1 --- /dev/null +++ b/pkg/webhook/configmapset/validating/configmapset_create_update_handler_test.go @@ -0,0 +1,563 @@ +package validating + +import ( + "context" + "encoding/json" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + "github.com/openkruise/kruise/pkg/controller/configmapset" +) + +func init() { + _ = appsv1alpha1.AddToScheme(scheme.Scheme) +} + +func TestValidatePartition(t *testing.T) { + cases := []struct { + name string + partition *intstr.IntOrString + expected bool + }{ + {"nil partition", nil, false}, + {"valid int partition", &intstr.IntOrString{Type: intstr.Int, IntVal: 1}, true}, + {"zero int partition", &intstr.IntOrString{Type: intstr.Int, IntVal: 0}, true}, + {"negative int partition", &intstr.IntOrString{Type: intstr.Int, IntVal: -1}, false}, + {"valid string partition", &intstr.IntOrString{Type: intstr.String, StrVal: "10%"}, true}, + {"zero string partition", &intstr.IntOrString{Type: intstr.String, StrVal: "0%"}, true}, + {"100 string partition", &intstr.IntOrString{Type: intstr.String, StrVal: "100%"}, true}, + {"invalid string partition - negative", &intstr.IntOrString{Type: intstr.String, StrVal: "-10%"}, false}, + {"invalid string partition - over 100", &intstr.IntOrString{Type: intstr.String, StrVal: "110%"}, false}, + {"invalid string partition - no percent", &intstr.IntOrString{Type: intstr.String, StrVal: "10"}, false}, + {"invalid string partition - non numeric", &intstr.IntOrString{Type: intstr.String, StrVal: "abc%"}, false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + result := validatePartition(tc.partition) + if result != tc.expected { + t.Errorf("expected %v, got %v for partition %v", tc.expected, result, tc.partition) + } + }) + } +} + +func TestValidateMaxUnavailable(t *testing.T) { + cases := []struct { + name string + maxUnavailable *intstr.IntOrString + expected bool + }{ + {"nil maxUnavailable", nil, false}, + {"valid int maxUnavailable", &intstr.IntOrString{Type: intstr.Int, IntVal: 1}, true}, + {"zero int maxUnavailable", &intstr.IntOrString{Type: intstr.Int, IntVal: 0}, false}, // maxUnavailable > 0 required + {"negative int maxUnavailable", &intstr.IntOrString{Type: intstr.Int, IntVal: -1}, false}, + {"valid string percent", &intstr.IntOrString{Type: intstr.String, StrVal: "10%"}, true}, + {"zero string percent", &intstr.IntOrString{Type: intstr.String, StrVal: "0%"}, false}, + {"100 string percent", &intstr.IntOrString{Type: intstr.String, StrVal: "100%"}, true}, + {"invalid string percent - over 100", &intstr.IntOrString{Type: intstr.String, StrVal: "110%"}, false}, + {"valid string int", &intstr.IntOrString{Type: intstr.String, StrVal: "10"}, true}, + {"zero string int", &intstr.IntOrString{Type: intstr.String, StrVal: "0"}, false}, + {"negative string int", &intstr.IntOrString{Type: intstr.String, StrVal: "-1"}, false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + result := validateMaxUnavailable(tc.maxUnavailable) + if result != tc.expected { + t.Errorf("expected %v, got %v for maxUnavailable %v", tc.expected, result, tc.maxUnavailable) + } + }) + } +} + +func TestValidateDeleteConfigMapSet(t *testing.T) { + podActive := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-active", + Namespace: "default", + Labels: map[string]string{"app": "test"}, + Annotations: map[string]string{ + configmapset.GetConfigMapSetEnabledKey(): "true", + }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + } + + podInactive := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-inactive", + Namespace: "default", + Labels: map[string]string{"app": "test"}, + Annotations: map[string]string{ + configmapset.GetConfigMapSetEnabledKey(): "true", + }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodSucceeded, // inactive + }, + } + + podNoLabel := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-no-label", + Namespace: "default", + Labels: map[string]string{"app": "test"}, + // No configmapset enabled annotation + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + } + + cases := []struct { + name string + pods []client.Object + expectError bool + }{ + { + name: "no pods", + pods: []client.Object{}, + expectError: false, + }, + { + name: "active pod using cms", + pods: []client.Object{podActive}, + expectError: true, + }, + { + name: "inactive pod using cms", + pods: []client.Object{podInactive}, + expectError: false, + }, + { + name: "active pod not using cms", + pods: []client.Object{podNoLabel}, + expectError: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fakeClient := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(tc.pods...).Build() + handler := &ConfigMapSetCreateUpdateHandler{ + Client: fakeClient, + } + cms := &appsv1alpha1.ConfigMapSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cms", + Namespace: "default", + }, + Spec: appsv1alpha1.ConfigMapSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + }, + } + err := handler.validateDeleteConfigMapSet(context.TODO(), cms) + if (err != nil) != tc.expectError { + t.Errorf("expected error: %v, got: %v", tc.expectError, err) + } + }) + } +} + +func TestValidateConfigMapSetSpec(t *testing.T) { + validPartition := intstr.FromInt(1) + validMaxUnavailable := intstr.FromInt(1) + zeroMaxUnavailable := intstr.FromInt(0) + invalidStringMaxUnavailable := intstr.FromString("invalid") + + cases := []struct { + name string + spec *appsv1alpha1.ConfigMapSetSpec + clientObjects []client.Object + expectErr bool + errField string + }{ + { + name: "valid spec", + spec: &appsv1alpha1.ConfigMapSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + Data: map[string]string{"config": "value"}, + UpdateStrategy: &appsv1alpha1.ConfigMapSetUpdateStrategy{ + Partition: &validPartition, + MaxUnavailable: &validMaxUnavailable, + }, + }, + expectErr: false, + }, + { + name: "missing selector", + spec: &appsv1alpha1.ConfigMapSetSpec{ + Data: map[string]string{"config": "value"}, + UpdateStrategy: &appsv1alpha1.ConfigMapSetUpdateStrategy{ + Partition: &validPartition, + MaxUnavailable: &validMaxUnavailable, + }, + }, + expectErr: true, + errField: "spec.selector", + }, + { + name: "missing data", + spec: &appsv1alpha1.ConfigMapSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + UpdateStrategy: &appsv1alpha1.ConfigMapSetUpdateStrategy{ + Partition: &validPartition, + MaxUnavailable: &validMaxUnavailable, + }, + }, + expectErr: true, + errField: "spec.data", + }, + { + name: "intersecting matchLabelKeys", + spec: &appsv1alpha1.ConfigMapSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test", "env": "prod"}, + }, + Data: map[string]string{"config": "value"}, + UpdateStrategy: &appsv1alpha1.ConfigMapSetUpdateStrategy{ + Partition: &validPartition, + MaxUnavailable: &validMaxUnavailable, + MatchLabelKeys: []string{"env"}, + }, + }, + expectErr: true, + errField: "spec.updateStrategy.matchLabelKeys", + }, + { + name: "invalid partition - negative int", + spec: &appsv1alpha1.ConfigMapSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + Data: map[string]string{"config": "value"}, + UpdateStrategy: &appsv1alpha1.ConfigMapSetUpdateStrategy{ + Partition: &intstr.IntOrString{Type: intstr.Int, IntVal: -1}, + MaxUnavailable: &validMaxUnavailable, + }, + }, + expectErr: true, + errField: "spec.updateStrategy", + }, + { + name: "invalid maxUnavailable - zero", + spec: &appsv1alpha1.ConfigMapSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + Data: map[string]string{"config": "value"}, + UpdateStrategy: &appsv1alpha1.ConfigMapSetUpdateStrategy{ + Partition: &validPartition, + MaxUnavailable: &zeroMaxUnavailable, + }, + }, + expectErr: true, + errField: "spec.updateStrategy", + }, + { + name: "invalid maxUnavailable - string format", + spec: &appsv1alpha1.ConfigMapSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + Data: map[string]string{"config": "value"}, + UpdateStrategy: &appsv1alpha1.ConfigMapSetUpdateStrategy{ + Partition: &validPartition, + MaxUnavailable: &invalidStringMaxUnavailable, + }, + }, + expectErr: true, + errField: "spec.updateStrategy", + }, + { + name: "duplicate container names", + spec: &appsv1alpha1.ConfigMapSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + Data: map[string]string{"config": "value"}, + Containers: []appsv1alpha1.ConfigMapSetContainer{ + {Name: "app-container"}, + {Name: "app-container"}, + }, + UpdateStrategy: &appsv1alpha1.ConfigMapSetUpdateStrategy{ + Partition: &validPartition, + MaxUnavailable: &validMaxUnavailable, + }, + }, + expectErr: true, + errField: "spec.containers[1].name", + }, + { + name: "invalid effect policy - posthook missing config", + spec: &appsv1alpha1.ConfigMapSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + Data: map[string]string{"config": "value"}, + EffectPolicy: &appsv1alpha1.EffectPolicy{ + Type: appsv1alpha1.EffectPolicyTypePostHook, + }, + UpdateStrategy: &appsv1alpha1.ConfigMapSetUpdateStrategy{ + Partition: &validPartition, + MaxUnavailable: &validMaxUnavailable, + }, + }, + expectErr: true, + errField: "spec.effectPolicy.postHook", + }, + { + name: "invalid effect policy - both http and tcp", + spec: &appsv1alpha1.ConfigMapSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + Data: map[string]string{"config": "value"}, + EffectPolicy: &appsv1alpha1.EffectPolicy{ + Type: appsv1alpha1.EffectPolicyTypePostHook, + PostHook: &appsv1alpha1.PostHookConfig{ + HTTPGet: []*corev1.HTTPGetAction{{}}, + TCPSocket: []*corev1.TCPSocketAction{{}}, + }, + }, + UpdateStrategy: &appsv1alpha1.ConfigMapSetUpdateStrategy{ + Partition: &validPartition, + MaxUnavailable: &validMaxUnavailable, + }, + }, + expectErr: true, + errField: "spec.effectPolicy.postHook", + }, + { + name: "valid sidecarset ref", + spec: &appsv1alpha1.ConfigMapSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + Data: map[string]string{"config": "value"}, + ReloadSidecarConfig: &appsv1alpha1.ReloadSidecarConfig{ + Type: appsv1alpha1.ReloadSidecarTypeSidecarSet, + Config: &appsv1alpha1.ReloadSidecarConfigData{ + SidecarSetRef: &appsv1alpha1.SidecarSetRef{ + Name: "test-sidecarset", + ContainerName: "reload-container", + }, + }, + }, + UpdateStrategy: &appsv1alpha1.ConfigMapSetUpdateStrategy{ + Partition: &validPartition, + MaxUnavailable: &validMaxUnavailable, + }, + }, + clientObjects: []client.Object{ + &appsv1alpha1.SidecarSet{ + ObjectMeta: metav1.ObjectMeta{Name: "test-sidecarset"}, + Spec: appsv1alpha1.SidecarSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, // matching + }, + Containers: []appsv1alpha1.SidecarContainer{ + { + Container: corev1.Container{Name: "reload-container", Image: "image:v1"}, + }, + }, + }, + }, + }, + expectErr: false, + }, + { + name: "invalid custom configmap - missing reload-sidecar key", + spec: &appsv1alpha1.ConfigMapSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + Data: map[string]string{"config": "value"}, + ReloadSidecarConfig: &appsv1alpha1.ReloadSidecarConfig{ + Type: appsv1alpha1.ReloadSidecarTypeCustom, + Config: &appsv1alpha1.ReloadSidecarConfigData{ + ConfigMapRef: &appsv1alpha1.ConfigMapRef{ + Name: "test-cm", + Namespace: "default", + }, + }, + }, + UpdateStrategy: &appsv1alpha1.ConfigMapSetUpdateStrategy{ + Partition: &validPartition, + MaxUnavailable: &validMaxUnavailable, + }, + }, + clientObjects: []client.Object{ + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "test-cm", Namespace: "default"}, + Data: map[string]string{"wrong-key": "{}"}, + }, + }, + expectErr: true, + errField: "spec.reloadSidecarConfig.config.configMapRef.name", + }, + { + name: "invalid custom configmap - not exist", + spec: &appsv1alpha1.ConfigMapSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + Data: map[string]string{"config": "value"}, + ReloadSidecarConfig: &appsv1alpha1.ReloadSidecarConfig{ + Type: appsv1alpha1.ReloadSidecarTypeCustom, + Config: &appsv1alpha1.ReloadSidecarConfigData{ + ConfigMapRef: &appsv1alpha1.ConfigMapRef{ + Name: "not-exist-cm", + Namespace: "default", + }, + }, + }, + UpdateStrategy: &appsv1alpha1.ConfigMapSetUpdateStrategy{ + Partition: &validPartition, + MaxUnavailable: &validMaxUnavailable, + }, + }, + expectErr: true, + errField: "spec.reloadSidecarConfig.config.configMapRef.name", + }, + { + name: "invalid sidecarset ref - not exist", + spec: &appsv1alpha1.ConfigMapSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + Data: map[string]string{"config": "value"}, + ReloadSidecarConfig: &appsv1alpha1.ReloadSidecarConfig{ + Type: appsv1alpha1.ReloadSidecarTypeSidecarSet, + Config: &appsv1alpha1.ReloadSidecarConfigData{ + SidecarSetRef: &appsv1alpha1.SidecarSetRef{ + Name: "not-exist-sidecarset", + }, + }, + }, + UpdateStrategy: &appsv1alpha1.ConfigMapSetUpdateStrategy{ + Partition: &validPartition, + MaxUnavailable: &validMaxUnavailable, + }, + }, + expectErr: true, + errField: "spec.reloadSidecarConfig.config.sidecarSetRef.name", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fakeClient := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(tc.clientObjects...).Build() + handler := &ConfigMapSetCreateUpdateHandler{ + Client: fakeClient, + } + err := handler.validateConfigMapSetSpec(context.TODO(), "test-cms", "default", tc.spec, field.NewPath("spec")) + if tc.expectErr { + if err == nil { + t.Fatalf("expected error on field %s but got none", tc.errField) + } + if err.Field != tc.errField { + t.Errorf("expected error on field %s, but got error: %v", tc.errField, err) + } + } else { + if err != nil { + t.Errorf("expected no error, but got: %v", err) + } + } + }) + } +} + +func TestRevisionHistoryLimitValidation(t *testing.T) { + // Construct a scenario where the Limit is reached + revisions := []configmapset.RevisionEntry{ + {Hash: "hash1", CustomVersion: "v1"}, + {Hash: "hash2", CustomVersion: "v2"}, + } + revBytes, _ := json.Marshal(revisions) + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configmapset.GetConfigMapSetHubName("test-cms"), + Namespace: "default", + }, + Data: map[string]string{ + "revisions": string(revBytes), + }, + } + + // Construct a Pod that is using the oldest revision "hash1" + pod1 := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod-1", + Namespace: "default", + Labels: map[string]string{"app": "test"}, + Annotations: map[string]string{ + configmapset.GetConfigMapSetCurrentRevisionKey("test-cms"): "hash1", + }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + } + + pod2 := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod-2", + Namespace: "default", + Labels: map[string]string{"app": "test"}, + Annotations: map[string]string{ + configmapset.GetConfigMapSetCurrentRevisionKey("test-cms"): "hash2", + }, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + }, + } + + fakeClient := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(cm, pod1, pod2).Build() + handler := &ConfigMapSetCreateUpdateHandler{ + Client: fakeClient, + } + + // Try to publish a new version hash3 + spec := &appsv1alpha1.ConfigMapSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + Data: map[string]string{"new": "data"}, + RevisionHistoryLimit: pointer.Int32(2), + UpdateStrategy: &appsv1alpha1.ConfigMapSetUpdateStrategy{ + Partition: &intstr.IntOrString{Type: intstr.Int, IntVal: 0}, + MaxUnavailable: &intstr.IntOrString{Type: intstr.Int, IntVal: 1}, + }, + } + + err := handler.validateConfigMapSetSpec(context.TODO(), "test-cms", "default", spec, field.NewPath("spec")) + + if err == nil { + t.Fatalf("expected error for exceeding RevisionHistoryLimit but got none") + } + + if err.Field != "spec.revisionHistoryLimit" { + t.Errorf("expected error on spec.revisionHistoryLimit, got: %v", err) + } +} diff --git a/pkg/webhook/configmapset/validating/webhooks.go b/pkg/webhook/configmapset/validating/webhooks.go new file mode 100644 index 0000000000..4c9becdf71 --- /dev/null +++ b/pkg/webhook/configmapset/validating/webhooks.go @@ -0,0 +1,38 @@ +/* +Copyright 2019 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package validating + +import ( + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + "github.com/openkruise/kruise/pkg/webhook/types" +) + +// +kubebuilder:webhook:path=/validate-apps-kruise-io-v1alpha1-configmapset,mutating=false,failurePolicy=fail,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups=apps.kruise.io,resources=configmapsets,verbs=create;update,versions=v1alpha1,name=vconfigmapset.kb.io + +var ( + // HandlerGetterMap contains admission webhook handlers + HandlerGetterMap = map[string]types.HandlerGetter{ + "validate-apps-kruise-io-v1alpha1-configmapset": func(mgr manager.Manager) admission.Handler { + return &ConfigMapSetCreateUpdateHandler{ + Client: mgr.GetClient(), + Decoder: admission.NewDecoder(mgr.GetScheme()), + } + }, + } +) diff --git a/pkg/webhook/pod/mutating/configmapset_mutating_test.go b/pkg/webhook/pod/mutating/configmapset_mutating_test.go new file mode 100644 index 0000000000..211868e5b2 --- /dev/null +++ b/pkg/webhook/pod/mutating/configmapset_mutating_test.go @@ -0,0 +1,305 @@ +package mutating + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + "github.com/openkruise/kruise/pkg/controller/configmapset" +) + +func init() { + _ = appsv1alpha1.AddToScheme(scheme.Scheme) +} + +func TestCheckConfigMapSetConflicts(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Labels: map[string]string{"app": "test"}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + {Name: "app-container"}, + }, + }, + } + + cases := []struct { + name string + cmsList []*appsv1alpha1.ConfigMapSet + expectError bool + }{ + { + name: "no conflicts", + cmsList: []*appsv1alpha1.ConfigMapSet{ + { + ObjectMeta: metav1.ObjectMeta{Name: "cms1"}, + Spec: appsv1alpha1.ConfigMapSetSpec{ + Containers: []appsv1alpha1.ConfigMapSetContainer{ + {Name: "app-container", MountPath: "/data/conf1"}, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "cms2"}, + Spec: appsv1alpha1.ConfigMapSetSpec{ + ReloadSidecarConfig: &appsv1alpha1.ReloadSidecarConfig{ + Type: appsv1alpha1.ReloadSidecarTypeK8s, + Config: &appsv1alpha1.ReloadSidecarConfigData{ + Name: "another-sidecar", + }, + }, + Containers: []appsv1alpha1.ConfigMapSetContainer{ + {Name: "app-container", MountPath: "/data/conf2"}, + }, + }, + }, + }, + expectError: false, + }, + { + name: "sidecar name conflict", + cmsList: []*appsv1alpha1.ConfigMapSet{ + { + ObjectMeta: metav1.ObjectMeta{Name: "cms1"}, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "cms2"}, + Spec: appsv1alpha1.ConfigMapSetSpec{ + ReloadSidecarConfig: &appsv1alpha1.ReloadSidecarConfig{ + Type: appsv1alpha1.ReloadSidecarTypeK8s, + Config: &appsv1alpha1.ReloadSidecarConfigData{ + Name: "cms1-reload-sidecar", // Conflicts with default name of cms1 + }, + }, + }, + }, + }, + expectError: true, + }, + { + name: "mount path conflict", + cmsList: []*appsv1alpha1.ConfigMapSet{ + { + ObjectMeta: metav1.ObjectMeta{Name: "cms1"}, + Spec: appsv1alpha1.ConfigMapSetSpec{ + Containers: []appsv1alpha1.ConfigMapSetContainer{ + {Name: "app-container", MountPath: "/data/conf1"}, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{Name: "cms2"}, + Spec: appsv1alpha1.ConfigMapSetSpec{ + ReloadSidecarConfig: &appsv1alpha1.ReloadSidecarConfig{ + Type: appsv1alpha1.ReloadSidecarTypeK8s, + Config: &appsv1alpha1.ReloadSidecarConfigData{ + Name: "another-sidecar", + }, + }, + Containers: []appsv1alpha1.ConfigMapSetContainer{ + {Name: "app-container", MountPath: "/data/conf1"}, // Same mount path + }, + }, + }, + }, + expectError: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fakeClient := fake.NewClientBuilder().WithScheme(scheme.Scheme).Build() + handler := &PodCreateHandler{Client: fakeClient} + err := handler.checkConfigMapSetConflicts(context.TODO(), pod, tc.cmsList) + if (err != nil) != tc.expectError { + t.Errorf("expected error: %v, got: %v", tc.expectError, err) + } + }) + } +} + +func TestHandlePodRevisionAnnotations(t *testing.T) { + cms := &appsv1alpha1.ConfigMapSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cms", + Namespace: "default", + }, + Spec: appsv1alpha1.ConfigMapSetSpec{ + Data: map[string]string{"key": "value"}, + CustomVersion: "v2", + UpdateStrategy: &appsv1alpha1.ConfigMapSetUpdateStrategy{ + Partition: &intstr.IntOrString{Type: intstr.String, StrVal: "50%"}, + }, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "test"}, + }, + }, + Status: appsv1alpha1.ConfigMapSetStatus{ + CurrentRevision: "old-hash", + CurrentCustomVersion: "v1", + }, + } + + updateHash, _ := configmapset.CalculateHash(cms.Spec.Data) + + podOld1 := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-old1", + Namespace: "default", + Labels: map[string]string{"app": "test"}, + Annotations: map[string]string{ + configmapset.GetConfigMapSetCurrentRevisionKey("test-cms"): "old-hash", + configmapset.GetConfigMapSetUpdateRevisionKey("test-cms"): "old-hash", + }, + }, + } + + // All cases should inject old version (CurrentRevision) if it exists, and new version if it doesn't. + // Since cms has Status.CurrentRevision = "old-hash", it should always get "old-hash". + + t.Run("should always get stable version", func(t *testing.T) { + fakeClient := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(podOld1).Build() + handler := &PodCreateHandler{Client: fakeClient} + + newPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-creating", + Namespace: "default", + Labels: map[string]string{"app": "test"}, + }, + } + + err := handler.handlePodRevisionAnnotations(context.TODO(), newPod, cms) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if newPod.Annotations[configmapset.GetConfigMapSetUpdateRevisionKey("test-cms")] != "old-hash" { + t.Errorf("expected old version %s, got %s", "old-hash", newPod.Annotations[configmapset.GetConfigMapSetUpdateRevisionKey("test-cms")]) + } + }) + + t.Run("should get new version if no current revision", func(t *testing.T) { + fakeClient := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(podOld1).Build() + handler := &PodCreateHandler{Client: fakeClient} + + cmsFirstRollout := cms.DeepCopy() + cmsFirstRollout.Status.CurrentRevision = "" + + newPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-creating", + Namespace: "default", + Labels: map[string]string{"app": "test"}, + }, + } + + err := handler.handlePodRevisionAnnotations(context.TODO(), newPod, cmsFirstRollout) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if newPod.Annotations[configmapset.GetConfigMapSetUpdateRevisionKey("test-cms")] != updateHash { + t.Errorf("expected new version %s, got %s", updateHash, newPod.Annotations[configmapset.GetConfigMapSetUpdateRevisionKey("test-cms")]) + } + }) +} + +func TestGetReloadSidecarName(t *testing.T) { + cases := []struct { + name string + cms *appsv1alpha1.ConfigMapSet + clientObjs []client.Object + expected string + expectError bool + }{ + { + name: "default", + cms: &appsv1alpha1.ConfigMapSet{ + ObjectMeta: metav1.ObjectMeta{Name: "test-cms"}, + }, + expected: "test-cms-reload-sidecar", + }, + { + name: "k8s config", + cms: &appsv1alpha1.ConfigMapSet{ + ObjectMeta: metav1.ObjectMeta{Name: "test-cms"}, + Spec: appsv1alpha1.ConfigMapSetSpec{ + ReloadSidecarConfig: &appsv1alpha1.ReloadSidecarConfig{ + Type: appsv1alpha1.ReloadSidecarTypeK8s, + Config: &appsv1alpha1.ReloadSidecarConfigData{ + Name: "custom-k8s-sidecar", + }, + }, + }, + }, + expected: "custom-k8s-sidecar", + }, + { + name: "sidecarset config", + cms: &appsv1alpha1.ConfigMapSet{ + ObjectMeta: metav1.ObjectMeta{Name: "test-cms"}, + Spec: appsv1alpha1.ConfigMapSetSpec{ + ReloadSidecarConfig: &appsv1alpha1.ReloadSidecarConfig{ + Type: appsv1alpha1.ReloadSidecarTypeSidecarSet, + Config: &appsv1alpha1.ReloadSidecarConfigData{ + SidecarSetRef: &appsv1alpha1.SidecarSetRef{ + ContainerName: "sidecarset-container", + }, + }, + }, + }, + }, + expected: "sidecarset-container", + }, + { + name: "custom configmap", + cms: &appsv1alpha1.ConfigMapSet{ + ObjectMeta: metav1.ObjectMeta{Name: "test-cms", Namespace: "default"}, + Spec: appsv1alpha1.ConfigMapSetSpec{ + ReloadSidecarConfig: &appsv1alpha1.ReloadSidecarConfig{ + Type: appsv1alpha1.ReloadSidecarTypeCustom, + Config: &appsv1alpha1.ReloadSidecarConfigData{ + ConfigMapRef: &appsv1alpha1.ConfigMapRef{ + Name: "custom-cm", + }, + }, + }, + }, + }, + clientObjs: []client.Object{ + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "custom-cm", Namespace: "default"}, + Data: map[string]string{ + "reload-sidecar": `{"name": "cm-sidecar", "image": "img:v1"}`, + }, + }, + }, + expected: "cm-sidecar", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fakeClient := fake.NewClientBuilder().WithScheme(scheme.Scheme).WithObjects(tc.clientObjs...).Build() + handler := &PodCreateHandler{Client: fakeClient} + name, err := handler.getReloadSidecarName(context.TODO(), tc.cms) + if (err != nil) != tc.expectError { + t.Fatalf("expected error: %v, got: %v", tc.expectError, err) + } + if name != tc.expected { + t.Errorf("expected name %s, got %s", tc.expected, name) + } + }) + } +} diff --git a/pkg/webhook/pod/mutating/pod_create_update_handler.go b/pkg/webhook/pod/mutating/pod_create_update_handler.go index b750653d5b..446be19c91 100644 --- a/pkg/webhook/pod/mutating/pod_create_update_handler.go +++ b/pkg/webhook/pod/mutating/pod_create_update_handler.go @@ -19,16 +19,44 @@ package mutating import ( "context" "encoding/json" + "flag" + "fmt" "net/http" + "sort" + "time" + kubecontroller "k8s.io/kubernetes/pkg/controller" + + admissionv1 "k8s.io/api/admission/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + "github.com/openkruise/kruise/apis/apps/pub" + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + "github.com/openkruise/kruise/pkg/controller/configmapset" "github.com/openkruise/kruise/pkg/features" utilfeature "github.com/openkruise/kruise/pkg/util/feature" ) +type RevisionKeys struct { + CurrentRevisionKey string + CurrentRevisionTimeStampKey string + TargetRevisionKey string + TargetRevisionTimeStampKey string + TargetCustomVersionKey string + CurrentCustomVersionKey string +} + +var defaultReloadImage string + +func init() { + flag.StringVar(&defaultReloadImage, "default-reload-image", "openkruise/reload-sidecar:v1.0.0", "Default reload sidecar image to use if not explicitly specified in ConfigMapSet.") +} + // PodCreateHandler handles Pod type PodCreateHandler struct { // To use the client, you need to do the following: @@ -76,6 +104,13 @@ func (h *PodCreateHandler) Handle(ctx context.Context, req admission.Request) ad changed = true } + // configMapSetMutatingPod must after sidecarsetMutatingPod + if skip, err := h.configMapSetMutatingPod(ctx, req, obj); err != nil { + return admission.Errored(http.StatusInternalServerError, err) + } else if !skip { + changed = true + } + // "the order matters and sidecarsetMutatingPod must precede containerLaunchPriorityInitialization" if skip, err := h.containerLaunchPriorityInitialization(ctx, req, obj); err != nil { return admission.Errored(http.StatusInternalServerError, err) @@ -130,3 +165,503 @@ func (h *PodCreateHandler) Handle(ctx context.Context, req admission.Request) ad } return admission.PatchResponseFromRaw(original, marshaled) } + +func (h *PodCreateHandler) injectSidecar4Pod(ctx context.Context, pod *corev1.Pod, cms *appsv1alpha1.ConfigMapSet) error { + configMapName := configmapset.GetConfigMapSetHubName(cms.Name) + defaultSidecarName := configmapset.GetConfigMapSetDefaultSidecarName(cms.Name) + volumeName := configmapset.GetConfigMapSetVolumeName(cms.Name) + + // get share path in reload-sidecar + configMountPath := configmapset.GetConfigMapSetConfigMountPath(cms.Name) + // get configmap mount path in reload-sidecar + configMapMountPath := configmapset.GetConfigMapSetConfigMapMountPath(cms.Name) + // build configmap volume with reload-sidecar + configMapVolume := corev1.Volume{ + Name: configmapset.GetConfigMapSetHubVolumeName(cms.Name), + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configMapName, + }, + }, + }, + } + // build downward api volume with reload-sidecar + podInfoVolume := corev1.Volume{ + Name: configmapset.GetConfigMapSetDownwardAPIVolumeName(cms.Name), + VolumeSource: corev1.VolumeSource{ + DownwardAPI: &corev1.DownwardAPIVolumeSource{ + Items: []corev1.DownwardAPIVolumeFile{ + { + Path: "target_revision", + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: fmt.Sprintf("metadata.annotations['%s']", configmapset.GetConfigMapSetUpdateRevisionKey(cms.Name)), + }, + }, + { + Path: "post_hook_config", + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: fmt.Sprintf("metadata.annotations['%s']", configmapset.GetConfigMapSetPostHookConfigKey(cms.Name)), + }, + }, + }, + }, + }, + } + + reloadSidecar := &corev1.Container{ + Name: defaultSidecarName, + Image: defaultReloadImage, + VolumeMounts: []corev1.VolumeMount{ + {Name: volumeName, MountPath: configMountPath}, + {Name: configMapVolume.Name, MountPath: configMapMountPath}, + {Name: podInfoVolume.Name, MountPath: fmt.Sprintf("/etc/cms_config")}, + }, + ReadinessProbe: configmapset.GetReloadSidecarReadinessProbe(), + Env: []corev1.EnvVar{{ + Name: configmapset.GetConfigMapSetEnvConfigPathName(), + Value: configMapMountPath, + }, { + Name: configmapset.GetConfigMapSetEnvSharePathName(), + Value: configMountPath, + }}, + } + + if cms.Spec.ReloadSidecarConfig != nil { + if cms.Spec.ReloadSidecarConfig.Type == appsv1alpha1.ReloadSidecarTypeK8s { + if cms.Spec.ReloadSidecarConfig.Config != nil { + if cms.Spec.ReloadSidecarConfig.Config.Name != "" { + reloadSidecar.Name = cms.Spec.ReloadSidecarConfig.Config.Name + } + if cms.Spec.ReloadSidecarConfig.Config.Image != "" { + reloadSidecar.Image = cms.Spec.ReloadSidecarConfig.Config.Image + } + if len(cms.Spec.ReloadSidecarConfig.Config.Command) > 0 { + reloadSidecar.Command = cms.Spec.ReloadSidecarConfig.Config.Command + } + if cms.Spec.ReloadSidecarConfig.Config.RestartPolicy != "" { + reloadSidecar.RestartPolicy = &cms.Spec.ReloadSidecarConfig.Config.RestartPolicy + } + } + } else if cms.Spec.ReloadSidecarConfig.Type == appsv1alpha1.ReloadSidecarTypeCustom { + if cms.Spec.ReloadSidecarConfig.Config != nil && cms.Spec.ReloadSidecarConfig.Config.ConfigMapRef != nil { + cmRef := cms.Spec.ReloadSidecarConfig.Config.ConfigMapRef + cmNamespace := cmRef.Namespace + if cmNamespace == "" { + cmNamespace = cms.Namespace + } + container, err := h.parseReloadSidecarByConfigMap(ctx, cmRef.Name, cmNamespace, reloadSidecar) + if err != nil { + return err + } + reloadSidecar = container + } else { + namespacedName := configmapset.GetDefaultCmsConfigMap() + container, err := h.parseReloadSidecarByConfigMap(ctx, namespacedName.Name, namespacedName.Namespace, reloadSidecar) + if err != nil { + return err + } + reloadSidecar = container + } + } else if cms.Spec.ReloadSidecarConfig.Type == appsv1alpha1.ReloadSidecarTypeSidecarSet { + klog.Infof("pod %s/%s will be injected by SidecarSet, skip full sidecar injection in ConfigMapSet webhook, just merge VolumeMounts and Env", pod.Namespace, pod.Name) + if cms.Spec.ReloadSidecarConfig.Config != nil && cms.Spec.ReloadSidecarConfig.Config.SidecarSetRef != nil { + // find reload-sidecar in Pod (because already injected by sidecarSet) + targetSidecarName := cms.Spec.ReloadSidecarConfig.Config.SidecarSetRef.ContainerName + container, err := h.praseReloadSidecarBySidecarSet(pod, reloadSidecar, targetSidecarName) + if err != nil { + return err + } + reloadSidecar = container + } else { + _, containerName := configmapset.GetDefaultCmsSidecarSet() + container, err := h.praseReloadSidecarBySidecarSet(pod, reloadSidecar, containerName) + if err != nil { + return err + } + reloadSidecar = container + } + } + } + + // Find if the same Sidecar container already exists + containerExistingIdx := -1 + for i, c := range pod.Spec.Containers { + if c.Name == reloadSidecar.Name { + containerExistingIdx = i + break + } + } + + // replace reload-sidecar if exist + if containerExistingIdx != -1 { + pod.Spec.Containers[containerExistingIdx] = *reloadSidecar + } else { + pod.Annotations[pub.ContainerLaunchPriorityKey] = pub.ContainerLaunchOrdered + pod.Spec.Containers = append([]corev1.Container{*reloadSidecar}, pod.Spec.Containers...) + } + + return h.injectVolumes(pod, configMapVolume, podInfoVolume) +} + +func (h *PodCreateHandler) praseReloadSidecarBySidecarSet(pod *corev1.Pod, reloadSidecar *corev1.Container, targetSidecarName string) (*corev1.Container, error) { + var container *corev1.Container + for _, c := range pod.Spec.Containers { + if c.Name == targetSidecarName { + container = c.DeepCopy() + break + } + } + if container == nil { + klog.Errorf("target container %s not found", targetSidecarName) + return nil, fmt.Errorf("target container %s not found", targetSidecarName) + } + return h.mergeReloadSidecar(reloadSidecar, container), nil +} + +func (h *PodCreateHandler) parseReloadSidecarByConfigMap(ctx context.Context, namespace string, name string, reloadSidecar *corev1.Container) (*corev1.Container, error) { + defaultCM := &corev1.ConfigMap{} + if err := h.Client.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, defaultCM); err != nil { + klog.Errorf("failed to get custom sidecar configmap %s/%s: %v", namespace, name, err) + return nil, err + } + if _, exists := defaultCM.Data["reload-sidecar"]; !exists { + klog.Errorf("custom sidecar configmap %s/%s missing key 'reload-sidecar'", namespace, name) + return nil, fmt.Errorf("custom sidecar configmap %s/%s missing key 'reload-sidecar'", namespace, name) + } + container := &corev1.Container{} + if unmarshalErr := json.Unmarshal([]byte(defaultCM.Data["reload-sidecar"]), container); unmarshalErr != nil { + klog.Errorf("failed to unmarshal custom sidecar configmap %s/%s data 'reload-sidecar': %v", namespace, name, unmarshalErr) + return nil, unmarshalErr + } + return h.mergeReloadSidecar(reloadSidecar, container), nil +} + +func (h *PodCreateHandler) mergeReloadSidecar(reloadSidecar *corev1.Container, container *corev1.Container) *corev1.Container { + if container.Name == "" { + container.Name = reloadSidecar.Name + } + + if container.Image == "" { + container.Image = reloadSidecar.Image + } + + // Merge VolumeMounts + for _, newMount := range reloadSidecar.VolumeMounts { + mountExists := false + for _, existingMount := range container.VolumeMounts { + if existingMount.Name == newMount.Name || existingMount.MountPath == newMount.MountPath { + mountExists = true + break + } + } + if !mountExists { + container.VolumeMounts = append(container.VolumeMounts, newMount) + } + } + + // Merge Env + for _, newEnv := range reloadSidecar.Env { + envExists := false + for j, existingEnv := range container.Env { + if existingEnv.Name == newEnv.Name { + container.Env[j] = newEnv + envExists = true + break + } + } + if !envExists { + container.Env = append(container.Env, newEnv) + } + } + + // Merge ReadinessProbe + if container.ReadinessProbe == nil { + container.ReadinessProbe = reloadSidecar.ReadinessProbe + } + + return container +} + +func (h *PodCreateHandler) injectVolumes(pod *corev1.Pod, configMapVolume corev1.Volume, podInfoVolume corev1.Volume) error { + volumeExistingIdx := -1 + for i, vol := range pod.Spec.Volumes { + if vol.Name == configMapVolume.Name { + volumeExistingIdx = i + break + } + } + if volumeExistingIdx != -1 { + pod.Spec.Volumes[volumeExistingIdx] = configMapVolume + } else { + pod.Spec.Volumes = append(pod.Spec.Volumes, configMapVolume) + } + + podInfoVolumeExistingIdx := -1 + for i, vol := range pod.Spec.Volumes { + if vol.Name == podInfoVolume.Name { + podInfoVolumeExistingIdx = i + break + } + } + if podInfoVolumeExistingIdx != -1 { + pod.Spec.Volumes[podInfoVolumeExistingIdx] = podInfoVolume + } else { + pod.Spec.Volumes = append(pod.Spec.Volumes, podInfoVolume) + } + + return nil +} + +func (h *PodCreateHandler) injectEmptyDir4Pod(pod *corev1.Pod, cms *appsv1alpha1.ConfigMapSet) error { + volumeName := configmapset.GetConfigMapSetVolumeName(cms.Name) + + volume := corev1.Volume{ + Name: volumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } + existingIdx := -1 + // Check if the volume already exists + for index, v := range pod.Spec.Volumes { + if v.Name == volume.Name { + existingIdx = index + break + } + } + if existingIdx != -1 { + pod.Spec.Volumes[existingIdx] = volume + } else { + pod.Spec.Volumes = append(pod.Spec.Volumes, volume) + } + + for _, injectC := range cms.Spec.Containers { + containerName := configmapset.GetContainerName(pod, injectC) + for i, c := range pod.Spec.Containers { + if containerName == c.Name { + h.applyVM4Container(&c, injectC, volume.Name) + pod.Spec.Containers[i] = c + break + } + } + } + return nil +} + +func (h *PodCreateHandler) applyVM4Container(c *corev1.Container, v appsv1alpha1.ConfigMapSetContainer, volumeName string) { + for i := range c.VolumeMounts { + if c.VolumeMounts[i].Name == volumeName { + // Update the mount path of the existing volume mount + c.VolumeMounts[i].MountPath = v.MountPath + return + } + } + c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{ + Name: volumeName, + MountPath: v.MountPath, // Mount path + ReadOnly: true, + }) +} + +func (h *PodCreateHandler) configMapSetMutatingPod(ctx context.Context, req admission.Request, pod *corev1.Pod) (skip bool, err error) { + // just only inject in pod create + if len(req.AdmissionRequest.SubResource) > 0 || + req.AdmissionRequest.Operation != admissionv1.Create || + req.AdmissionRequest.Resource.Resource != "pods" { + return true, nil + } + + if !kubecontroller.IsPodActive(pod) { + return true, nil + } + + // no need inject with configMapSet + if pod.Annotations == nil || pod.Annotations[configmapset.GetConfigMapSetEnabledKey()] != "true" { + return true, nil + } + + cmsList, err := configmapset.GetMatchConfigMapSets(h.Client, pod) + if err != nil { // Return error if it's not a NotFound error + klog.Errorf("get matched cms for pod %s/%s failed, error : %v", pod.Namespace, pod.Name, err) + return false, fmt.Errorf("get related cms for pod %s/%s failed, error : %v", pod.Namespace, pod.Name, err) + } + + if len(cmsList) == 0 { + return true, nil + } + + // Check for conflicts among matched ConfigMapSets + if err := h.checkConfigMapSetConflicts(ctx, pod, cmsList); err != nil { + klog.Errorf("ConfigMapSet conflict for pod %s/%s: %v", pod.Namespace, pod.Name, err) + return false, err + } + + copyPod := pod.DeepCopy() + // sort by cms name, for inject Pod stable + sort.Slice(cmsList, func(i, j int) bool { + return cmsList[i].Name < cmsList[j].Name + }) + for _, cms := range cmsList { + if cms.DeletionTimestamp != nil { + continue + } + // Handle version annotations on creation + err = h.handlePodRevisionAnnotations(ctx, pod, cms) + if err != nil { + klog.Errorf("handle revision annotations for pod %s/%s failed, error : %v", pod.Namespace, pod.Name, err) + return false, fmt.Errorf("handle revision annotations for pod %s/%s failed, error : %v", pod.Namespace, pod.Name, err) + } + + // inject share emptyDir volume + klog.Infof("inject emptyDir for pod %s/%s", pod.Namespace, pod.Name) + err = h.injectEmptyDir4Pod(pod, cms) + if err != nil { + klog.Errorf("inject emptyDir for pod %s/%s failed, error : %v", pod.Namespace, pod.Name, err) + return false, fmt.Errorf("inject emptyDir for pod %s/%s failed, error : %v", pod.Namespace, pod.Name, err) + } + + // inject reload sidecar + klog.Infof("inject sidecar for pod %s/%s", pod.Namespace, pod.Name) + err = h.injectSidecar4Pod(ctx, pod, cms) + if err != nil { + klog.Errorf("inject sidecar for pod %s/%s failed, error : %v", pod.Namespace, pod.Name, err) + return false, fmt.Errorf("inject sidecar for pod %s/%s failed, error : %v", pod.Namespace, pod.Name, err) + } + } + + if equality.Semantic.DeepEqual(copyPod, pod) { + return true, nil + } + + return false, nil +} + +func (h *PodCreateHandler) checkConfigMapSetConflicts(ctx context.Context, pod *corev1.Pod, cmsList []*appsv1alpha1.ConfigMapSet) error { + if len(cmsList) <= 1 { + return nil + } + + sidecarNames := make(map[string]string) // Sidecar Container Name -> ConfigMapSet Name + mountPaths := make(map[string]map[string]string) // ContainerName -> MountPath -> ConfigMapSet Name + + for _, cms := range cmsList { + // 1. Check for reload-sidecar name conflicts + sidecarName, err := h.getReloadSidecarName(ctx, cms) + if err != nil { + return err + } + if sidecarName != "" { + if existingCmsName, exists := sidecarNames[sidecarName]; exists { + return fmt.Errorf("ConfigMapSet conflict: both %s and %s attempt to inject reload-sidecar with the same name '%s' into pod %s", existingCmsName, cms.Name, sidecarName, pod.Name) + } + sidecarNames[sidecarName] = cms.Name + } + + // 2. Check for container mount path conflicts + for _, c := range cms.Spec.Containers { + targetContainerName := configmapset.GetContainerName(pod, c) + if targetContainerName == "" { + continue + } + if mountPaths[targetContainerName] == nil { + mountPaths[targetContainerName] = make(map[string]string) + } + if existingCmsName, exists := mountPaths[targetContainerName][c.MountPath]; exists { + return fmt.Errorf("ConfigMapSet conflict: both %s and %s attempt to mount to '%s' on container '%s' in pod %s", existingCmsName, cms.Name, c.MountPath, targetContainerName, pod.Name) + } + mountPaths[targetContainerName][c.MountPath] = cms.Name + } + } + + return nil +} + +func (h *PodCreateHandler) getReloadSidecarName(ctx context.Context, cms *appsv1alpha1.ConfigMapSet) (string, error) { + if cms.Spec.ReloadSidecarConfig == nil || cms.Spec.ReloadSidecarConfig.Config == nil { + return configmapset.GetConfigMapSetDefaultSidecarName(cms.Name), nil + } + + config := cms.Spec.ReloadSidecarConfig.Config + switch cms.Spec.ReloadSidecarConfig.Type { + case appsv1alpha1.ReloadSidecarTypeK8s: + if config.Name != "" { + return config.Name, nil + } + case appsv1alpha1.ReloadSidecarTypeSidecarSet: + if config.SidecarSetRef != nil { + return config.SidecarSetRef.ContainerName, nil + } + case appsv1alpha1.ReloadSidecarTypeCustom: + if config.ConfigMapRef != nil { + cmRef := config.ConfigMapRef + customerCM := &corev1.ConfigMap{} + cmNamespace := cmRef.Namespace + if cmNamespace == "" { + cmNamespace = cms.Namespace + } + if err := h.Client.Get(ctx, types.NamespacedName{Name: cmRef.Name, Namespace: cmNamespace}, customerCM); err == nil { + if containerData, exists := customerCM.Data["reload-sidecar"]; exists { + var reloadSidecar corev1.Container + if unmarshalErr := json.Unmarshal([]byte(containerData), &reloadSidecar); unmarshalErr == nil { + if reloadSidecar.Name != "" { + return reloadSidecar.Name, nil + } + } + } + } + return fmt.Sprintf("custom-sidecar-%s", cms.Name), nil + } + } + return configmapset.GetConfigMapSetDefaultSidecarName(cms.Name), nil +} + +func (h *PodCreateHandler) handlePodRevisionAnnotations(ctx context.Context, pod *corev1.Pod, cms *appsv1alpha1.ConfigMapSet) error { + targetRevisionKey := configmapset.GetConfigMapSetUpdateRevisionKey(cms.Name) + currentRevisionKey := configmapset.GetConfigMapSetCurrentRevisionKey(cms.Name) + targetCustomVersionKey := configmapset.GetConfigMapSetUpdateCustomVersionKey(cms.Name) + currentCustomVersionKey := configmapset.GetConfigMapSetCurrentCustomVersionKey(cms.Name) + currentRevisionTimestampKey := configmapset.GetConfigMapSetCurrentRevisionTimeStampKey(cms.Name) + updateRevisionTimestampKey := configmapset.GetConfigMapSetUpdateRevisionTimeStampKey(cms.Name) + + reloadSidecarRestartKey := configmapset.GetConfigMapSetReloadSidecarRestartKey(cms.Name) + + if pod.Annotations == nil { + pod.Annotations = make(map[string]string) + } + + now := time.Now().Format("2006-01-02 15:04:05") + + // Strategy 1: Prioritize using CurrentRevision. If empty, it means this is the first release, so use the target version calculated from Data. + targetVersion := cms.Status.CurrentRevision + if targetVersion == "" { + hash, err := configmapset.CalculateHash(cms.Spec.Data) + if err == nil { + targetVersion = hash + } + } + + pod.Annotations[targetRevisionKey] = targetVersion + pod.Annotations[currentRevisionKey] = targetVersion + pod.Annotations[targetCustomVersionKey] = cms.Status.CurrentCustomVersion + pod.Annotations[currentCustomVersionKey] = cms.Status.CurrentCustomVersion + pod.Annotations[currentRevisionTimestampKey] = now + pod.Annotations[updateRevisionTimestampKey] = now + + expectHash := configmapset.GetContainerHash(pod, targetVersion) + pod.Annotations[reloadSidecarRestartKey] = expectHash + for _, container := range cms.Spec.Containers { + containerRestartKey := configmapset.GetConfigMapSetContainerRestartKey(cms.Name, container.Name) + pod.Annotations[containerRestartKey] = expectHash + } + + // Inject PostHook config into Pod Annotations + if cms.Spec.EffectPolicy != nil && cms.Spec.EffectPolicy.Type == appsv1alpha1.EffectPolicyTypePostHook && cms.Spec.EffectPolicy.PostHook != nil { + hookConfigBytes, err := json.Marshal(cms.Spec.EffectPolicy.PostHook) + if err == nil { + pod.Annotations[configmapset.GetConfigMapSetPostHookConfigKey(cms.Name)] = string(hookConfigBytes) + } + } + + return nil +} diff --git a/test/e2e/apps/v1alpha1/configmapset.go b/test/e2e/apps/v1alpha1/configmapset.go new file mode 100644 index 0000000000..0882b4ff32 --- /dev/null +++ b/test/e2e/apps/v1alpha1/configmapset.go @@ -0,0 +1,378 @@ +package v1alpha1 + +import ( + "context" + "fmt" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/wait" + + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + "github.com/openkruise/kruise/pkg/controller/configmapset" + kruiseframework "github.com/openkruise/kruise/test/e2e/framework/v1alpha1" +) + +var _ = ginkgo.Describe("ConfigMapSet", func() { + f := kruiseframework.NewDefaultFramework("configmapset") + var tester *kruiseframework.ConfigMapSetTester + var namespace string + + ginkgo.BeforeEach(func() { + tester = kruiseframework.NewConfigMapSetTester(f.KruiseClientSet, f.ClientSet) + namespace = f.Namespace.Name + }) + + ginkgo.AfterEach(func() { + // Clean up is handled by framework + }) + + ginkgo.It("should be able to create, update and delete a ConfigMapSet with simple pod updates", func() { + cmsName := "test-cms-basic" + podName := "test-pod-basic" + + ginkgo.By(fmt.Sprintf("1. Create ConfigMapSet %s", cmsName)) + cms := &appsv1alpha1.ConfigMapSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmsName, + Namespace: namespace, + }, + Spec: appsv1alpha1.ConfigMapSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "cms-test"}, + }, + Data: map[string]string{ + "config.yaml": "key: v1", + }, + CustomVersion: "v1", + Containers: []appsv1alpha1.ConfigMapSetContainer{ + { + Name: "main", + MountPath: "/etc/config", + }, + }, + EffectPolicy: &appsv1alpha1.EffectPolicy{ + Type: appsv1alpha1.EffectPolicyTypeHotUpdate, + }, + UpdateStrategy: &appsv1alpha1.ConfigMapSetUpdateStrategy{ + Partition: &intstr.IntOrString{Type: intstr.Int, IntVal: 0}, + MaxUnavailable: &intstr.IntOrString{Type: intstr.Int, IntVal: 1}, + }, + }, + } + + tester.CreateConfigMapSet(cms) + + ginkgo.By(fmt.Sprintf("2. Create a Pod %s matching the ConfigMapSet", podName)) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: namespace, + Labels: map[string]string{ + "app": "cms-test", + }, + Annotations: map[string]string{ + configmapset.GetConfigMapSetEnabledKey(): "true", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "main", + Image: "nginx:alpine", + Command: []string{"sleep", "3600"}, + }, + }, + }, + } + tester.CreatePod(pod) + + // Wait for pod to be running and injected + tester.WaitForPodAnnotation(namespace, podName, configmapset.GetConfigMapSetCurrentCustomVersionKey(cmsName), "v1") + + ginkgo.By(fmt.Sprintf("3. Update ConfigMapSet %s data", cmsName)) + cms = tester.GetConfigMapSet(namespace, cmsName) + cms.Spec.Data["config.yaml"] = "key: v2" + cms.Spec.CustomVersion = "v2" + tester.UpdateConfigMapSet(cms) + + // Wait for pod annotation to be updated + tester.WaitForPodAnnotation(namespace, podName, configmapset.GetConfigMapSetCurrentCustomVersionKey(cmsName), "v2") + + ginkgo.By(fmt.Sprintf("4. Check ConfigMapSet %s status", cmsName)) + tester.WaitForConfigMapSetStatus(namespace, cmsName, func(c *appsv1alpha1.ConfigMapSet) bool { + return c.Status.CurrentCustomVersion == "v2" && c.Status.UpdatedReplicas == 1 + }) + + ginkgo.By(fmt.Sprintf("5. Delete Pod %s", podName)) + tester.DeletePod(namespace, podName) + }) + + ginkgo.It("should respect partition during updates", func() { + cmsName := "test-cms-partition" + + ginkgo.By(fmt.Sprintf("1. Create ConfigMapSet %s with 50%% partition", cmsName)) + cms := &appsv1alpha1.ConfigMapSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmsName, + Namespace: namespace, + }, + Spec: appsv1alpha1.ConfigMapSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "cms-partition-test"}, + }, + Data: map[string]string{ + "config.yaml": "key: v1", + }, + CustomVersion: "v1", + EffectPolicy: &appsv1alpha1.EffectPolicy{ + Type: appsv1alpha1.EffectPolicyTypeHotUpdate, + }, + UpdateStrategy: &appsv1alpha1.ConfigMapSetUpdateStrategy{ + Partition: &intstr.IntOrString{Type: intstr.Int, IntVal: 1}, // Leave 1 old pod + MaxUnavailable: &intstr.IntOrString{Type: intstr.Int, IntVal: 2}, + }, + }, + } + + tester.CreateConfigMapSet(cms) + + ginkgo.By("2. Create 2 Pods matching the ConfigMapSet") + for i := 0; i < 2; i++ { + podName := fmt.Sprintf("test-pod-partition-%d", i) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: namespace, + Labels: map[string]string{ + "app": "cms-partition-test", + }, + Annotations: map[string]string{ + configmapset.GetConfigMapSetEnabledKey(): "true", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "main", + Image: "nginx:alpine", + Command: []string{"sleep", "3600"}, + }, + }, + }, + } + tester.CreatePod(pod) + } + + // Wait for both pods to be v1 + for i := 0; i < 2; i++ { + podName := fmt.Sprintf("test-pod-partition-%d", i) + tester.WaitForPodAnnotation(namespace, podName, configmapset.GetConfigMapSetCurrentCustomVersionKey(cmsName), "v1") + } + + ginkgo.By(fmt.Sprintf("3. Update ConfigMapSet %s data", cmsName)) + cms = tester.GetConfigMapSet(namespace, cmsName) + cms.Spec.Data["config.yaml"] = "key: v2" + cms.Spec.CustomVersion = "v2" + tester.UpdateConfigMapSet(cms) + + // With Partition=1 and Replicas=2, exactly 1 pod should be updated + ginkgo.By("4. Check that only 1 Pod is updated") + tester.WaitForConfigMapSetStatus(namespace, cmsName, func(c *appsv1alpha1.ConfigMapSet) bool { + return c.Status.UpdatedReplicas == 1 && c.Status.Replicas == 2 + }) + + // Wait a bit to ensure the other one is not updated + time.Sleep(3 * time.Second) + + updatedCount := 0 + for i := 0; i < 2; i++ { + podName := fmt.Sprintf("test-pod-partition-%d", i) + pod := tester.GetPod(namespace, podName) + if pod.Annotations[configmapset.GetConfigMapSetCurrentCustomVersionKey(cmsName)] == "v2" { + updatedCount++ + } + } + + gomega.Expect(updatedCount).To(gomega.Equal(1)) + }) + + ginkgo.It("should always inject current revision for newly created pods during rolling update", func() { + cmsName := "test-cms-fallback" + + ginkgo.By(fmt.Sprintf("1. Create ConfigMapSet %s with 50%% partition", cmsName)) + cms := &appsv1alpha1.ConfigMapSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmsName, + Namespace: namespace, + }, + Spec: appsv1alpha1.ConfigMapSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "cms-fallback-test"}, + }, + Data: map[string]string{ + "config.yaml": "key: v1", + }, + CustomVersion: "v1", + EffectPolicy: &appsv1alpha1.EffectPolicy{ + Type: appsv1alpha1.EffectPolicyTypeHotUpdate, + }, + UpdateStrategy: &appsv1alpha1.ConfigMapSetUpdateStrategy{ + Partition: &intstr.IntOrString{Type: intstr.Int, IntVal: 1}, + }, + }, + } + + tester.CreateConfigMapSet(cms) + + ginkgo.By("2. Create 1 Pod matching the ConfigMapSet") + podName1 := "test-pod-fallback-1" + pod1 := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName1, + Namespace: namespace, + Labels: map[string]string{ + "app": "cms-fallback-test", + }, + Annotations: map[string]string{ + configmapset.GetConfigMapSetEnabledKey(): "true", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "main", + Image: "nginx:alpine", + Command: []string{"sleep", "3600"}, + }, + }, + }, + } + tester.CreatePod(pod1) + tester.WaitForPodAnnotation(namespace, podName1, configmapset.GetConfigMapSetCurrentCustomVersionKey(cmsName), "v1") + + ginkgo.By(fmt.Sprintf("3. Update ConfigMapSet %s to v2", cmsName)) + cms = tester.GetConfigMapSet(namespace, cmsName) + cms.Spec.Data["config.yaml"] = "key: v2" + cms.Spec.CustomVersion = "v2" + tester.UpdateConfigMapSet(cms) + + // Wait for the first pod to be updated to v2 (since partition is 1, but we only have 1 pod right now, it might update or wait depending on calculation, actually expectedUpdate = 1 - 1 = 0, so it will NOT update yet!) + // Wait, partition=1, replicas=1. expectedUpdate = 1 - 1 = 0. + // Let's create the second pod. + + ginkgo.By("4. Create 2nd Pod concurrently") + podName2 := "test-pod-fallback-2" + pod2 := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName2, + Namespace: namespace, + Labels: map[string]string{ + "app": "cms-fallback-test", + }, + Annotations: map[string]string{ + configmapset.GetConfigMapSetEnabledKey(): "true", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "main", + Image: "nginx:alpine", + Command: []string{"sleep", "3600"}, + }, + }, + }, + } + tester.CreatePod(pod2) + + // The webhook should inject "v1" (CurrentRevision) into the new pod, NOT v2. + // Even though it's during a rollout. + ginkgo.By("5. Check that new pod is initially injected with v1 by webhook") + pod2 = tester.GetPod(namespace, podName2) + gomega.Expect(pod2.Annotations[configmapset.GetConfigMapSetCurrentCustomVersionKey(cmsName)]).To(gomega.Equal("v1")) + + // Now we have 2 pods. Partition is 1. Expected update is 2 - 1 = 1. + // The controller will eventually pick one of them to update to v2. + ginkgo.By("6. Wait for controller to update exactly 1 pod to v2") + tester.WaitForConfigMapSetStatus(namespace, cmsName, func(c *appsv1alpha1.ConfigMapSet) bool { + return c.Status.UpdatedReplicas == 1 && c.Status.Replicas == 2 + }) + }) + + ginkgo.It("should reject deletion if pods are still using it via webhook", func() { + cmsName := "test-cms-delete" + podName := "test-pod-delete" + + ginkgo.By(fmt.Sprintf("1. Create ConfigMapSet %s", cmsName)) + cms := &appsv1alpha1.ConfigMapSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: cmsName, + Namespace: namespace, + }, + Spec: appsv1alpha1.ConfigMapSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "cms-delete-test"}, + }, + Data: map[string]string{ + "config.yaml": "key: v1", + }, + }, + } + + tester.CreateConfigMapSet(cms) + + ginkgo.By(fmt.Sprintf("2. Create a Pod %s matching the ConfigMapSet", podName)) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: namespace, + Labels: map[string]string{ + "app": "cms-delete-test", + }, + Annotations: map[string]string{ + configmapset.GetConfigMapSetEnabledKey(): "true", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "main", + Image: "nginx:alpine", + Command: []string{"sleep", "3600"}, + }, + }, + }, + } + tester.CreatePod(pod) + + // Wait for pod to be running + tester.WaitForPodAnnotation(namespace, podName, configmapset.GetConfigMapSetCurrentRevisionKey(cmsName), "") // just wait for it to be processed + + ginkgo.By(fmt.Sprintf("3. Try to delete ConfigMapSet %s, should fail", cmsName)) + cmsToDelete := tester.GetConfigMapSet(namespace, cmsName) + err := tester.Kc.AppsV1alpha1().ConfigMapSets(cmsToDelete.Namespace).Delete(context.TODO(), cmsToDelete.Name, metav1.DeleteOptions{}) + gomega.Expect(err).To(gomega.HaveOccurred(), "Expected deletion to fail due to webhook validation") + gomega.Expect(err.Error()).To(gomega.ContainSubstring("is still used configmapSet")) + + ginkgo.By(fmt.Sprintf("4. Delete Pod %s", podName)) + tester.DeletePod(namespace, podName) + + // Wait for pod to disappear + err = wait.PollImmediate(time.Second, time.Minute, func() (bool, error) { + _, err := tester.K8sClient.CoreV1().Pods(namespace).Get(context.TODO(), podName, metav1.GetOptions{}) + if err != nil { + return true, nil // Gone + } + return false, nil + }) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + ginkgo.By(fmt.Sprintf("5. Try to delete ConfigMapSet %s again, should succeed", cmsName)) + tester.DeleteConfigMapSet(namespace, cmsName) + }) +}) diff --git a/test/e2e/framework/v1alpha1/configmapset_util.go b/test/e2e/framework/v1alpha1/configmapset_util.go new file mode 100644 index 0000000000..c63ab23600 --- /dev/null +++ b/test/e2e/framework/v1alpha1/configmapset_util.go @@ -0,0 +1,95 @@ +package v1alpha1 + +import ( + "context" + "time" + + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + + appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1" + kruiseclientset "github.com/openkruise/kruise/pkg/client/clientset/versioned" +) + +type ConfigMapSetTester struct { + Kc kruiseclientset.Interface + K8sClient clientset.Interface +} + +func NewConfigMapSetTester(kc kruiseclientset.Interface, k8sClient clientset.Interface) *ConfigMapSetTester { + return &ConfigMapSetTester{ + Kc: kc, + K8sClient: k8sClient, + } +} + +func (t *ConfigMapSetTester) CreateConfigMapSet(cms *appsv1alpha1.ConfigMapSet) *appsv1alpha1.ConfigMapSet { + _, err := t.Kc.AppsV1alpha1().ConfigMapSets(cms.Namespace).Create(context.TODO(), cms, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return t.GetConfigMapSet(cms.Namespace, cms.Name) +} + +func (t *ConfigMapSetTester) UpdateConfigMapSet(cms *appsv1alpha1.ConfigMapSet) *appsv1alpha1.ConfigMapSet { + _, err := t.Kc.AppsV1alpha1().ConfigMapSets(cms.Namespace).Update(context.TODO(), cms, metav1.UpdateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return t.GetConfigMapSet(cms.Namespace, cms.Name) +} + +func (t *ConfigMapSetTester) GetConfigMapSet(namespace, name string) *appsv1alpha1.ConfigMapSet { + cms, err := t.Kc.AppsV1alpha1().ConfigMapSets(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return cms +} + +func (t *ConfigMapSetTester) DeleteConfigMapSet(namespace, name string) { + err := t.Kc.AppsV1alpha1().ConfigMapSets(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) +} + +func (t *ConfigMapSetTester) CreatePod(pod *corev1.Pod) *corev1.Pod { + _, err := t.K8sClient.CoreV1().Pods(pod.Namespace).Create(context.TODO(), pod, metav1.CreateOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return t.GetPod(pod.Namespace, pod.Name) +} + +func (t *ConfigMapSetTester) GetPod(namespace, name string) *corev1.Pod { + pod, err := t.K8sClient.CoreV1().Pods(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return pod +} + +func (t *ConfigMapSetTester) DeletePod(namespace, name string) { + err := t.K8sClient.CoreV1().Pods(namespace).Delete(context.TODO(), name, metav1.DeleteOptions{}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) +} + +func (t *ConfigMapSetTester) WaitForConfigMapSetStatus(namespace, name string, condition func(*appsv1alpha1.ConfigMapSet) bool) *appsv1alpha1.ConfigMapSet { + var finalCMS *appsv1alpha1.ConfigMapSet + err := wait.PollImmediate(time.Second, time.Minute*3, func() (bool, error) { + cms := t.GetConfigMapSet(namespace, name) + if condition(cms) { + finalCMS = cms + return true, nil + } + return false, nil + }) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return finalCMS +} + +func (t *ConfigMapSetTester) WaitForPodAnnotation(namespace, name string, key string, expectedValue string) *corev1.Pod { + var finalPod *corev1.Pod + err := wait.PollImmediate(time.Second, time.Minute, func() (bool, error) { + pod := t.GetPod(namespace, name) + if pod.Annotations != nil && pod.Annotations[key] == expectedValue { + finalPod = pod + return true, nil + } + return false, nil + }) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + return finalPod +}