Skip to content

Commit c43f52f

Browse files
leon-apeapecloud-bot
authored andcommitted
fix: remove global affinity and tolerations (#9949)
(cherry picked from commit 47a06ab)
1 parent acc381a commit c43f52f

File tree

14 files changed

+17
-770
lines changed

14 files changed

+17
-770
lines changed

cmd/manager/main.go

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -271,18 +271,12 @@ func validateRequiredToParseConfigs() error {
271271
if err := validateAffinity(viper.GetString(constant.CfgKeyCtrlrMgrAffinity)); err != nil {
272272
return err
273273
}
274-
if cmNodeSelector := viper.GetString(constant.CfgKeyCtrlrMgrNodeSelector); cmNodeSelector != "" {
275-
nodeSelector := map[string]string{}
276-
if err := json.Unmarshal([]byte(cmNodeSelector), &nodeSelector); err != nil {
274+
if nodeSelector := viper.GetString(constant.CfgKeyCtrlrMgrNodeSelector); nodeSelector != "" {
275+
__ := map[string]string{}
276+
if err := json.Unmarshal([]byte(nodeSelector), &__); err != nil {
277277
return err
278278
}
279279
}
280-
if err := validateTolerations(viper.GetString(constant.CfgKeyDataPlaneTolerations)); err != nil {
281-
return err
282-
}
283-
if err := validateAffinity(viper.GetString(constant.CfgKeyDataPlaneAffinity)); err != nil {
284-
return err
285-
}
286280

287281
if imagePullSecrets := viper.GetString(constant.KBImagePullSecrets); imagePullSecrets != "" {
288282
secrets := make([]corev1.LocalObjectReference, 0)

controllers/apps/cluster/suite_test.go

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ package cluster
2121

2222
import (
2323
"context"
24-
"fmt"
2524
"go/build"
2625
"path/filepath"
2726
"testing"
@@ -57,11 +56,6 @@ import (
5756
// These tests use Ginkgo (BDD-style Go testing framework). Refer to
5857
// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
5958

60-
const (
61-
testDataPlaneNodeAffinityKey = "testDataPlaneNodeAffinityKey"
62-
testDataPlaneTolerationKey = "testDataPlaneTolerationKey"
63-
)
64-
6559
var cfg *rest.Config
6660
var k8sClient client.Client
6761
var testEnv *envtest.Environment
@@ -92,10 +86,6 @@ var _ = BeforeSuite(func() {
9286
}
9387

9488
viper.SetDefault(constant.CfgKeyCtrlrReconcileRetryDurationMS, 10)
95-
viper.Set(constant.CfgKeyDataPlaneTolerations,
96-
fmt.Sprintf("[{\"key\":\"%s\", \"operator\": \"Exists\", \"effect\": \"NoSchedule\"}]", testDataPlaneTolerationKey))
97-
viper.Set(constant.CfgKeyDataPlaneAffinity,
98-
fmt.Sprintf("{\"nodeAffinity\":{\"preferredDuringSchedulingIgnoredDuringExecution\":[{\"preference\":{\"matchExpressions\":[{\"key\":\"%s\",\"operator\":\"In\",\"values\":[\"true\"]}]},\"weight\":100}]}}", testDataPlaneNodeAffinityKey))
9989

10090
ctx, cancel = context.WithCancel(context.TODO())
10191
logger = logf.FromContext(ctx).WithValues()

controllers/apps/component/suite_test.go

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ package component
2121

2222
import (
2323
"context"
24-
"fmt"
2524
"go/build"
2625
"path/filepath"
2726
"testing"
@@ -59,11 +58,6 @@ import (
5958
// These tests use Ginkgo (BDD-style Go testing framework). Refer to
6059
// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
6160

62-
const (
63-
testDataPlaneNodeAffinityKey = "testDataPlaneNodeAffinityKey"
64-
testDataPlaneTolerationKey = "testDataPlaneTolerationKey"
65-
)
66-
6761
var cfg *rest.Config
6862
var k8sClient client.Client
6963
var testEnv *envtest.Environment
@@ -94,10 +88,6 @@ var _ = BeforeSuite(func() {
9488
}
9589

9690
viper.SetDefault(constant.CfgKeyCtrlrReconcileRetryDurationMS, 10)
97-
viper.Set(constant.CfgKeyDataPlaneTolerations,
98-
fmt.Sprintf("[{\"key\":\"%s\", \"operator\": \"Exists\", \"effect\": \"NoSchedule\"}]", testDataPlaneTolerationKey))
99-
viper.Set(constant.CfgKeyDataPlaneAffinity,
100-
fmt.Sprintf("{\"nodeAffinity\":{\"preferredDuringSchedulingIgnoredDuringExecution\":[{\"preference\":{\"matchExpressions\":[{\"key\":\"%s\",\"operator\":\"In\",\"values\":[\"true\"]}]},\"weight\":100}]}}", testDataPlaneNodeAffinityKey))
10191

10292
ctx, cancel = context.WithCancel(context.TODO())
10393
logger = logf.FromContext(ctx).WithValues()

controllers/apps/suite_test.go

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ package apps
2121

2222
import (
2323
"context"
24-
"fmt"
2524
"go/build"
2625
"path/filepath"
2726
"testing"
@@ -53,11 +52,6 @@ import (
5352
// These tests use Ginkgo (BDD-style Go testing framework). Refer to
5453
// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
5554

56-
const (
57-
testDataPlaneNodeAffinityKey = "testDataPlaneNodeAffinityKey"
58-
testDataPlaneTolerationKey = "testDataPlaneTolerationKey"
59-
)
60-
6155
var cfg *rest.Config
6256
var k8sClient client.Client
6357
var testEnv *envtest.Environment
@@ -87,10 +81,6 @@ var _ = BeforeSuite(func() {
8781
}
8882

8983
viper.SetDefault(constant.CfgKeyCtrlrReconcileRetryDurationMS, 10)
90-
viper.Set(constant.CfgKeyDataPlaneTolerations,
91-
fmt.Sprintf("[{\"key\":\"%s\", \"operator\": \"Exists\", \"effect\": \"NoSchedule\"}]", testDataPlaneTolerationKey))
92-
viper.Set(constant.CfgKeyDataPlaneAffinity,
93-
fmt.Sprintf("{\"nodeAffinity\":{\"preferredDuringSchedulingIgnoredDuringExecution\":[{\"preference\":{\"matchExpressions\":[{\"key\":\"%s\",\"operator\":\"In\",\"values\":[\"true\"]}]},\"weight\":100}]}}", testDataPlaneNodeAffinityKey))
9484

9585
ctx, cancel = context.WithCancel(context.TODO())
9686
logger = logf.FromContext(ctx).WithValues()

controllers/operations/suite_test.go

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ package operations
2121

2222
import (
2323
"context"
24-
"fmt"
2524
"go/build"
2625
"path/filepath"
2726
"testing"
@@ -64,11 +63,6 @@ import (
6463
// These tests use Ginkgo (BDD-style Go testing framework). Refer to
6564
// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
6665

67-
const (
68-
testDataPlaneNodeAffinityKey = "testDataPlaneNodeAffinityKey"
69-
testDataPlaneTolerationKey = "testDataPlaneTolerationKey"
70-
)
71-
7266
var cfg *rest.Config
7367
var k8sClient client.Client
7468
var testEnv *envtest.Environment
@@ -99,10 +93,6 @@ var _ = BeforeSuite(func() {
9993
}
10094

10195
viper.SetDefault(constant.CfgKeyCtrlrReconcileRetryDurationMS, 10)
102-
viper.Set(constant.CfgKeyDataPlaneTolerations,
103-
fmt.Sprintf("[{\"key\":\"%s\", \"operator\": \"Exists\", \"effect\": \"NoSchedule\"}]", testDataPlaneTolerationKey))
104-
viper.Set(constant.CfgKeyDataPlaneAffinity,
105-
fmt.Sprintf("{\"nodeAffinity\":{\"preferredDuringSchedulingIgnoredDuringExecution\":[{\"preference\":{\"matchExpressions\":[{\"key\":\"%s\",\"operator\":\"In\",\"values\":[\"true\"]}]},\"weight\":100}]}}", testDataPlaneNodeAffinityKey))
10696

10797
ctx, cancel = context.WithCancel(context.TODO())
10898
logger = logf.FromContext(ctx).WithValues()

deploy/helm/templates/configmap.yaml

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,6 @@ metadata:
77
{{- include "kubeblocks.labels" . | nindent 4 }}
88
data:
99
config.yaml: |
10-
{{- with .Values.dataPlane }}
11-
# data plane tolerations
12-
DATA_PLANE_TOLERATIONS: {{ toJson .tolerations | squote }}
13-
14-
# data plane affinity
15-
DATA_PLANE_AFFINITY: {{ toJson .affinity | squote }}
16-
{{- end }}
17-
1810
# the default storage class name.
1911
DEFAULT_STORAGE_CLASS: {{ include "kubeblocks.defaultStorageClass" . | quote }}
2012

deploy/helm/values.yaml

Lines changed: 0 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -271,7 +271,6 @@ autoscaling:
271271
# targetMemoryUtilizationPercentage: 80
272272

273273

274-
275274
## @param nodeSelector
276275
##
277276
nodeSelector: {}
@@ -284,7 +283,6 @@ tolerations:
284283
value: "true"
285284
effect: NoSchedule
286285

287-
288286
## @param affinity
289287
##
290288
affinity:
@@ -297,25 +295,7 @@ affinity:
297295
operator: In
298296
values:
299297
- "true"
300-
## @param data plane settings
301-
##
302-
dataPlane:
303-
tolerations:
304-
- key: kb-data
305-
operator: Equal
306-
value: "true"
307-
effect: NoSchedule
308298

309-
affinity:
310-
nodeAffinity:
311-
preferredDuringSchedulingIgnoredDuringExecution:
312-
- weight: 100
313-
preference:
314-
matchExpressions:
315-
- key: kb-data
316-
operator: In
317-
values:
318-
- "true"
319299

320300
# dynamically replace instance's registry. ref: https://github.com/apecloud/kubeblocks/pull/8018
321301
# e.g.

pkg/constant/viper_config.go

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,6 @@ const (
4040
// addon charts config keys
4141
CfgAddonChartsImgPullPolicy = "KUBEBLOCKS_ADDON_CHARTS_IMAGE_PULL_POLICY"
4242

43-
// data plane config key
44-
CfgKeyDataPlaneTolerations = "DATA_PLANE_TOLERATIONS"
45-
CfgKeyDataPlaneAffinity = "DATA_PLANE_AFFINITY"
46-
4743
// storage config keys
4844
CfgKeyDefaultStorageClass = "DEFAULT_STORAGE_CLASS"
4945

pkg/controller/component/component.go

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,10 +56,6 @@ func GetClusterUID(comp *appsv1.Component) (string, error) {
5656

5757
// BuildComponent builds a new Component object from cluster component spec and definition.
5858
func BuildComponent(cluster *appsv1.Cluster, compSpec *appsv1.ClusterComponentSpec, labels, annotations map[string]string) (*appsv1.Component, error) {
59-
schedulingPolicy, err := scheduling.BuildSchedulingPolicy(cluster, compSpec)
60-
if err != nil {
61-
return nil, err
62-
}
6359
compBuilder := builder.NewComponentBuilder(cluster.Namespace, FullName(cluster.Name, compSpec.Name), compSpec.ComponentDef).
6460
AddAnnotations(constant.KubeBlocksGenerationKey, strconv.FormatInt(cluster.Generation, 10)).
6561
AddAnnotations(constant.CRDAPIVersionAnnotationKey, appsv1.GroupVersion.String()).
@@ -72,7 +68,7 @@ func BuildComponent(cluster *appsv1.Cluster, compSpec *appsv1.ClusterComponentSp
7268
SetLabels(compSpec.Labels).
7369
SetAnnotations(compSpec.Annotations).
7470
SetEnv(compSpec.Env).
75-
SetSchedulingPolicy(schedulingPolicy).
71+
SetSchedulingPolicy(scheduling.BuildSchedulingPolicy(cluster, compSpec)).
7672
SetDisableExporter(compSpec.DisableExporter).
7773
SetReplicas(compSpec.Replicas).
7874
SetResources(compSpec.Resources).

pkg/controller/scheduling/scheduling_utils.go

Lines changed: 5 additions & 121 deletions
Original file line numberDiff line numberDiff line change
@@ -20,15 +20,9 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
2020
package scheduling
2121

2222
import (
23-
"encoding/json"
24-
2523
corev1 "k8s.io/api/core/v1"
26-
"k8s.io/apimachinery/pkg/api/equality"
2724

2825
appsv1 "github.com/apecloud/kubeblocks/apis/apps/v1"
29-
"github.com/apecloud/kubeblocks/pkg/constant"
30-
intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil"
31-
viper "github.com/apecloud/kubeblocks/pkg/viperx"
3226
)
3327

3428
// ApplySchedulingPolicyToPodSpec overrides podSpec with schedulingPolicy, if schedulingPolicy is not nil
@@ -43,122 +37,12 @@ func ApplySchedulingPolicyToPodSpec(podSpec *corev1.PodSpec, schedulingPolicy *a
4337
}
4438
}
4539

46-
func BuildSchedulingPolicy(cluster *appsv1.Cluster, compSpec *appsv1.ClusterComponentSpec) (*appsv1.SchedulingPolicy, error) {
47-
if cluster.Spec.SchedulingPolicy != nil || (compSpec != nil && compSpec.SchedulingPolicy != nil) {
48-
return buildSchedulingPolicy(cluster, compSpec)
49-
}
50-
return nil, nil
51-
}
52-
53-
func buildSchedulingPolicy(cluster *appsv1.Cluster, compSpec *appsv1.ClusterComponentSpec) (*appsv1.SchedulingPolicy, error) {
54-
schedulingPolicy := cluster.Spec.SchedulingPolicy.DeepCopy()
55-
if compSpec != nil && compSpec.SchedulingPolicy != nil {
56-
schedulingPolicy = compSpec.SchedulingPolicy.DeepCopy()
57-
}
58-
59-
mergeGlobalAffinity := func() error {
60-
affinity, err := buildClusterWideAffinity()
61-
if err != nil {
62-
return err
63-
}
64-
schedulingPolicy.Affinity = MergeAffinity(schedulingPolicy.Affinity, affinity)
65-
return nil
66-
}
67-
68-
mergeGlobalTolerations := func() error {
69-
tolerations, err := buildClusterWideTolerations()
70-
if err != nil {
71-
return err
72-
}
73-
intctrlutil.MergeList(&schedulingPolicy.Tolerations, &tolerations, makeCmp[corev1.Toleration]())
40+
func BuildSchedulingPolicy(cluster *appsv1.Cluster, compSpec *appsv1.ClusterComponentSpec) *appsv1.SchedulingPolicy {
41+
if cluster.Spec.SchedulingPolicy == nil && (compSpec == nil || compSpec.SchedulingPolicy == nil) {
7442
return nil
7543
}
76-
77-
if err := mergeGlobalAffinity(); err != nil {
78-
return nil, err
79-
}
80-
if err := mergeGlobalTolerations(); err != nil {
81-
return nil, err
82-
}
83-
return schedulingPolicy, nil
84-
}
85-
86-
// buildClusterWideAffinity builds data plane affinity from global config
87-
func buildClusterWideAffinity() (*corev1.Affinity, error) {
88-
affinity := new(corev1.Affinity)
89-
if val := viper.GetString(constant.CfgKeyDataPlaneAffinity); val != "" {
90-
if err := json.Unmarshal([]byte(val), &affinity); err != nil {
91-
return nil, err
92-
}
93-
}
94-
return affinity, nil
95-
}
96-
97-
// buildClusterWideTolerations builds data plane tolerations from global config
98-
func buildClusterWideTolerations() ([]corev1.Toleration, error) {
99-
// build data plane tolerations from config
100-
var tolerations []corev1.Toleration
101-
if val := viper.GetString(constant.CfgKeyDataPlaneTolerations); val != "" {
102-
if err := json.Unmarshal([]byte(val), &tolerations); err != nil {
103-
return nil, err
104-
}
105-
}
106-
return tolerations, nil
107-
}
108-
109-
func makeCmp[E any]() func(E) func(E) bool {
110-
return func(a E) func(E) bool {
111-
return func(b E) bool {
112-
return equality.Semantic.DeepEqual(a, b)
113-
}
114-
}
115-
}
116-
117-
// MergeAffinity merges src to dst, return value is deepcopied
118-
// Items in src will overwrite items in dst, if possible.
119-
func MergeAffinity(src, dst *corev1.Affinity) *corev1.Affinity {
120-
if src == nil {
121-
return dst.DeepCopy()
122-
}
123-
124-
if dst == nil {
125-
return src.DeepCopy()
126-
}
127-
128-
rtn := dst.DeepCopy()
129-
130-
// Merge PodAffinity
131-
if src.PodAffinity != nil {
132-
if rtn.PodAffinity == nil {
133-
rtn.PodAffinity = &corev1.PodAffinity{}
134-
}
135-
intctrlutil.MergeList(&src.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution, &rtn.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution, makeCmp[corev1.PodAffinityTerm]())
136-
intctrlutil.MergeList(&src.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution, &rtn.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution, makeCmp[corev1.WeightedPodAffinityTerm]())
137-
}
138-
139-
// Merge PodAntiAffinity
140-
if src.PodAntiAffinity != nil {
141-
if rtn.PodAntiAffinity == nil {
142-
rtn.PodAntiAffinity = &corev1.PodAntiAffinity{}
143-
}
144-
intctrlutil.MergeList(&src.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, &rtn.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, makeCmp[corev1.PodAffinityTerm]())
145-
intctrlutil.MergeList(&src.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution, &rtn.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution, makeCmp[corev1.WeightedPodAffinityTerm]())
146-
}
147-
148-
// Merge NodeAffinity
149-
if src.NodeAffinity != nil {
150-
if rtn.NodeAffinity == nil {
151-
rtn.NodeAffinity = &corev1.NodeAffinity{}
152-
}
153-
if src.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil {
154-
if rtn.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil {
155-
rtn.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &corev1.NodeSelector{}
156-
}
157-
// FIXME: NodeSelectorTerms are ORed, this can be a problem
158-
intctrlutil.MergeList(&src.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, &rtn.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, makeCmp[corev1.NodeSelectorTerm]())
159-
}
160-
intctrlutil.MergeList(&src.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution, &rtn.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution, makeCmp[corev1.PreferredSchedulingTerm]())
44+
if compSpec != nil && compSpec.SchedulingPolicy != nil {
45+
return compSpec.SchedulingPolicy.DeepCopy()
16146
}
162-
163-
return rtn
47+
return cluster.Spec.SchedulingPolicy.DeepCopy()
16448
}

0 commit comments

Comments
 (0)