Skip to content

Commit dbc5e59

Browse files
joelanfordjoelanford
committed
multiNamespaceCache: support custom newCache funcs per namespace
Signed-off-by: Joe Lanford <[email protected]>
1 parent 62f0538 commit dbc5e59

File tree

2 files changed

+261
-16
lines changed

2 files changed

+261
-16
lines changed

‎pkg/cache/cache_test.go

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,192 @@ var _ = Describe("Multi-Namespace Informer Cache", func() {
121121
var _ = Describe("Informer Cache without DeepCopy", func() {
122122
CacheTest(cache.New, cache.Options{UnsafeDisableDeepCopyByObject: cache.DisableDeepCopyByObject{cache.ObjectAll{}: true}})
123123
})
124+
var _ = Describe("ByNamespace Cache", func() {
125+
defer GinkgoRecover()
126+
var (
127+
informerCache cache.Cache
128+
informerCacheCtx context.Context
129+
informerCacheCancel context.CancelFunc
130+
pod1a client.Object
131+
pod1b client.Object
132+
pod2a client.Object
133+
pod2b client.Object
134+
pod2c client.Object
135+
pod3a client.Object
136+
pod3b client.Object
137+
)
138+
BeforeEach(func() {
139+
informerCacheCtx, informerCacheCancel = context.WithCancel(context.Background())
140+
Expect(cfg).NotTo(BeNil())
141+
cl, err := client.New(cfg, client.Options{})
142+
Expect(err).NotTo(HaveOccurred())
143+
err = ensureNamespace(testNamespaceOne, cl)
144+
Expect(err).NotTo(HaveOccurred())
145+
err = ensureNamespace(testNamespaceTwo, cl)
146+
Expect(err).NotTo(HaveOccurred())
147+
err = ensureNamespace(testNamespaceThree, cl)
148+
Expect(err).NotTo(HaveOccurred())
149+
err = ensureNode(testNodeOne, cl)
150+
Expect(err).NotTo(HaveOccurred())
151+
// namespace 1 stuff
152+
pod1a = createPod("pod-1a", testNamespaceOne, corev1.RestartPolicyNever) // matches (everything matches)
153+
pod1b = createPodWithLabels("pod-1b", testNamespaceOne, corev1.RestartPolicyNever, map[string]string{"other-match": "true"}) // matches (everything matches)
154+
// namespace 2 stuff
155+
pod2a = createPodWithLabels("pod-2a", testNamespaceTwo, corev1.RestartPolicyNever, map[string]string{"ns2-match": "false"}) // no match (does not match ns2 label selector)
156+
pod2b = createPodWithLabels("pod-2b", testNamespaceTwo, corev1.RestartPolicyNever, map[string]string{"ns2-match": "true"}) // matches (matches ns2 label selector)
157+
pod2c = createPodWithLabels("pod-2c", testNamespaceTwo, corev1.RestartPolicyNever, map[string]string{"other-match": "true"}) // no match (does not match ns2 label selector)
158+
// namespace 3 stuff
159+
pod3a = createPodWithLabels("pod-3a", testNamespaceThree, corev1.RestartPolicyNever, map[string]string{"other-match": "false"}) // no match (does not match default cache label selector)
160+
pod3b = createPodWithLabels("pod-3b", testNamespaceThree, corev1.RestartPolicyNever, map[string]string{"other-match": "true"}) // matches (matches default cache label selector)
161+
By("creating the informer cache")
162+
informerCache, err = cache.BuilderByNamespace(cache.ByNamespaceOptions{
163+
NewNamespaceCaches: map[string]cache.NewCacheFunc{
164+
// Everything in ns1
165+
testNamespaceOne: cache.New,
166+
// Only things in ns2 with label "ns2-match"="true"
167+
testNamespaceTwo: cache.BuilderWithOptions(cache.Options{
168+
DefaultSelector: cache.ObjectSelector{
169+
Label: labels.Set{"ns2-match": "true"}.AsSelector(),
170+
},
171+
}),
172+
},
173+
// For all other namespaces, match "other-match"="true"
174+
NewDefaultNamespaceCache: cache.BuilderWithOptions(cache.Options{
175+
DefaultSelector: cache.ObjectSelector{
176+
Label: labels.Set{"other-match": "true"}.AsSelector(),
177+
},
178+
}),
179+
// For cluster-scoped objects, only match metadata.name = "test-node-1"
180+
NewClusterCache: cache.BuilderWithOptions(cache.Options{
181+
DefaultSelector: cache.ObjectSelector{
182+
Field: fields.OneTermEqualSelector("metadata.name", testNodeOne),
183+
},
184+
}),
185+
})(cfg, cache.Options{})
186+
Expect(err).NotTo(HaveOccurred())
187+
By("running the cache and waiting for it to sync")
188+
// pass as an arg so that we don't race between close and re-assign
189+
go func(ctx context.Context) {
190+
defer GinkgoRecover()
191+
Expect(informerCache.Start(ctx)).To(Succeed())
192+
}(informerCacheCtx)
193+
Expect(informerCache.WaitForCacheSync(informerCacheCtx)).To(BeTrue())
194+
})
195+
Describe("Get", func() {
196+
It("should get an item from a namespace cache", func() {
197+
pod := &corev1.Pod{}
198+
err := informerCache.Get(informerCacheCtx, client.ObjectKeyFromObject(pod1a), pod)
199+
Expect(err).NotTo(HaveOccurred())
200+
})
201+
It("should get an item from the default namespace cache", func() {
202+
pod := &corev1.Pod{}
203+
err := informerCache.Get(informerCacheCtx, client.ObjectKeyFromObject(pod3b), pod)
204+
Expect(err).NotTo(HaveOccurred())
205+
})
206+
It("should get a cluster-scoped item", func() {
207+
node := &corev1.Node{}
208+
err := informerCache.Get(informerCacheCtx, client.ObjectKey{Name: testNodeOne}, node)
209+
Expect(err).NotTo(HaveOccurred())
210+
})
211+
It("should not find an item from a namespace-specific cache if it is not matched", func() {
212+
pod := &corev1.Pod{}
213+
err := informerCache.Get(informerCacheCtx, client.ObjectKeyFromObject(pod2a), pod)
214+
Expect(apierrors.IsNotFound(err)).To(BeTrue())
215+
})
216+
It("should not find an item from the default namespace cache if it is not matched", func() {
217+
pod := &corev1.Pod{}
218+
err := informerCache.Get(informerCacheCtx, client.ObjectKeyFromObject(pod3a), pod)
219+
Expect(apierrors.IsNotFound(err)).To(BeTrue())
220+
})
221+
It("should not find an item at the cluster-scope if it is not matched", func() {
222+
ns := &corev1.Namespace{}
223+
err := informerCache.Get(informerCacheCtx, client.ObjectKey{Name: testNamespaceOne}, ns)
224+
Expect(apierrors.IsNotFound(err)).To(BeTrue())
225+
})
226+
})
227+
Describe("List", func() {
228+
When("Request is cluster-scoped", func() {
229+
It("Should list all pods and find exactly four", func() {
230+
var pods corev1.PodList
231+
err := informerCache.List(informerCacheCtx, &pods)
232+
Expect(err).NotTo(HaveOccurred())
233+
sort.Slice(pods.Items, func(i, j int) bool {
234+
if pods.Items[i].Namespace != pods.Items[j].Namespace {
235+
return pods.Items[i].Namespace < pods.Items[j].Namespace
236+
}
237+
return pods.Items[i].Name < pods.Items[j].Name
238+
})
239+
Expect(pods.Items).To(HaveLen(4))
240+
Expect(pods.Items[0].Namespace).To(Equal(testNamespaceOne))
241+
Expect(pods.Items[0].Name).To(Equal("pod-1a"))
242+
Expect(pods.Items[1].Namespace).To(Equal(testNamespaceOne))
243+
Expect(pods.Items[1].Name).To(Equal("pod-1b"))
244+
Expect(pods.Items[2].Namespace).To(Equal(testNamespaceTwo))
245+
Expect(pods.Items[2].Name).To(Equal("pod-2b"))
246+
Expect(pods.Items[3].Namespace).To(Equal(testNamespaceThree))
247+
Expect(pods.Items[3].Name).To(Equal("pod-3b"))
248+
})
249+
It("Should list nodes and find exactly one", func() {
250+
var nodes corev1.NodeList
251+
err := informerCache.List(informerCacheCtx, &nodes)
252+
Expect(err).NotTo(HaveOccurred())
253+
Expect(nodes.Items).To(HaveLen(1))
254+
Expect(nodes.Items[0].Namespace).To(Equal(""))
255+
Expect(nodes.Items[0].Name).To(Equal(testNodeOne))
256+
})
257+
It("Should list namespaces and find none", func() {
258+
var namespaces corev1.NamespaceList
259+
err := informerCache.List(informerCacheCtx, &namespaces)
260+
Expect(err).NotTo(HaveOccurred())
261+
Expect(namespaces.Items).To(HaveLen(0))
262+
})
263+
})
264+
When("Request is namespace-scoped", func() {
265+
It("Should list pods in namespace one", func() {
266+
var pods corev1.PodList
267+
err := informerCache.List(informerCacheCtx, &pods, client.InNamespace(testNamespaceOne))
268+
Expect(err).NotTo(HaveOccurred())
269+
sort.Slice(pods.Items, func(i, j int) bool {
270+
if pods.Items[i].Namespace != pods.Items[j].Namespace {
271+
return pods.Items[i].Namespace < pods.Items[j].Namespace
272+
}
273+
return pods.Items[i].Name < pods.Items[j].Name
274+
})
275+
Expect(pods.Items).To(HaveLen(2))
276+
Expect(pods.Items[0].Namespace).To(Equal(testNamespaceOne))
277+
Expect(pods.Items[0].Name).To(Equal("pod-1a"))
278+
Expect(pods.Items[1].Namespace).To(Equal(testNamespaceOne))
279+
Expect(pods.Items[1].Name).To(Equal("pod-1b"))
280+
})
281+
It("Should list pods in namespace two", func() {
282+
var pods corev1.PodList
283+
err := informerCache.List(informerCacheCtx, &pods, client.InNamespace(testNamespaceTwo))
284+
Expect(err).NotTo(HaveOccurred())
285+
Expect(pods.Items).To(HaveLen(1))
286+
Expect(pods.Items[0].Namespace).To(Equal(testNamespaceTwo))
287+
Expect(pods.Items[0].Name).To(Equal("pod-2b"))
288+
})
289+
It("Should list pods in namespace three", func() {
290+
var pods corev1.PodList
291+
err := informerCache.List(informerCacheCtx, &pods, client.InNamespace(testNamespaceThree))
292+
Expect(err).NotTo(HaveOccurred())
293+
Expect(pods.Items).To(HaveLen(1))
294+
Expect(pods.Items[0].Namespace).To(Equal(testNamespaceThree))
295+
Expect(pods.Items[0].Name).To(Equal("pod-3b"))
296+
})
297+
})
298+
})
299+
AfterEach(func() {
300+
deletePod(pod1a)
301+
deletePod(pod1b)
302+
deletePod(pod2a)
303+
deletePod(pod2b)
304+
deletePod(pod2c)
305+
deletePod(pod3a)
306+
deletePod(pod3b)
307+
informerCacheCancel()
308+
})
309+
})
124310

125311
var _ = Describe("Cache with transformers", func() {
126312
var (

‎pkg/cache/multi_namespace_cache.go

Lines changed: 75 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,12 @@ import (
2323

2424
corev1 "k8s.io/api/core/v1"
2525
apimeta "k8s.io/apimachinery/pkg/api/meta"
26+
"k8s.io/apimachinery/pkg/fields"
2627
"k8s.io/apimachinery/pkg/runtime"
2728
"k8s.io/apimachinery/pkg/runtime/schema"
2829
"k8s.io/client-go/rest"
2930
toolscache "k8s.io/client-go/tools/cache"
31+
3032
"sigs.k8s.io/controller-runtime/pkg/client"
3133
"sigs.k8s.io/controller-runtime/pkg/internal/objectutil"
3234
)
@@ -37,39 +39,88 @@ type NewCacheFunc func(config *rest.Config, opts Options) (Cache, error)
3739
// a new global namespaced cache to handle cluster scoped resources.
3840
const globalCache = "_cluster-scope"
3941

40-
// MultiNamespacedCacheBuilder - Builder function to create a new multi-namespaced cache.
41-
// This will scope the cache to a list of namespaces. Listing for all namespaces
42-
// will list for all the namespaces that this knows about. By default this will create
43-
// a global cache for cluster scoped resource. Note that this is not intended
44-
// to be used for excluding namespaces, this is better done via a Predicate. Also note that
45-
// you may face performance issues when using this with a high number of namespaces.
46-
func MultiNamespacedCacheBuilder(namespaces []string) NewCacheFunc {
42+
// ByNamespaceOptions is used to configure the functions used to create caches
43+
// on a per-namespace basis.
44+
type ByNamespaceOptions struct {
45+
NewNamespaceCaches map[string]NewCacheFunc
46+
NewClusterCache NewCacheFunc
47+
NewDefaultNamespaceCache NewCacheFunc
48+
}
49+
50+
// BuilderByNamespace builds a composite cache that delegates to per-namespace
51+
// caches built according to the passed ByNamespaceOptions. If NewDefaultNamespaceCache
52+
// is defined, it will be used as a catch-all for objects not in the namespaces defined in
53+
// NewNamespaceCaches. The default namespace cache is automatically configured with extra
54+
// field selectors to avoid duplicate caching of objects between namespace-specific caches
55+
// and this catch-all cache. NewClusterCache is used to create a cache for cluster-scoped
56+
// objects. If it is undefined, a default cache will be created using the New function.
57+
func BuilderByNamespace(byNamespaceOpts ByNamespaceOptions) NewCacheFunc {
4758
return func(config *rest.Config, opts Options) (Cache, error) {
4859
opts, err := defaultOpts(config, opts)
4960
if err != nil {
5061
return nil, err
5162
}
5263

53-
caches := map[string]Cache{}
54-
55-
// create a cache for cluster scoped resources
56-
gCache, err := New(config, opts)
64+
if byNamespaceOpts.NewClusterCache == nil {
65+
byNamespaceOpts.NewClusterCache = New
66+
}
67+
clusterCache, err := byNamespaceOpts.NewClusterCache(config, opts)
5768
if err != nil {
58-
return nil, fmt.Errorf("error creating global cache: %w", err)
69+
return nil, err
5970
}
6071

61-
for _, ns := range namespaces {
72+
nsToCache := map[string]Cache{}
73+
if byNamespaceOpts.NewDefaultNamespaceCache != nil {
74+
defaultNamespaceCache, err := byNamespaceOpts.NewDefaultNamespaceCache(config, ignoreNamespaces(opts, byNamespaceOpts.NewNamespaceCaches))
75+
if err != nil {
76+
return nil, err
77+
}
78+
nsToCache[corev1.NamespaceAll] = defaultNamespaceCache
79+
}
80+
81+
for ns, newCacheFunc := range byNamespaceOpts.NewNamespaceCaches {
6282
opts.Namespace = ns
63-
c, err := New(config, opts)
83+
nsToCache[ns], err = newCacheFunc(config, opts)
6484
if err != nil {
6585
return nil, err
6686
}
67-
caches[ns] = c
6887
}
69-
return &multiNamespaceCache{namespaceToCache: caches, Scheme: opts.Scheme, RESTMapper: opts.Mapper, clusterCache: gCache}, nil
88+
89+
return &multiNamespaceCache{
90+
namespaceToCache: nsToCache,
91+
clusterCache: clusterCache,
92+
RESTMapper: opts.Mapper,
93+
Scheme: opts.Scheme,
94+
}, nil
7095
}
7196
}
7297

98+
func ignoreNamespaces(opts Options, newObjectCaches map[string]NewCacheFunc) Options {
99+
fieldSelectors := []fields.Selector{}
100+
if opts.DefaultSelector.Field != nil {
101+
fieldSelectors = append(fieldSelectors, opts.DefaultSelector.Field)
102+
}
103+
for ns := range newObjectCaches {
104+
fieldSelectors = append(fieldSelectors, fields.OneTermNotEqualSelector("metadata.namespace", ns))
105+
}
106+
opts.DefaultSelector.Field = fields.AndSelectors(fieldSelectors...)
107+
return opts
108+
}
109+
110+
// MultiNamespacedCacheBuilder - Builder function to create a new multi-namespaced cache.
111+
// This will scope the cache to a list of namespaces. Listing for all namespaces
112+
// will list for all the namespaces that this knows about. By default this will create
113+
// a global cache for cluster scoped resource. Note that this is not intended
114+
// to be used for excluding namespaces, this is better done via a Predicate. Also note that
115+
// you may face performance issues when using this with a high number of namespaces.
116+
func MultiNamespacedCacheBuilder(namespaces []string) NewCacheFunc {
117+
byNamespaceOpts := ByNamespaceOptions{NewNamespaceCaches: map[string]NewCacheFunc{}}
118+
for _, ns := range namespaces {
119+
byNamespaceOpts.NewNamespaceCaches[ns] = New
120+
}
121+
return BuilderByNamespace(byNamespaceOpts)
122+
}
123+
73124
// multiNamespaceCache knows how to handle multiple namespaced caches
74125
// Use this feature when scoping permissions for your
75126
// operator to a list of namespaces instead of watching every namespace
@@ -212,6 +263,10 @@ func (c *multiNamespaceCache) Get(ctx context.Context, key client.ObjectKey, obj
212263
}
213264

214265
cache, ok := c.namespaceToCache[key.Namespace]
266+
if !ok {
267+
// Use the default/catch-all namespace cache if we have one.
268+
cache, ok = c.namespaceToCache[corev1.NamespaceAll]
269+
}
215270
if !ok {
216271
return fmt.Errorf("unable to get: %v because of unknown namespace for the cache", key)
217272
}
@@ -235,6 +290,10 @@ func (c *multiNamespaceCache) List(ctx context.Context, list client.ObjectList,
235290

236291
if listOpts.Namespace != corev1.NamespaceAll {
237292
cache, ok := c.namespaceToCache[listOpts.Namespace]
293+
if !ok {
294+
// Use the default/catch-all namespace cache if we have one.
295+
cache, ok = c.namespaceToCache[corev1.NamespaceAll]
296+
}
238297
if !ok {
239298
return fmt.Errorf("unable to get: %v because of unknown namespace for the cache", listOpts.Namespace)
240299
}

0 commit comments

Comments
 (0)