Skip to content

Commit 2fcc4c0

Browse files
committed
Update bpfd deployment
Before this patch we deployed the bpf daemon manually via kustomize... However this method couldn't be packaged into a bundle via the operator SDK (see operator-framework/operator-lifecycle-manager#1022 (comment)) This commit changes the way its deployed to have the bpfd-operator itself reconcile/deploy the bpfd daemon. Mainly it now ensures the daemon is configured correctly via the bpfd configmap and deployed. Signed-off-by: Andrew Stoycos <[email protected]>
1 parent 9a7aa7c commit 2fcc4c0

File tree

14 files changed

+374
-506
lines changed

14 files changed

+374
-506
lines changed

bpfd-operator/cmd/bpfd-operator/main.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ import (
3232
"sigs.k8s.io/controller-runtime/pkg/log/zap"
3333

3434
bpfdiov1alpha1 "github.com/redhat-et/bpfd/bpfd-operator/api/v1alpha1"
35-
"github.com/redhat-et/bpfd/bpfd-operator/pkg/bpfd-operator"
35+
bpfdoperator "github.com/redhat-et/bpfd/bpfd-operator/pkg/bpfd-operator"
3636
//+kubebuilder:scaffold:imports
3737
)
3838

bpfd-operator/config/bpfd-deployment/config.yaml

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,17 @@ metadata:
44
name: config
55
namespace: kube-system
66
data:
7+
## Can be configured at runtime
8+
bpfd.namespace: bpfd
9+
bpfd.agent.image: quay.io/bpfd/bpfd-agent:main
10+
bpfd.image: quay.io/bpfd/bpfd:main
11+
bpfd.log.level: "debug"
12+
## Must be configured at startup
713
bpfd.toml: |
814
[tls] # REQUIRED
915
ca_cert = "/etc/bpfd/certs/ca/ca.crt"
1016
cert = "/etc/bpfd/certs/bpfd/tls.crt"
1117
key = "/etc/bpfd/certs/bpfd/tls.key"
1218
client_cert = "/etc/bpfd/certs/bpfd-client/tls.crt"
1319
client_key = "/etc/bpfd/certs/bpfd-client/tls.key"
14-
bpfd.log.level: "debug"
1520

bpfd-operator/config/bpfd-deployment/kustomization.yaml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ resources:
22
- cert-issuer.yaml
33
- config.yaml
44
- certs.yaml
5-
- daemonset.yaml
65
apiVersion: kustomize.config.k8s.io/v1beta1
76
kind: Kustomization
87
images:

bpfd-operator/config/default/kustomization.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,5 +42,5 @@ kind: Kustomization
4242
resources:
4343
- ../crd
4444
- ../rbac
45-
- ../bpfd-operator-deployment
4645
- ../bpfd-deployment
46+
- ../bpfd-operator-deployment

bpfd-operator/config/manifests/kustomization.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# used to generate the 'manifests/' directory in a bundle.
33
resources:
44
- bases/bpfd-operator.clusterserviceversion.yaml
5-
- ../default
5+
- ../openshift
66
- ../samples
77
- ../scorecard
88

bpfd-operator/config/rbac/bpfd-operator-role/role.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,14 @@ rules:
5151
- get
5252
- list
5353
- watch
54+
- apiGroups:
55+
- ""
56+
resources:
57+
- configmaps
58+
verbs:
59+
- get
60+
- list
61+
- watch
5462
- apiGroups:
5563
- ""
5664
resources:

bpfd-operator/go.mod

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ require (
77
github.com/pelletier/go-toml v1.9.5
88
github.com/redhat-et/bpfd/clients/gobpfd v0.0.0-20221122124347-10ccb612fd2a
99
google.golang.org/grpc v1.51.0
10-
gopkg.in/yaml.v2 v2.4.0
1110
k8s.io/api v0.26.0
1211
k8s.io/apimachinery v0.26.0
1312
k8s.io/client-go v0.26.0
@@ -24,7 +23,7 @@ require (
2423
github.com/go-logr/zapr v1.2.3 // indirect
2524
github.com/go-openapi/jsonpointer v0.19.5 // indirect
2625
github.com/go-openapi/jsonreference v0.20.0 // indirect
27-
github.com/go-openapi/swag v0.19.14 // indirect
26+
github.com/go-openapi/swag v0.22.3 // indirect
2827
github.com/gogo/protobuf v1.3.2 // indirect
2928
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
3029
github.com/golang/protobuf v1.5.2 // indirect
@@ -35,7 +34,8 @@ require (
3534
github.com/imdario/mergo v0.3.12 // indirect
3635
github.com/josharian/intern v1.0.0 // indirect
3736
github.com/json-iterator/go v1.1.12 // indirect
38-
github.com/mailru/easyjson v0.7.6 // indirect
37+
github.com/kr/pretty v0.3.0 // indirect
38+
github.com/mailru/easyjson v0.7.7 // indirect
3939
github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect
4040
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
4141
github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -60,6 +60,7 @@ require (
6060
google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
6161
google.golang.org/protobuf v1.28.1 // indirect
6262
gopkg.in/inf.v0 v0.9.1 // indirect
63+
gopkg.in/yaml.v2 v2.4.0 // indirect
6364
gopkg.in/yaml.v3 v3.0.1 // indirect
6465
k8s.io/apiextensions-apiserver v0.26.0 // indirect
6566
k8s.io/component-base v0.26.0 // indirect

bpfd-operator/go.sum

Lines changed: 7 additions & 388 deletions
Large diffs are not rendered by default.

bpfd-operator/internal/bpfd-client-helpers.go

Lines changed: 49 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,15 +29,22 @@ import (
2929
bpfdiov1alpha1 "github.com/redhat-et/bpfd/bpfd-operator/api/v1alpha1"
3030
gobpfd "github.com/redhat-et/bpfd/clients/gobpfd/v1"
3131
"google.golang.org/grpc/credentials"
32+
appsv1 "k8s.io/api/apps/v1"
33+
corev1 "k8s.io/api/core/v1"
3234
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
35+
"k8s.io/client-go/kubernetes/scheme"
36+
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
3337
)
3438

3539
const (
36-
bpfdMapFs = "/run/bpfd/fs/maps"
37-
DefaultConfigPath = " /etc/bpfd/bpfd.toml"
38-
DefaultRootCaPath = "/etc/bpfd/certs/ca/ca.crt"
39-
DefaultClientCertPath = "/etc/bpfd/certs/bpfd-client/tls.crt"
40-
DefaultClientKeyPath = "/etc/bpfd/certs/bpfd-client/tls.key"
40+
BpfdDsName = "bpfd-daemon"
41+
BpfdConfigName = "bpfd-config"
42+
BpfdDaemonManifestPath = "./config/bpfd-deployment/daemonset.yaml"
43+
bpfdMapFs = "/run/bpfd/fs/maps"
44+
DefaultConfigPath = " /etc/bpfd/bpfd.toml"
45+
DefaultRootCaPath = "/etc/bpfd/certs/ca/ca.crt"
46+
DefaultClientCertPath = "/etc/bpfd/certs/bpfd-client/tls.crt"
47+
DefaultClientKeyPath = "/etc/bpfd/certs/bpfd-client/tls.key"
4148
)
4249

4350
type Tls struct {
@@ -280,3 +287,40 @@ func AttachConversion(attachment BpfdAttachType) *bpfdiov1alpha1.BpfProgramAttac
280287

281288
panic("Attachment Type is unknown")
282289
}
290+
291+
func LoadAndConfigureBpfdDs(config *corev1.ConfigMap) *appsv1.DaemonSet {
292+
// Load static bpfd deployment from disk
293+
file, err := os.Open(BpfdDaemonManifestPath)
294+
if err != nil {
295+
panic(err)
296+
}
297+
298+
b, err := ioutil.ReadAll(file)
299+
if err != nil {
300+
panic(err)
301+
}
302+
303+
decode := scheme.Codecs.UniversalDeserializer().Decode
304+
obj, _, _ := decode(b, nil, nil)
305+
306+
staticBpfdDeployment := obj.(*appsv1.DaemonSet)
307+
308+
// Runtime Configurable fields
309+
bpfdNamespace := config.Data["bpfd.namespace"]
310+
bpfdImage := config.Data["bpfd.image"]
311+
bpfdAgentImage := config.Data["bpfd.agent.image"]
312+
bpfdLogLevel := config.Data["bpfd.log.level"]
313+
314+
// Annotate the log level on the ds so we get automatic restarts on changes.
315+
if staticBpfdDeployment.Spec.Template.ObjectMeta.Annotations == nil {
316+
staticBpfdDeployment.Spec.Template.ObjectMeta.Annotations = make(map[string]string)
317+
}
318+
staticBpfdDeployment.Spec.Template.ObjectMeta.Annotations["bpfd.io.bpfd.loglevel"] = bpfdLogLevel
319+
staticBpfdDeployment.Name = "bpfd-daemon"
320+
staticBpfdDeployment.Namespace = bpfdNamespace
321+
staticBpfdDeployment.Spec.Template.Spec.Containers[0].Image = bpfdImage
322+
staticBpfdDeployment.Spec.Template.Spec.Containers[1].Image = bpfdAgentImage
323+
controllerutil.AddFinalizer(staticBpfdDeployment, "bpfd.io.operator/finalizer")
324+
325+
return staticBpfdDeployment
326+
}

bpfd-operator/pkg/bpfd-agent/bpfd-agent.go

Lines changed: 26 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ type bpfProgramConditionType string
6262

6363
const (
6464
BpfdAgentFinalizer = "bpfd.io.agent/finalizer"
65-
retryDurationAgent = 10 * time.Second
65+
retryDurationAgent = 5 * time.Second
6666
BpfProgCondLoaded bpfProgramConditionType = "Loaded"
6767
BpfProgCondNotLoaded bpfProgramConditionType = "NotLoaded"
6868
BpfProgCondNotUnloaded bpfProgramConditionType = "NotUnLoaded"
@@ -115,10 +115,6 @@ func (b bpfProgramConditionType) Condition() metav1.Condition {
115115

116116
// Reconcile is part of the main kubernetes reconciliation loop which aims to
117117
// move the current state of the cluster closer to the desired state.
118-
// TODO(user): Modify the Reconcile function to compare the state specified by
119-
// the BpfProgram object against the actual cluster state, and then
120-
// perform operations to make the cluster state reflect the state specified by
121-
// the user.
122118
// This should be called in the following scenarios
123119
// 1. A new BpfProgramConfig Object is created
124120
// 2. An BpfProgramConfig Object is Updated (i.e one of the following fields change
@@ -129,7 +125,7 @@ func (b bpfProgramConditionType) Condition() metav1.Condition {
129125
//
130126
// 3. Our NodeLabels are updated and the Node is no longer selected by an BpfProgramConfig
131127
//
132-
// 4. An bpfProgramCongfig Object is deleted
128+
// 4. An bpfProgramConfig Object is deleted
133129
// For more details, check Reconcile and its Result here:
134130
// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/reconcile
135131
func (r *BpfProgramReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
@@ -174,7 +170,7 @@ func (r *BpfProgramReconciler) Reconcile(ctx context.Context, req ctrl.Request)
174170
// Reconcile every BpfProgramConfig Object
175171
// note: This doesn't necessarily result in any extra grpc calls to bpfd
176172
for _, BpfProgramConfig := range BpfProgramConfigs.Items {
177-
retry, err := r.reconcilBpfProgramConfig(ctx, &BpfProgramConfig, ourNode, existingConfigs)
173+
retry, err := r.reconcileBpfProgramConfig(ctx, &BpfProgramConfig, ourNode, existingConfigs)
178174
if err != nil {
179175
r.Logger.Error(err, "Reconciling BpfProgramConfig Failed", "BpfProgramConfigName", BpfProgramConfig.Name, "Retrying", retry)
180176
return ctrl.Result{Requeue: retry, RequeueAfter: retryDurationAgent}, nil
@@ -184,9 +180,9 @@ func (r *BpfProgramReconciler) Reconcile(ctx context.Context, req ctrl.Request)
184180
return ctrl.Result{Requeue: false}, nil
185181
}
186182

187-
// reconcilBpfProgramConfig reconciles the existing node state to the user intent
183+
// reconcileBpfProgramConfig reconciles the existing node state to the user intent
188184
// within a single BpfProgramConfig Object.
189-
func (r *BpfProgramReconciler) reconcilBpfProgramConfig(ctx context.Context,
185+
func (r *BpfProgramReconciler) reconcileBpfProgramConfig(ctx context.Context,
190186
BpfProgramConfig *bpfdiov1alpha1.BpfProgramConfig,
191187
ourNode *v1.Node,
192188
nodeState map[string]internal.ExistingReq) (bool, error) {
@@ -348,17 +344,14 @@ func (r *BpfProgramReconciler) reconcilBpfProgramConfig(ctx context.Context,
348344
Maps: maps,
349345
}
350346

351-
r.Logger.V(1).Info("Updating programs", "Programs", bpfProgram.Spec.Programs)
347+
r.Logger.V(1).Info("Updating programs from nodestate", "Programs", bpfProgram.Spec.Programs)
352348
// Update bpfProgram once successfully loaded
353349
if err = r.Update(ctx, bpfProgram, &client.UpdateOptions{}); err != nil {
354350
return false, fmt.Errorf("failed to create bpfProgram object: %v",
355351
err)
356352
}
357353

358354
r.updateStatus(ctx, bpfProgram, BpfProgCondLoaded)
359-
360-
r.Logger.Info("Updated Programs on bpfProgram Object from nodeState",
361-
"bpfProgramName", bpfProgram.Name)
362355
} else {
363356
// Program exists and bpfProgram K8s Object is up to date
364357
r.Logger.Info("Ignoring Object Change nothing to reconcile on node")
@@ -390,10 +383,10 @@ func (r *BpfProgramReconciler) loadBpfdProgram(ctx context.Context, loadRequest
390383
Maps: maps,
391384
}
392385

393-
r.Logger.V(1).Info("Updating programs", "Programs", prog.Spec.Programs)
386+
r.Logger.V(1).Info("updating programs after load", "Programs", prog.Spec.Programs)
394387
// Update bpfProgram once successfully loaded
395388
if err = r.Update(ctx, prog, &client.UpdateOptions{}); err != nil {
396-
return false, fmt.Errorf("failed to create bpfProgram object: %v",
389+
return false, fmt.Errorf("failed to update bpfProgram programs: %v",
397390
err)
398391
}
399392
return false, nil
@@ -416,6 +409,14 @@ func (r *BpfProgramReconciler) unloadBpfdProgram(ctx context.Context, prog *bpfd
416409
return true, fmt.Errorf("failed to unload bpfProgram via bpfd: %v",
417410
err)
418411
}
412+
delete(prog.Spec.Programs, uuid)
413+
}
414+
415+
r.Logger.V(1).Info("updating programs after unload", "Programs", prog.Spec.Programs)
416+
// Update bpfProgram once successfully unloaded
417+
if err := r.Update(ctx, prog, &client.UpdateOptions{}); err != nil {
418+
return false, fmt.Errorf("failed to update bpfProgram programs: %v",
419+
err)
419420
}
420421

421422
return false, nil
@@ -459,11 +460,20 @@ func (r *BpfProgramReconciler) SetupWithManager(mgr ctrl.Manager) error {
459460
Complete(r)
460461
}
461462

462-
// Only return node updates for our node
463+
// Only return node updates for our node (all events)
463464
func nodePredicate(nodeName string) predicate.Funcs {
464465
return predicate.Funcs{
465466
GenericFunc: func(e event.GenericEvent) bool {
466467
return e.Object.GetLabels()["kubernetes.io/hostname"] == nodeName
467468
},
469+
CreateFunc: func(e event.CreateEvent) bool {
470+
return e.Object.GetLabels()["kubernetes.io/hostname"] == nodeName
471+
},
472+
UpdateFunc: func(e event.UpdateEvent) bool {
473+
return e.ObjectNew.GetLabels()["kubernetes.io/hostname"] == nodeName
474+
},
475+
DeleteFunc: func(e event.DeleteEvent) bool {
476+
return e.Object.GetLabels()["kubernetes.io/hostname"] == nodeName
477+
},
468478
}
469479
}

0 commit comments

Comments
 (0)