MULTIARCH-4989: Remove the kube-rbac-proxy

Images provided under gcr.io/kubebuilder/ will be unavailable from March 18, 2025.
Projects initialized with Kubebuilder versions v3.14 or lower utilize gcr.io/kubebuilder/kube-rbac-proxy to protect the metrics endpoint.

Following the work in kubernetes-sigs/kubebuilder#4003, this commit removes the kube-rbac-proxy container and let the main container of the controller expose the metrics via HTTPS and by using the WithAuthenticatoinAndAuthorization filter.

This also includes a minor fix in BuildService escaped during the resolution of some conflicts during a rebase.

Related to kubernetes-sigs/kubebuilder#3871

Author: aleskandro

bundle/manifests/multiarch-tuning-operator-controller-manager-service_v1_service.yaml

@@ -5,8 +5,12 @@ metadata:
   name: multiarch-tuning-operator-controller-manager-service
 spec:
   ports:
-  - port: 443
+  - name: webhook
+    port: 443
     targetPort: 9443
+  - name: metrics
+    port: 8443
+    targetPort: 8443
   selector:
     control-plane: controller-manager
 status:

bundle/manifests/multiarch-tuning-operator.clusterserviceversion.yaml

@@ -27,7 +27,7 @@ metadata:
     categories: OpenShift Optional, Other
     console.openshift.io/disable-operand-delete: "false"
     containerImage: registry.ci.openshift.org/origin/multiarch-tuning-operator:main
-    createdAt: "2024-08-27T22:08:50Z"
+    createdAt: "2024-09-05T20:29:54Z"
     features.operators.openshift.io/cnf: "false"
     features.operators.openshift.io/cni: "false"
     features.operators.openshift.io/csi: "false"
@@ -350,7 +350,7 @@ spec:
               containers:
               - args:
                 - --health-probe-bind-address=:8081
-                - --metrics-bind-address=127.0.0.1:8080
+                - --metrics-bind-address=:8443
                 - --leader-elect
                 - --enable-operator
                 command:
@@ -373,6 +373,13 @@ spec:
                   initialDelaySeconds: 15
                   periodSeconds: 20
                 name: manager
+                ports:
+                - containerPort: 8081
+                  name: health
+                  protocol: TCP
+                - containerPort: 8443
+                  name: https
+                  protocol: TCP
                 readinessProbe:
                   httpGet:
                     path: /readyz
@@ -395,26 +402,6 @@ spec:
                 - mountPath: /etc/ssl/certs/
                   name: ca-projected-volume
                   readOnly: true
-              - args:
-                - --secure-listen-address=0.0.0.0:8443
-                - --upstream=http://127.0.0.1:8080/
-                - --logtostderr=true
-                - --v=0
-                image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.1@sha256:d4883d7c622683b3319b5e6b3a7edfbf2594c18060131a8bf64504805f875522
-                name: kube-rbac-proxy
-                ports:
-                - containerPort: 8443
-                  name: https
-                  protocol: TCP
-                resources:
-                  requests:
-                    cpu: 10m
-                    memory: 64Mi
-                securityContext:
-                  allowPrivilegeEscalation: false
-                  capabilities:
-                    drop:
-                    - ALL
               priorityClassName: system-cluster-critical
               securityContext:
                 runAsNonRoot: true

config/default/kustomization.yaml

@@ -25,14 +25,6 @@ bases:
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
 #- ../prometheus
 
-patchesStrategicMerge:
-# Protect the /metrics endpoint by putting it behind auth.
-# If you want your controller-manager to expose the /metrics
-# endpoint w/o any authn/z, please comment the following line.
-- manager_auth_proxy_patch.yaml
-- manager_config_patch.yaml
-
-
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml
 #- manager_webhook_patch.yaml

config/default/manager_auth_proxy_patch.yaml

@@ -8,26 +8,6 @@ metadata:
 spec:
   template:
     spec:
-      securityContext:
-        runAsNonRoot: true
-        seccompProfile:
-          type: RuntimeDefault
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                - key: kubernetes.io/arch
-                  operator: In
-                  values:
-                    - amd64
-                    - arm64
-                    - ppc64le
-                    - s390x
-                - key: kubernetes.io/os
-                  operator: In
-                  values:
-                    - linux
       containers:
       - name: kube-rbac-proxy
         securityContext:
@@ -49,18 +29,3 @@ spec:
           requests:
             cpu: 10m
             memory: 64Mi
-      - name: manager
-        env:
-          - name: NAMESPACE
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.namespace
-          - name: IMAGE
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.annotations['multiarch.openshift.io/image']
-        args:
-        - "--health-probe-bind-address=:8081"
-        - "--metrics-bind-address=127.0.0.1:8080"
-        - "--leader-elect"
-        - "--enable-operator"

config/default/manager_config_patch.yaml

@@ -41,43 +41,54 @@ spec:
       labels:
         control-plane: controller-manager
     spec:
-      # TODO(user): Uncomment the following code to configure the nodeAffinity expression
-      # according to the platforms which are supported by your solution.
-      # It is considered best practice to support multiple architectures. You can
-      # build your manager image using the makefile target docker-buildx.
-      # affinity:
-      #   nodeAffinity:
-      #     requiredDuringSchedulingIgnoredDuringExecution:
-      #       nodeSelectorTerms:
-      #         - matchExpressions:
-      #           - key: kubernetes.io/arch
-      #             operator: In
-      #             values:
-      #               - amd64
-      #               - arm64
-      #               - ppc64le
-      #               - s390x
-      #           - key: kubernetes.io/os
-      #             operator: In
-      #             values:
-      #               - linux
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                - key: kubernetes.io/arch
+                  operator: In
+                  values:
+                    - amd64
+                    - arm64
+                    - ppc64le
+                    - s390x
+                - key: kubernetes.io/os
+                  operator: In
+                  values:
+                    - linux
       securityContext:
         runAsNonRoot: true
-        # TODO(user): For common cases that do not require escalating privileges
-        # it is recommended to ensure that all your Pods/Containers are restrictive.
-        # More info: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted
-        # Please uncomment the following code if your project does NOT have to work on old Kubernetes
-        # versions < 1.19 or on vendors versions which do NOT support this field by default (i.e. Openshift < 4.11 ).
-        # seccompProfile:
-        #   type: RuntimeDefault
+        seccompProfile:
+          type: RuntimeDefault
       containers:
       - command:
         - /manager
         args:
-        - --leader-elect
+          - "--health-probe-bind-address=:8081"
+          - "--metrics-bind-address=:8443"
+          - "--leader-elect"
+          - "--enable-operator"
+        env:
+          - name: NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: IMAGE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.annotations['multiarch.openshift.io/image']
         image: controller:latest
-        imagePullPolicy: Always # TODO[aleskandro]: this is for testing reasons.
+        imagePullPolicy: Always
         name: manager
+        ports:
+        - containerPort: 8081
+          name: health
+          protocol: TCP
+        - containerPort: 8443
+          name: https # This should be "metrics", but the automated bundle generation tooling requires the name to be https
+          # for backwards compatibility with the previous version of kubebuilder that used kube-rbac-proxy
+          protocol: TCP
         securityContext:
           allowPrivilegeEscalation: false
           capabilities:
@@ -95,12 +106,36 @@ spec:
             port: 8081
           initialDelaySeconds: 5
           periodSeconds: 10
-        # TODO(user): Configure the resources accordingly based on the project requirements.
-        # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
         resources:
           requests:
             cpu: 10m
             memory: 64Mi
+        volumeMounts:
+        - mountPath: /var/run/manager/tls
+          name: multiarch-tuning-operator-controller-manager-service-cert
+          readOnly: true
+        - mountPath: /etc/ssl/certs/
+          name: ca-projected-volume
+          readOnly: true
       priorityClassName: system-cluster-critical
       serviceAccountName: controller-manager
       terminationGracePeriodSeconds: 10
+      volumes:
+        - name: multiarch-tuning-operator-controller-manager-service-cert
+          secret:
+            secretName: multiarch-tuning-operator-controller-manager-service-cert
+            defaultMode: 420
+        - name: ca-projected-volume
+          projected:
+            sources:
+              - configMap:
+                  name: openshift-service-ca.crt
+                  items:
+                    - key: service-ca.crt
+                      path: openshift-ca.crt
+                  optional: true
+              - configMap:
+                  name: kube-root-ca.crt
+                  items:
+                    - key: ca.crt
+                      path: kube-root-ca.crt

config/manager/manager.yaml

@@ -7,5 +7,9 @@ spec:
   ports:
     - port: 443
       targetPort: 9443
+      name: webhook
+    - port: 8443
+      targetPort: 8443
+      name: metrics
   selector:
     control-plane: controller-manager

config/webhook/service.yaml

@@ -27,7 +27,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	errorutils "k8s.io/apimachinery/pkg/util/errors"
-	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/kubernetes"
 
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -211,14 +210,6 @@ func (r *ClusterPodPlacementConfigReconciler) handleDelete(ctx context.Context,
 			NamespacedTypedClient: r.ClientSet.CoreV1().Services(utils.Namespace()),
 			ObjName:               utils.PodPlacementWebhookName,
 		},
-		{
-			NamespacedTypedClient: r.ClientSet.CoreV1().Services(utils.Namespace()),
-			ObjName:               utils.PodPlacementControllerMetricsServiceName,
-		},
-		{
-			NamespacedTypedClient: r.ClientSet.CoreV1().Services(utils.Namespace()),
-			ObjName:               utils.PodPlacementWebhookMetricsServiceName,
-		},
 		{
 			NamespacedTypedClient: r.ClientSet.AppsV1().Deployments(utils.Namespace()),
 			ObjName:               utils.PodPlacementWebhookName,
@@ -306,6 +297,10 @@ func (r *ClusterPodPlacementConfigReconciler) handleDelete(ctx context.Context,
 		}
 	}
 	objsToDelete = []utils.ToDeleteRef{
+		{
+			NamespacedTypedClient: r.ClientSet.CoreV1().Services(utils.Namespace()),
+			ObjName:               utils.PodPlacementControllerName,
+		},
 		{
 			NamespacedTypedClient: r.ClientSet.AppsV1().Deployments(utils.Namespace()),
 			ObjName:               utils.PodPlacementControllerName,
@@ -354,25 +349,8 @@ func (r *ClusterPodPlacementConfigReconciler) reconcile(ctx context.Context, clu
 	objects := []client.Object{
 		// The finalizer will not affect the reconciliation of ReplicaSets and Pods
 		// when updates to the ClusterPodPlacementConfig are made.
-		buildService(utils.PodPlacementControllerName, utils.PodPlacementControllerName,
-			443, intstr.FromInt32(9443)),
-		buildService(utils.PodPlacementWebhookName, utils.PodPlacementWebhookName,
-			443, intstr.FromInt32(9443)),
-		buildService(
-			utils.PodPlacementControllerMetricsServiceName, utils.PodPlacementControllerName,
-			8443, intstr.FromInt32(8443)),
-		buildService(
-			utils.PodPlacementWebhookMetricsServiceName, utils.PodPlacementWebhookName,
-			8443, intstr.FromInt32(8443)), buildService(utils.PodPlacementControllerName, utils.PodPlacementControllerName,
-			443, intstr.FromInt32(9443)),
-		buildService(utils.PodPlacementWebhookName, utils.PodPlacementWebhookName,
-			443, intstr.FromInt32(9443)),
-		buildService(
-			utils.PodPlacementControllerMetricsServiceName, utils.PodPlacementControllerName,
-			8443, intstr.FromInt32(8443)),
-		buildService(
-			utils.PodPlacementWebhookMetricsServiceName, utils.PodPlacementWebhookName,
-			8443, intstr.FromInt32(8443)),
+		buildService(utils.PodPlacementControllerName),
+		buildService(utils.PodPlacementWebhookName),
 		buildClusterRoleController(), buildClusterRoleWebhook(), buildRoleController(),
 		buildServiceAccount(utils.PodPlacementWebhookName), buildServiceAccount(utils.PodPlacementControllerName),
 		buildClusterRoleBinding(utils.PodPlacementControllerName, rbacv1.RoleRef{