Read-only container root filesystem

In line with the "Principle of least privilege", add
readOnlyRootFilesystem to the NTO operand's container securityContext.

Key changes:
  * NTO's operand daemonset sets the readOnlyRootFilesystem container
    securityContext.
  * /tmp is symlinked to /run/ocp-tuned and the directory ownership is set
    to the operator ID.  This allows:
    * the operand's TuneD daemon writing temporary files when using profiles
      such as the cpu-partitioning profile.
  * Make /var/lib/tuned directory persistent on the host.
  * Change the ocp-tuned-one-shot systemd service to mount the hosts's
    persistent host /var/lib/{ocp-,}tuned directories to
    /host/var/lib/{ocp-,}tuned to simplify the operand code.

Author: jmencak

Dockerfile

@@ -19,13 +19,13 @@ COPY hack/dockerfile_install_support.sh /tmp
 RUN /bin/bash /tmp/dockerfile_install_support.sh
 
 COPY manifests/*.yaml manifests/image-references /manifests/
-ENV APP_ROOT=/var/lib/ocp-tuned
-ENV PATH=${APP_ROOT}/bin:${PATH}
-ENV HOME=${APP_ROOT}
+ENV HOME=/run/ocp-tuned
 ENV SYSTEMD_IGNORE_CHROOT=1
-WORKDIR ${APP_ROOT}
+WORKDIR ${HOME}
+
 RUN dnf clean all && \
-    rm -rf /var/cache/yum ~/patches /root/rpms && \
+    rm -rf /var/cache/yum ~/patches /root/rpms /tmp && \
+    ln -s /run/ocp-tuned /tmp && \
     useradd -r -u 499 cluster-node-tuning-operator
 ENTRYPOINT ["/usr/bin/cluster-node-tuning-operator"]
 LABEL io.k8s.display-name="OpenShift cluster-node-tuning-operator" \

Dockerfile.rhel9

@@ -15,14 +15,13 @@ COPY hack/dockerfile_install_support.sh /tmp
 RUN /bin/bash /tmp/dockerfile_install_support.sh
 
 COPY manifests/*.yaml manifests/image-references /manifests/
-ENV APP_ROOT=/var/lib/ocp-tuned
-ENV PATH=${APP_ROOT}/bin:${PATH}
-ENV HOME=${APP_ROOT}
+ENV HOME=/run/ocp-tuned
 ENV SYSTEMD_IGNORE_CHROOT=1
-WORKDIR ${APP_ROOT}
+WORKDIR ${HOME}
 
 RUN dnf clean all && \
-    rm -rf /var/cache/yum ~/patches /root/rpms && \
+    rm -rf /var/cache/yum ~/patches /root/rpms /tmp && \
+    ln -s /run/ocp-tuned /tmp && \
     useradd -r -u 499 cluster-node-tuning-operator
 ENTRYPOINT ["/usr/bin/cluster-node-tuning-operator"]
 LABEL io.k8s.display-name="OpenShift cluster-node-tuning-operator" \

assets/performanceprofile/configs/ocp-tuned-one-shot.service

@@ -27,7 +27,8 @@ ExecStart=/usr/bin/podman run \
     --security-opt label=disable \
     --log-driver=none \
     --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
-    --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+    --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \
+    --volume /var/lib/tuned:/host/var/lib/tuned:rslave \
     --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
     --volume /etc/sysconfig:/etc/sysconfig:rslave \
     --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \

assets/tuned/manifests/ds-tuned.yaml

@@ -33,6 +33,7 @@ spec:
         name: tuned
         securityContext:
           privileged: true
+          readOnlyRootFilesystem: true
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: FallbackToLogsOnError
         volumeMounts:

hack/dockerfile_install_support.sh

@@ -8,6 +8,7 @@ INSTALL_PKGS="nmap-ncat procps-ng pciutils"
 # TuneD pre-installation steps
 cp -r /root/assets/bin/* /usr/local/bin
 mkdir -p /etc/grub.d/ /boot /run/ocp-tuned
+chown -R 499:499 /run/ocp-tuned	# the operator must be able to write metrics client CA in a temporary directory
 
 source /etc/os-release
 if [[ "${ID}" == "centos" ]]; then
@@ -43,8 +44,12 @@ else
 fi
 
 # TuneD post-installation steps
-rm -rf /etc/tuned/recommend.d
+rm -rf /etc/tuned/recommend.d /var/lib/tuned
 echo auto > /etc/tuned/profile_mode
 sed -Ei 's|^#?\s*enable_unix_socket\s*=.*$|enable_unix_socket = 1|;s|^#?\s*rollback\s*=.*$|rollback = not_on_exit|;s|^#?\s*profile_dirs\s*=.*$|profile_dirs = /usr/lib/tuned/profiles,/usr/lib/tuned,/var/lib/ocp-tuned/profiles|' \
   /etc/tuned/tuned-main.conf
+mv /etc/tuned /etc/tuned.orig
+ln -s /var/lib/ocp-tuned/tuned /etc/tuned
+ln -s /host/var/lib/ocp-tuned /var/lib/ocp-tuned
+ln -s /host/var/lib/tuned /var/lib/tuned
 touch /etc/sysctl.conf

pkg/tuned/controller.go

@@ -81,7 +81,9 @@ const (
 	tunedGracefulExitWait = time.Second * time.Duration(10)
 	ocpTunedHome          = "/var/lib/ocp-tuned"
 	ocpTunedRunDir        = "/run/" + programName
+	ocpTunedPersist       = ocpTunedRunDir + "/persist"
 	ocpTunedProvider      = ocpTunedHome + "/provider"
+	tunedPersistHome      = "/var/lib/tuned"
 	// With the less aggressive rate limiter, retries will happen at 100ms*2^(retry_n-1):
 	// 100ms, 200ms, 400ms, 800ms, 1.6s, 3.2s, 6.4s, 12.8s, 25.6s, 51.2s, 102.4s, 3.4m, 6.8m, 13.7m, 27.3m
 	maxRetries = 15
@@ -92,6 +94,10 @@ const (
 	ocpTunedImageEnv           = ocpTunedHome + "/image.env"
 	tunedProfilesDirCustomHost = ocpTunedHome + "/profiles"
 	tunedRecommendDirHost      = ocpTunedHome + "/recommend.d"
+	// The persistent ocp-tuned TuneD artifacts directory.
+	ocpTunedHomeHost = "/host" + ocpTunedHome
+	// The persistent tuned directory for files such as ksm-masked coming from cpu-partitioning profile.
+	tunedPersistHomeHost = "/host" + tunedPersistHome
 
 	// How do we detect a reboot? The NTO operand owns and uses two separate files to track deferred updates.
 	// 1. /var/lib/... - persistent storage which will survive across reboots. Contains the actual data.
@@ -640,37 +646,11 @@ func providerSync(provider string) (bool, error) {
 	return true, providerExtract(provider)
 }
 
-// switchTunedHome changes "native" container's home directory as defined by the
-// Containerfile to the container's home directory on the host itself.
-func switchTunedHome() error {
-	const (
-		ocpTunedHomeHost = "/host" + ocpTunedHome
-	)
-
-	// Create the container's home directory on the host.
-	if err := os.MkdirAll(ocpTunedHomeHost, os.ModePerm); err != nil {
-		return fmt.Errorf("failed to create directory %q: %v", ocpTunedHomeHost, err)
-	}
-
-	// Delete the container's home directory.  We need a recursive delete, because some cross-compiling environments
-	// populate the directory with hidden cache directories.
-	if err := os.RemoveAll(ocpTunedHome); err != nil {
-		return fmt.Errorf("failed to delete: %q: %v", ocpTunedHome, err)
-	}
-
-	if err := util.Symlink(ocpTunedHomeHost, ocpTunedHome); err != nil {
-		return fmt.Errorf("failed to link %q -> %q: %v", ocpTunedHome, ocpTunedHomeHost, err)
-	}
-
-	err := os.Chdir(ocpTunedHome)
-	if err != nil {
+func prepareOpenShiftTunedDir() error {
+	if err := TunedRsyncEtcToHost(); err != nil {
 		return err
 	}
 
-	return nil
-}
-
-func prepareOpenShiftTunedDir() error {
 	// Create the following directories unless they exist.
 	dirs := []string{
 		tunedRecommendDirHost,
@@ -1701,8 +1681,24 @@ func retryLoop(c *Controller) (err error) {
 func RunInCluster(stopCh <-chan struct{}, version string) error {
 	klog.Infof("starting in-cluster %s %s", programName, version)
 
-	if err := switchTunedHome(); err != nil {
-		return err
+	dirs := []string{
+		ocpTunedHomeHost,
+		tunedPersistHomeHost,
+	}
+	for _, d := range dirs {
+		if err := os.MkdirAll(d, os.ModePerm); err != nil {
+			return fmt.Errorf("failed to create directory %q: %v", d, err)
+		}
+	}
+
+	links := map[string]string{
+		ocpTunedHomeHost:     ocpTunedPersist,
+		tunedPersistHomeHost: tunedPersistHome,
+	}
+	for target, source := range links {
+		if err := util.Symlink(target, source); err != nil {
+			return fmt.Errorf("failed to link %q -> %q: %v", source, target, err)
+		}
 	}
 
 	if err := prepareOpenShiftTunedDir(); err != nil {

pkg/tuned/run.go

@@ -72,6 +72,21 @@ func configDaemonMode() (func(), error) {
 	return restoreF, nil
 }
 
+func TunedRsyncEtcToHost() error {
+	const (
+		source = "/etc/tuned.orig/"
+		target = ocpTunedHome + "/tuned"
+	)
+
+	cmd := exec.Command("rsync", "--delete", "-av", source, target)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("rsync of %q to %q failed: %v\n%s", source, target, err, out)
+	}
+
+	return nil
+}
+
 func TunedRunNoDaemon(timeout time.Duration) error {
 	var (
 		cmd    *exec.Cmd

test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-master_machineconfig.yaml

@@ -195,7 +195,8 @@ spec:
               --security-opt label=disable \
               --log-driver=none \
               --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
-              --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/tuned:/host/var/lib/tuned:rslave \
               --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
               --volume /etc/sysconfig:/etc/sysconfig:rslave \
               --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \

test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-worker_machineconfig.yaml

@@ -195,7 +195,8 @@ spec:
               --security-opt label=disable \
               --log-driver=none \
               --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
-              --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/tuned:/host/var/lib/tuned:rslave \
               --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
               --volume /etc/sysconfig:/etc/sysconfig:rslave \
               --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \

test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml

@@ -195,7 +195,8 @@ spec:
               --security-opt label=disable \
               --log-driver=none \
               --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
-              --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/tuned:/host/var/lib/tuned:rslave \
               --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
               --volume /etc/sysconfig:/etc/sysconfig:rslave \
               --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \

test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml

@@ -195,7 +195,8 @@ spec:
               --security-opt label=disable \
               --log-driver=none \
               --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
-              --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/tuned:/host/var/lib/tuned:rslave \
               --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
               --volume /etc/sysconfig:/etc/sysconfig:rslave \
               --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \

test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml

@@ -195,7 +195,8 @@ spec:
               --security-opt label=disable \
               --log-driver=none \
               --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
-              --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/tuned:/host/var/lib/tuned:rslave \
               --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
               --volume /etc/sysconfig:/etc/sysconfig:rslave \
               --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \

test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-worker_machineconfig.yaml

@@ -195,7 +195,8 @@ spec:
               --security-opt label=disable \
               --log-driver=none \
               --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
-              --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/tuned:/host/var/lib/tuned:rslave \
               --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
               --volume /etc/sysconfig:/etc/sysconfig:rslave \
               --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \

test/e2e/performanceprofile/testdata/render-expected-output/default/cpuFrequency/manual_machineconfig.yaml

@@ -197,7 +197,8 @@ spec:
               --security-opt label=disable \
               --log-driver=none \
               --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
-              --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/tuned:/host/var/lib/tuned:rslave \
               --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
               --volume /etc/sysconfig:/etc/sysconfig:rslave \
               --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \

test/e2e/performanceprofile/testdata/render-expected-output/default/manual_machineconfig.yaml

@@ -213,7 +213,8 @@ spec:
               --security-opt label=disable \
               --log-driver=none \
               --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
-              --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/tuned:/host/var/lib/tuned:rslave \
               --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
               --volume /etc/sysconfig:/etc/sysconfig:rslave \
               --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \

test/e2e/performanceprofile/testdata/render-expected-output/no-ref/manual_machineconfig.yaml

@@ -212,7 +212,8 @@ spec:
               --security-opt label=disable \
               --log-driver=none \
               --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
-              --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/tuned:/host/var/lib/tuned:rslave \
               --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
               --volume /etc/sysconfig:/etc/sysconfig:rslave \
               --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \