PSAP-1428: read only root filesystem

Dockerfile

@@ -19,11 +19,10 @@ COPY hack/dockerfile_install_support.sh /tmp
 RUN /bin/bash /tmp/dockerfile_install_support.sh
 
 COPY manifests/*.yaml manifests/image-references /manifests/
-ENV APP_ROOT=/var/lib/ocp-tuned
-ENV PATH=${APP_ROOT}/bin:${PATH}
-ENV HOME=${APP_ROOT}
+ENV HOME=/run/ocp-tuned
 ENV SYSTEMD_IGNORE_CHROOT=1
-WORKDIR ${APP_ROOT}
+WORKDIR ${HOME}
+
 RUN dnf clean all && \
     rm -rf /var/cache/yum ~/patches /root/rpms && \
     useradd -r -u 499 cluster-node-tuning-operator

Dockerfile.rhel9

@@ -15,11 +15,9 @@ COPY hack/dockerfile_install_support.sh /tmp
 RUN /bin/bash /tmp/dockerfile_install_support.sh
 
 COPY manifests/*.yaml manifests/image-references /manifests/
-ENV APP_ROOT=/var/lib/ocp-tuned
-ENV PATH=${APP_ROOT}/bin:${PATH}
-ENV HOME=${APP_ROOT}
+ENV HOME=/run/ocp-tuned
 ENV SYSTEMD_IGNORE_CHROOT=1
-WORKDIR ${APP_ROOT}
+WORKDIR ${HOME}
 
 RUN dnf clean all && \
     rm -rf /var/cache/yum ~/patches /root/rpms && \

assets/performanceprofile/configs/ocp-tuned-one-shot.service

@@ -28,7 +28,8 @@ ExecStart=/usr/bin/podman run \
     --security-opt label=disable \
     --log-driver=none \
     --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
-    --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+    --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \
+    --volume /var/lib/tuned:/host/var/lib/tuned:rslave \
     --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
     --volume /etc/sysconfig:/etc/sysconfig:rslave \
     --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \

assets/tuned/manifests/ds-tuned.yaml

@@ -33,6 +33,7 @@ spec:
         name: tuned
         securityContext:
           privileged: true
+          readOnlyRootFilesystem: true
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: FallbackToLogsOnError
         volumeMounts:
@@ -57,12 +58,16 @@ spec:
         - mountPath: /etc/systemd
           name: etc-systemd
           mountPropagation: HostToContainer
+        - mountPath: /etc/tuned
+          name: etc-tuned
         - mountPath: /run
           name: run
           mountPropagation: HostToContainer
         - mountPath: /sys
           name: sys
           mountPropagation: HostToContainer
+        - mountPath: /tmp
+          name: tmp
         - mountPath: /lib/modules
           name: lib-modules
           mountPropagation: HostToContainer
@@ -136,6 +141,12 @@ spec:
         hostPath:
           path: /
           type: Directory
+      - name: etc-tuned
+        emptyDir:
+          medium: Memory
+      - name: tmp
+        emptyDir:
+          medium: Memory
       dnsPolicy: ClusterFirst
       nodeSelector:
         kubernetes.io/os: linux

hack/dockerfile_install_support.sh

@@ -52,8 +52,11 @@ else
 fi
 
 # TuneD post-installation steps
-rm -rf /etc/tuned/recommend.d
+rm -rf /etc/tuned/recommend.d /var/lib/tuned
 echo auto > /etc/tuned/profile_mode
 sed -Ei 's|^#?\s*enable_unix_socket\s*=.*$|enable_unix_socket = 1|;s|^#?\s*rollback\s*=.*$|rollback = not_on_exit|;s|^#?\s*profile_dirs\s*=.*$|profile_dirs = /usr/lib/tuned/profiles,/usr/lib/tuned,/var/lib/ocp-tuned/profiles|' \
   /etc/tuned/tuned-main.conf
+mv /etc/tuned /etc/tuned.orig
+ln -s /host/var/lib/ocp-tuned /var/lib/ocp-tuned
+ln -s /host/var/lib/tuned /var/lib/tuned
 touch /etc/sysctl.conf

pkg/tuned/cmd/render/render.go

@@ -191,6 +191,36 @@ func render(inputDir []string, outputDir string, mcpName string) error {
 		return e
 	}
 
+	if err := tunedpkg.TunedRsyncEtc(); err != nil {
+		e := fmt.Errorf("unable to prepare /etc/tuned directory: %w", err)
+		klog.Error(e)
+		return e
+	}
+
+	// Not removing the symbolic links and creating the following directories
+	// would cause issues when extracting TuneD profiles when /host directory does not exist,
+	// such as when invoking "render-bootcmd-mc" during installer bootstrap.
+	dirs := []string{
+		"/var/lib/ocp-tuned", // /var/lib/ocp-tuned -> /host/var/lib/ocp-tuned
+		"/var/lib/tuned",     // /var/lib/tuned -> /host/var/lib/tuned
+	}
+	for _, d := range dirs {
+		if err := os.RemoveAll(d); err != nil {
+			klog.Error(err)
+			return err
+		}
+		if err := os.MkdirAll(d, os.ModePerm); err != nil {
+			klog.Error(err)
+			return err
+		}
+	}
+
+	// Make output dir if not present
+	err = os.MkdirAll("/var/lib/tuned", os.ModePerm)
+	if err != nil {
+		return fmt.Errorf("unable to create  %s : %w", outputDir, err)
+	}
+
 	tuneDrecommended := operator.TunedRecommend(tuneD)
 	if len(tuneDrecommended) == 0 {
 		e := fmt.Errorf("unable to get recommended profile")

pkg/tuned/controller.go

@@ -81,7 +81,9 @@ const (
 	tunedGracefulExitWait = time.Second * time.Duration(10)
 	ocpTunedHome          = "/var/lib/ocp-tuned"
 	ocpTunedRunDir        = "/run/" + programName
+	ocpTunedPersist       = ocpTunedRunDir + "/persist"
 	ocpTunedProvider      = ocpTunedHome + "/provider"
+	tunedPersistHome      = "/var/lib/tuned"
 	// With the less aggressive rate limiter, retries will happen at 100ms*2^(retry_n-1):
 	// 100ms, 200ms, 400ms, 800ms, 1.6s, 3.2s, 6.4s, 12.8s, 25.6s, 51.2s, 102.4s, 3.4m, 6.8m, 13.7m, 27.3m
 	maxRetries = 15
@@ -92,6 +94,10 @@ const (
 	ocpTunedImageEnv           = ocpTunedHome + "/image.env"
 	tunedProfilesDirCustomHost = ocpTunedHome + "/profiles"
 	tunedRecommendDirHost      = ocpTunedHome + "/recommend.d"
+	// The persistent ocp-tuned TuneD artifacts directory.
+	ocpTunedHomeHost = "/host" + ocpTunedHome
+	// The persistent tuned directory for files such as ksm-masked coming from cpu-partitioning profile.
+	tunedPersistHomeHost = "/host" + tunedPersistHome
 
 	// How do we detect a reboot? The NTO operand owns and uses two separate files to track deferred updates.
 	// 1. /var/lib/... - persistent storage which will survive across reboots. Contains the actual data.
@@ -632,32 +638,11 @@ func providerSync(provider string) (bool, error) {
 	return true, providerExtract(provider)
 }
 
-// switchTunedHome changes "native" container's home directory as defined by the
-// Containerfile to the container's home directory on the host itself.
-func switchTunedHome() error {
-	const (
-		ocpTunedHomeHost = "/host" + ocpTunedHome
-	)
-
-	// Create the container's home directory on the host.
-	if err := os.MkdirAll(ocpTunedHomeHost, os.ModePerm); err != nil {
-		return fmt.Errorf("failed to create directory %q: %v", ocpTunedHomeHost, err)
-	}
-
-	// Delete the container's home directory.  We need a recursive delete, because some cross-compiling environments
-	// populate the directory with hidden cache directories.
-	if err := os.RemoveAll(ocpTunedHome); err != nil {
-		return fmt.Errorf("failed to delete: %q: %v", ocpTunedHome, err)
-	}
-
-	if err := util.Symlink(ocpTunedHomeHost, ocpTunedHome); err != nil {
-		return fmt.Errorf("failed to link %q -> %q: %v", ocpTunedHome, ocpTunedHomeHost, err)
+func PrepareOpenShiftTunedDir() error {
+	if err := TunedRsyncEtc(); err != nil {
+		return err
 	}
 
-	return os.Chdir(ocpTunedHome)
-}
-
-func prepareOpenShiftTunedDir() error {
 	// Create the following directories unless they exist.
 	dirs := []string{
 		tunedRecommendDirHost,
@@ -1680,11 +1665,27 @@ func retryLoop(c *Controller) (err error) {
 func RunInCluster(stopCh <-chan struct{}, version string) error {
 	klog.Infof("starting in-cluster %s %s", programName, version)
 
-	if err := switchTunedHome(); err != nil {
-		return err
+	dirs := []string{
+		ocpTunedHomeHost,
+		tunedPersistHomeHost,
+	}
+	for _, d := range dirs {
+		if err := os.MkdirAll(d, os.ModePerm); err != nil {
+			return fmt.Errorf("failed to create directory %q: %v", d, err)
+		}
+	}
+
+	links := map[string]string{
+		ocpTunedHomeHost:     ocpTunedPersist,
+		tunedPersistHomeHost: tunedPersistHome,
+	}
+	for target, source := range links {
+		if err := util.Symlink(target, source); err != nil {
+			return fmt.Errorf("failed to link %q -> %q: %v", source, target, err)
+		}
 	}
 
-	if err := prepareOpenShiftTunedDir(); err != nil {
+	if err := PrepareOpenShiftTunedDir(); err != nil {
 		return err
 	}
 
@@ -1756,7 +1757,7 @@ func restartReason(isNodeReboot bool) string {
 func RunOutOfClusterOneShot(stopCh <-chan struct{}, version string) error {
 	klog.Infof("starting out-of-cluster %s %s", programName, version)
 
-	if err := prepareOpenShiftTunedDir(); err != nil {
+	if err := PrepareOpenShiftTunedDir(); err != nil {
 		return err
 	}
 

pkg/tuned/run.go

@@ -72,6 +72,24 @@ func configDaemonMode() (func(), error) {
 	return restoreF, nil
 }
 
+// TunedRsyncEtcToHost propagates the changes from container's read-only TuneD /etc/tuned.orig
+// directory to the container's Memory-backed read-write TuneD /etc/tuned directory.
+// This function only serves the purpose to enable readOnlyRootFilesystem for the NTO operand.
+func TunedRsyncEtc() error {
+	const (
+		source = "/etc/tuned.orig/"
+		target = tunedEtcDir
+	)
+
+	cmd := exec.Command("rsync", "--delete", "-av", source, target)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("rsync of %q to %q failed: %v\n%s", source, target, err, out)
+	}
+
+	return nil
+}
+
 func TunedRunNoDaemon(timeout time.Duration) error {
 	var (
 		cmd    *exec.Cmd

test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-master_machineconfig.yaml

@@ -196,7 +196,8 @@ spec:
               --security-opt label=disable \
               --log-driver=none \
               --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
-              --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/tuned:/host/var/lib/tuned:rslave \
               --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
               --volume /etc/sysconfig:/etc/sysconfig:rslave \
               --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \

test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-worker_machineconfig.yaml

@@ -196,7 +196,8 @@ spec:
               --security-opt label=disable \
               --log-driver=none \
               --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
-              --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/tuned:/host/var/lib/tuned:rslave \
               --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
               --volume /etc/sysconfig:/etc/sysconfig:rslave \
               --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \

test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml

@@ -196,7 +196,8 @@ spec:
               --security-opt label=disable \
               --log-driver=none \
               --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
-              --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/tuned:/host/var/lib/tuned:rslave \
               --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
               --volume /etc/sysconfig:/etc/sysconfig:rslave \
               --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \

test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml

@@ -196,7 +196,8 @@ spec:
               --security-opt label=disable \
               --log-driver=none \
               --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
-              --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/tuned:/host/var/lib/tuned:rslave \
               --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
               --volume /etc/sysconfig:/etc/sysconfig:rslave \
               --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \

test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml

@@ -196,7 +196,8 @@ spec:
               --security-opt label=disable \
               --log-driver=none \
               --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
-              --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/tuned:/host/var/lib/tuned:rslave \
               --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
               --volume /etc/sysconfig:/etc/sysconfig:rslave \
               --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \

test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-worker_machineconfig.yaml

@@ -196,7 +196,8 @@ spec:
               --security-opt label=disable \
               --log-driver=none \
               --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
-              --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/tuned:/host/var/lib/tuned:rslave \
               --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
               --volume /etc/sysconfig:/etc/sysconfig:rslave \
               --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \

test/e2e/performanceprofile/testdata/render-expected-output/default/cpuFrequency/manual_machineconfig.yaml

@@ -198,7 +198,8 @@ spec:
               --security-opt label=disable \
               --log-driver=none \
               --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
-              --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/tuned:/host/var/lib/tuned:rslave \
               --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
               --volume /etc/sysconfig:/etc/sysconfig:rslave \
               --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \

test/e2e/performanceprofile/testdata/render-expected-output/default/manual_machineconfig.yaml

@@ -214,7 +214,8 @@ spec:
               --security-opt label=disable \
               --log-driver=none \
               --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
-              --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/tuned:/host/var/lib/tuned:rslave \
               --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
               --volume /etc/sysconfig:/etc/sysconfig:rslave \
               --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \

test/e2e/performanceprofile/testdata/render-expected-output/no-ref/manual_machineconfig.yaml

@@ -213,7 +213,8 @@ spec:
               --security-opt label=disable \
               --log-driver=none \
               --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
-              --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \
+              --volume /var/lib/tuned:/host/var/lib/tuned:rslave \
               --volume /etc/modprobe.d:/etc/modprobe.d:rslave \
               --volume /etc/sysconfig:/etc/sysconfig:rslave \
               --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \