diff --git a/Dockerfile b/Dockerfile index 2bbbbe6b38..b05fc5f7e4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,11 +19,10 @@ COPY hack/dockerfile_install_support.sh /tmp RUN /bin/bash /tmp/dockerfile_install_support.sh COPY manifests/*.yaml manifests/image-references /manifests/ -ENV APP_ROOT=/var/lib/ocp-tuned -ENV PATH=${APP_ROOT}/bin:${PATH} -ENV HOME=${APP_ROOT} +ENV HOME=/run/ocp-tuned ENV SYSTEMD_IGNORE_CHROOT=1 -WORKDIR ${APP_ROOT} +WORKDIR ${HOME} + RUN dnf clean all && \ rm -rf /var/cache/yum ~/patches /root/rpms && \ useradd -r -u 499 cluster-node-tuning-operator diff --git a/Dockerfile.rhel9 b/Dockerfile.rhel9 index cc10638765..5bc30df680 100644 --- a/Dockerfile.rhel9 +++ b/Dockerfile.rhel9 @@ -15,11 +15,9 @@ COPY hack/dockerfile_install_support.sh /tmp RUN /bin/bash /tmp/dockerfile_install_support.sh COPY manifests/*.yaml manifests/image-references /manifests/ -ENV APP_ROOT=/var/lib/ocp-tuned -ENV PATH=${APP_ROOT}/bin:${PATH} -ENV HOME=${APP_ROOT} +ENV HOME=/run/ocp-tuned ENV SYSTEMD_IGNORE_CHROOT=1 -WORKDIR ${APP_ROOT} +WORKDIR ${HOME} RUN dnf clean all && \ rm -rf /var/cache/yum ~/patches /root/rpms && \ diff --git a/assets/performanceprofile/configs/ocp-tuned-one-shot.service b/assets/performanceprofile/configs/ocp-tuned-one-shot.service index 8a2614d538..3443318c72 100644 --- a/assets/performanceprofile/configs/ocp-tuned-one-shot.service +++ b/assets/performanceprofile/configs/ocp-tuned-one-shot.service @@ -28,7 +28,8 @@ ExecStart=/usr/bin/podman run \ --security-opt label=disable \ --log-driver=none \ --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \ - --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \ + --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \ + --volume /var/lib/tuned:/host/var/lib/tuned:rslave \ --volume /etc/modprobe.d:/etc/modprobe.d:rslave \ --volume /etc/sysconfig:/etc/sysconfig:rslave \ --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \ diff --git a/assets/tuned/manifests/ds-tuned.yaml b/assets/tuned/manifests/ds-tuned.yaml index c94842f797..5fac249a8a 100644 --- a/assets/tuned/manifests/ds-tuned.yaml +++ b/assets/tuned/manifests/ds-tuned.yaml @@ -33,6 +33,7 @@ spec: name: tuned securityContext: privileged: true + readOnlyRootFilesystem: true terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: @@ -57,12 +58,16 @@ spec: - mountPath: /etc/systemd name: etc-systemd mountPropagation: HostToContainer + - mountPath: /etc/tuned + name: etc-tuned - mountPath: /run name: run mountPropagation: HostToContainer - mountPath: /sys name: sys mountPropagation: HostToContainer + - mountPath: /tmp + name: tmp - mountPath: /lib/modules name: lib-modules mountPropagation: HostToContainer @@ -136,6 +141,12 @@ spec: hostPath: path: / type: Directory + - name: etc-tuned + emptyDir: + medium: Memory + - name: tmp + emptyDir: + medium: Memory dnsPolicy: ClusterFirst nodeSelector: kubernetes.io/os: linux diff --git a/hack/dockerfile_install_support.sh b/hack/dockerfile_install_support.sh index c0c1330745..27c398c4f4 100644 --- a/hack/dockerfile_install_support.sh +++ b/hack/dockerfile_install_support.sh @@ -52,8 +52,11 @@ else fi # TuneD post-installation steps -rm -rf /etc/tuned/recommend.d +rm -rf /etc/tuned/recommend.d /var/lib/tuned echo auto > /etc/tuned/profile_mode sed -Ei 's|^#?\s*enable_unix_socket\s*=.*$|enable_unix_socket = 1|;s|^#?\s*rollback\s*=.*$|rollback = not_on_exit|;s|^#?\s*profile_dirs\s*=.*$|profile_dirs = /usr/lib/tuned/profiles,/usr/lib/tuned,/var/lib/ocp-tuned/profiles|' \ /etc/tuned/tuned-main.conf +mv /etc/tuned /etc/tuned.orig +ln -s /host/var/lib/ocp-tuned /var/lib/ocp-tuned +ln -s /host/var/lib/tuned /var/lib/tuned touch /etc/sysctl.conf diff --git a/pkg/tuned/cmd/render/render.go b/pkg/tuned/cmd/render/render.go index 7765da20c7..52405228e3 100644 --- a/pkg/tuned/cmd/render/render.go +++ b/pkg/tuned/cmd/render/render.go @@ -191,6 +191,36 @@ func render(inputDir []string, outputDir string, mcpName string) error { return e } + if err := tunedpkg.TunedRsyncEtc(); err != nil { + e := fmt.Errorf("unable to prepare /etc/tuned directory: %w", err) + klog.Error(e) + return e + } + + // Not removing the symbolic links and creating the following directories + // would cause issues when extracting TuneD profiles when /host directory does not exist, + // such as when invoking "render-bootcmd-mc" during installer bootstrap. + dirs := []string{ + "/var/lib/ocp-tuned", // /var/lib/ocp-tuned -> /host/var/lib/ocp-tuned + "/var/lib/tuned", // /var/lib/tuned -> /host/var/lib/tuned + } + for _, d := range dirs { + if err := os.RemoveAll(d); err != nil { + klog.Error(err) + return err + } + if err := os.MkdirAll(d, os.ModePerm); err != nil { + klog.Error(err) + return err + } + } + + // Make output dir if not present + err = os.MkdirAll("/var/lib/tuned", os.ModePerm) + if err != nil { + return fmt.Errorf("unable to create %s : %w", outputDir, err) + } + tuneDrecommended := operator.TunedRecommend(tuneD) if len(tuneDrecommended) == 0 { e := fmt.Errorf("unable to get recommended profile") diff --git a/pkg/tuned/controller.go b/pkg/tuned/controller.go index 750865f265..6f218d734a 100644 --- a/pkg/tuned/controller.go +++ b/pkg/tuned/controller.go @@ -81,7 +81,9 @@ const ( tunedGracefulExitWait = time.Second * time.Duration(10) ocpTunedHome = "/var/lib/ocp-tuned" ocpTunedRunDir = "/run/" + programName + ocpTunedPersist = ocpTunedRunDir + "/persist" ocpTunedProvider = ocpTunedHome + "/provider" + tunedPersistHome = "/var/lib/tuned" // With the less aggressive rate limiter, retries will happen at 100ms*2^(retry_n-1): // 100ms, 200ms, 400ms, 800ms, 1.6s, 3.2s, 6.4s, 12.8s, 25.6s, 51.2s, 102.4s, 3.4m, 6.8m, 13.7m, 27.3m maxRetries = 15 @@ -92,6 +94,10 @@ const ( ocpTunedImageEnv = ocpTunedHome + "/image.env" tunedProfilesDirCustomHost = ocpTunedHome + "/profiles" tunedRecommendDirHost = ocpTunedHome + "/recommend.d" + // The persistent ocp-tuned TuneD artifacts directory. + ocpTunedHomeHost = "/host" + ocpTunedHome + // The persistent tuned directory for files such as ksm-masked coming from cpu-partitioning profile. + tunedPersistHomeHost = "/host" + tunedPersistHome // How do we detect a reboot? The NTO operand owns and uses two separate files to track deferred updates. // 1. /var/lib/... - persistent storage which will survive across reboots. Contains the actual data. @@ -632,32 +638,11 @@ func providerSync(provider string) (bool, error) { return true, providerExtract(provider) } -// switchTunedHome changes "native" container's home directory as defined by the -// Containerfile to the container's home directory on the host itself. -func switchTunedHome() error { - const ( - ocpTunedHomeHost = "/host" + ocpTunedHome - ) - - // Create the container's home directory on the host. - if err := os.MkdirAll(ocpTunedHomeHost, os.ModePerm); err != nil { - return fmt.Errorf("failed to create directory %q: %v", ocpTunedHomeHost, err) - } - - // Delete the container's home directory. We need a recursive delete, because some cross-compiling environments - // populate the directory with hidden cache directories. - if err := os.RemoveAll(ocpTunedHome); err != nil { - return fmt.Errorf("failed to delete: %q: %v", ocpTunedHome, err) - } - - if err := util.Symlink(ocpTunedHomeHost, ocpTunedHome); err != nil { - return fmt.Errorf("failed to link %q -> %q: %v", ocpTunedHome, ocpTunedHomeHost, err) +func PrepareOpenShiftTunedDir() error { + if err := TunedRsyncEtc(); err != nil { + return err } - return os.Chdir(ocpTunedHome) -} - -func prepareOpenShiftTunedDir() error { // Create the following directories unless they exist. dirs := []string{ tunedRecommendDirHost, @@ -1680,11 +1665,27 @@ func retryLoop(c *Controller) (err error) { func RunInCluster(stopCh <-chan struct{}, version string) error { klog.Infof("starting in-cluster %s %s", programName, version) - if err := switchTunedHome(); err != nil { - return err + dirs := []string{ + ocpTunedHomeHost, + tunedPersistHomeHost, + } + for _, d := range dirs { + if err := os.MkdirAll(d, os.ModePerm); err != nil { + return fmt.Errorf("failed to create directory %q: %v", d, err) + } + } + + links := map[string]string{ + ocpTunedHomeHost: ocpTunedPersist, + tunedPersistHomeHost: tunedPersistHome, + } + for target, source := range links { + if err := util.Symlink(target, source); err != nil { + return fmt.Errorf("failed to link %q -> %q: %v", source, target, err) + } } - if err := prepareOpenShiftTunedDir(); err != nil { + if err := PrepareOpenShiftTunedDir(); err != nil { return err } @@ -1756,7 +1757,7 @@ func restartReason(isNodeReboot bool) string { func RunOutOfClusterOneShot(stopCh <-chan struct{}, version string) error { klog.Infof("starting out-of-cluster %s %s", programName, version) - if err := prepareOpenShiftTunedDir(); err != nil { + if err := PrepareOpenShiftTunedDir(); err != nil { return err } diff --git a/pkg/tuned/run.go b/pkg/tuned/run.go index eb78749b0a..a181f3f49f 100644 --- a/pkg/tuned/run.go +++ b/pkg/tuned/run.go @@ -72,6 +72,24 @@ func configDaemonMode() (func(), error) { return restoreF, nil } +// TunedRsyncEtcToHost propagates the changes from container's read-only TuneD /etc/tuned.orig +// directory to the container's Memory-backed read-write TuneD /etc/tuned directory. +// This function only serves the purpose to enable readOnlyRootFilesystem for the NTO operand. +func TunedRsyncEtc() error { + const ( + source = "/etc/tuned.orig/" + target = tunedEtcDir + ) + + cmd := exec.Command("rsync", "--delete", "-av", source, target) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("rsync of %q to %q failed: %v\n%s", source, target, err, out) + } + + return nil +} + func TunedRunNoDaemon(timeout time.Duration) error { var ( cmd *exec.Cmd diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-master_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-master_machineconfig.yaml index 3c15585ad9..4978822d0e 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-master_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-master_machineconfig.yaml @@ -196,7 +196,8 @@ spec: --security-opt label=disable \ --log-driver=none \ --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \ - --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \ + --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \ + --volume /var/lib/tuned:/host/var/lib/tuned:rslave \ --volume /etc/modprobe.d:/etc/modprobe.d:rslave \ --volume /etc/sysconfig:/etc/sysconfig:rslave \ --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \ diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-worker_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-worker_machineconfig.yaml index db0487a8ff..67abea13eb 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-worker_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-worker_machineconfig.yaml @@ -196,7 +196,8 @@ spec: --security-opt label=disable \ --log-driver=none \ --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \ - --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \ + --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \ + --volume /var/lib/tuned:/host/var/lib/tuned:rslave \ --volume /etc/modprobe.d:/etc/modprobe.d:rslave \ --volume /etc/sysconfig:/etc/sysconfig:rslave \ --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \ diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml index 973050fad6..8afae2ee87 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml @@ -196,7 +196,8 @@ spec: --security-opt label=disable \ --log-driver=none \ --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \ - --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \ + --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \ + --volume /var/lib/tuned:/host/var/lib/tuned:rslave \ --volume /etc/modprobe.d:/etc/modprobe.d:rslave \ --volume /etc/sysconfig:/etc/sysconfig:rslave \ --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \ diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml index db0487a8ff..67abea13eb 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-worker_machineconfig.yaml @@ -196,7 +196,8 @@ spec: --security-opt label=disable \ --log-driver=none \ --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \ - --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \ + --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \ + --volume /var/lib/tuned:/host/var/lib/tuned:rslave \ --volume /etc/modprobe.d:/etc/modprobe.d:rslave \ --volume /etc/sysconfig:/etc/sysconfig:rslave \ --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \ diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml index 973050fad6..8afae2ee87 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-master_machineconfig.yaml @@ -196,7 +196,8 @@ spec: --security-opt label=disable \ --log-driver=none \ --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \ - --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \ + --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \ + --volume /var/lib/tuned:/host/var/lib/tuned:rslave \ --volume /etc/modprobe.d:/etc/modprobe.d:rslave \ --volume /etc/sysconfig:/etc/sysconfig:rslave \ --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \ diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-worker_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-worker_machineconfig.yaml index db0487a8ff..67abea13eb 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-worker_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/no-mcp/openshift-bootstrap-worker_machineconfig.yaml @@ -196,7 +196,8 @@ spec: --security-opt label=disable \ --log-driver=none \ --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \ - --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \ + --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \ + --volume /var/lib/tuned:/host/var/lib/tuned:rslave \ --volume /etc/modprobe.d:/etc/modprobe.d:rslave \ --volume /etc/sysconfig:/etc/sysconfig:rslave \ --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \ diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/default/cpuFrequency/manual_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/default/cpuFrequency/manual_machineconfig.yaml index 1862aa805d..50e3aca32e 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/default/cpuFrequency/manual_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/default/cpuFrequency/manual_machineconfig.yaml @@ -198,7 +198,8 @@ spec: --security-opt label=disable \ --log-driver=none \ --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \ - --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \ + --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \ + --volume /var/lib/tuned:/host/var/lib/tuned:rslave \ --volume /etc/modprobe.d:/etc/modprobe.d:rslave \ --volume /etc/sysconfig:/etc/sysconfig:rslave \ --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \ diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/default/manual_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/default/manual_machineconfig.yaml index 545a58abfe..2fd75bd352 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/default/manual_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/default/manual_machineconfig.yaml @@ -214,7 +214,8 @@ spec: --security-opt label=disable \ --log-driver=none \ --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \ - --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \ + --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \ + --volume /var/lib/tuned:/host/var/lib/tuned:rslave \ --volume /etc/modprobe.d:/etc/modprobe.d:rslave \ --volume /etc/sysconfig:/etc/sysconfig:rslave \ --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \ diff --git a/test/e2e/performanceprofile/testdata/render-expected-output/no-ref/manual_machineconfig.yaml b/test/e2e/performanceprofile/testdata/render-expected-output/no-ref/manual_machineconfig.yaml index 6052149999..f0d86ce9c2 100644 --- a/test/e2e/performanceprofile/testdata/render-expected-output/no-ref/manual_machineconfig.yaml +++ b/test/e2e/performanceprofile/testdata/render-expected-output/no-ref/manual_machineconfig.yaml @@ -213,7 +213,8 @@ spec: --security-opt label=disable \ --log-driver=none \ --volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \ - --volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \ + --volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \ + --volume /var/lib/tuned:/host/var/lib/tuned:rslave \ --volume /etc/modprobe.d:/etc/modprobe.d:rslave \ --volume /etc/sysconfig:/etc/sysconfig:rslave \ --volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \