Skip to content

Commit 76c3005

Browse files
jmencakjmencak
committed
Read-only container root filesystem
In line with the "Principle of least privilege", add readOnlyRootFilesystem to the NTO operand's container securityContext. Key changes: * NTO's operand daemonset sets the readOnlyRootFilesystem container securityContext. * /tmp is symlinked to /run/ocp-tuned and the directory ownership is set to the operator ID. This allows: * the operand's TuneD daemon writing temporary files when using profiles such as the cpu-partitioning profile. * Make /var/lib/tuned directory persistent on the host. * Change the ocp-tuned-one-shot systemd service to mount the hosts's persistent host /var/lib/{ocp-,}tuned directories to /host/var/lib/{ocp-,}tuned to simplify the operand code.
1 parent 1685a8f commit 76c3005

16 files changed

+77
-51
lines changed

‎Dockerfile‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,13 @@ COPY hack/dockerfile_install_support.sh /tmp
1919
RUN /bin/bash /tmp/dockerfile_install_support.sh
2020

2121
COPY manifests/*.yaml manifests/image-references /manifests/
22-
ENV APP_ROOT=/var/lib/ocp-tuned
23-
ENV PATH=${APP_ROOT}/bin:${PATH}
24-
ENV HOME=${APP_ROOT}
22+
ENV HOME=/run/ocp-tuned
2523
ENV SYSTEMD_IGNORE_CHROOT=1
26-
WORKDIR ${APP_ROOT}
24+
WORKDIR ${HOME}
25+
2726
RUN dnf clean all && \
28-
rm -rf /var/cache/yum ~/patches /root/rpms && \
27+
rm -rf /var/cache/yum ~/patches /root/rpms /tmp && \
28+
ln -s /run/ocp-tuned /tmp && \
2929
useradd -r -u 499 cluster-node-tuning-operator
3030
ENTRYPOINT ["/usr/bin/cluster-node-tuning-operator"]
3131
LABEL io.k8s.display-name="OpenShift cluster-node-tuning-operator" \

‎Dockerfile.rhel9‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,13 @@ COPY hack/dockerfile_install_support.sh /tmp
1515
RUN /bin/bash /tmp/dockerfile_install_support.sh
1616

1717
COPY manifests/*.yaml manifests/image-references /manifests/
18-
ENV APP_ROOT=/var/lib/ocp-tuned
19-
ENV PATH=${APP_ROOT}/bin:${PATH}
20-
ENV HOME=${APP_ROOT}
18+
ENV HOME=/run/ocp-tuned
2119
ENV SYSTEMD_IGNORE_CHROOT=1
22-
WORKDIR ${APP_ROOT}
20+
WORKDIR ${HOME}
2321

2422
RUN dnf clean all && \
25-
rm -rf /var/cache/yum ~/patches /root/rpms && \
23+
rm -rf /var/cache/yum ~/patches /root/rpms /tmp && \
24+
ln -s /run/ocp-tuned /tmp && \
2625
useradd -r -u 499 cluster-node-tuning-operator
2726
ENTRYPOINT ["/usr/bin/cluster-node-tuning-operator"]
2827
LABEL io.k8s.display-name="OpenShift cluster-node-tuning-operator" \

‎assets/performanceprofile/configs/ocp-tuned-one-shot.service‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ ExecStart=/usr/bin/podman run \
2727
--security-opt label=disable \
2828
--log-driver=none \
2929
--volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
30-
--volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
30+
--volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \
31+
--volume /var/lib/tuned:/host/var/lib/tuned:rslave \
3132
--volume /etc/modprobe.d:/etc/modprobe.d:rslave \
3233
--volume /etc/sysconfig:/etc/sysconfig:rslave \
3334
--volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \

‎assets/tuned/manifests/ds-tuned.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ spec:
3333
name: tuned
3434
securityContext:
3535
privileged: true
36+
readOnlyRootFilesystem: true
3637
terminationMessagePath: /dev/termination-log
3738
terminationMessagePolicy: FallbackToLogsOnError
3839
volumeMounts:

‎hack/dockerfile_install_support.sh‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ INSTALL_PKGS="nmap-ncat procps-ng pciutils"
88
# TuneD pre-installation steps
99
cp -r /root/assets/bin/* /usr/local/bin
1010
mkdir -p /etc/grub.d/ /boot /run/ocp-tuned
11+
chown -R 499:499 /run/ocp-tuned # the operator must be able to write metrics client CA in a temporary directory
1112

1213
source /etc/os-release
1314
if [[ "${ID}" == "centos" ]]; then
@@ -43,8 +44,12 @@ else
4344
fi
4445

4546
# TuneD post-installation steps
46-
rm -rf /etc/tuned/recommend.d
47+
rm -rf /etc/tuned/recommend.d /var/lib/tuned
4748
echo auto > /etc/tuned/profile_mode
4849
sed -Ei 's|^#?\s*enable_unix_socket\s*=.*$|enable_unix_socket = 1|;s|^#?\s*rollback\s*=.*$|rollback = not_on_exit|;s|^#?\s*profile_dirs\s*=.*$|profile_dirs = /usr/lib/tuned/profiles,/usr/lib/tuned,/var/lib/ocp-tuned/profiles|' \
4950
/etc/tuned/tuned-main.conf
51+
mv /etc/tuned /etc/tuned.orig
52+
ln -s /var/lib/ocp-tuned/tuned /etc/tuned
53+
ln -s /host/var/lib/ocp-tuned /var/lib/ocp-tuned
54+
ln -s /host/var/lib/tuned /var/lib/tuned
5055
touch /etc/sysctl.conf

‎pkg/tuned/controller.go‎

Lines changed: 26 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,9 @@ const (
8181
tunedGracefulExitWait = time.Second * time.Duration(10)
8282
ocpTunedHome = "/var/lib/ocp-tuned"
8383
ocpTunedRunDir = "/run/" + programName
84+
ocpTunedPersist = ocpTunedRunDir + "/persist"
8485
ocpTunedProvider = ocpTunedHome + "/provider"
86+
tunedPersistHome = "/var/lib/tuned"
8587
// With the less aggressive rate limiter, retries will happen at 100ms*2^(retry_n-1):
8688
// 100ms, 200ms, 400ms, 800ms, 1.6s, 3.2s, 6.4s, 12.8s, 25.6s, 51.2s, 102.4s, 3.4m, 6.8m, 13.7m, 27.3m
8789
maxRetries = 15
@@ -92,6 +94,10 @@ const (
9294
ocpTunedImageEnv = ocpTunedHome + "/image.env"
9395
tunedProfilesDirCustomHost = ocpTunedHome + "/profiles"
9496
tunedRecommendDirHost = ocpTunedHome + "/recommend.d"
97+
// The persistent ocp-tuned TuneD artifacts directory.
98+
ocpTunedHomeHost = "/host" + ocpTunedHome
99+
// The persistent tuned directory for files such as ksm-masked coming from cpu-partitioning profile.
100+
tunedPersistHomeHost = "/host" + tunedPersistHome
95101

96102
// How do we detect a reboot? The NTO operand owns and uses two separate files to track deferred updates.
97103
// 1. /var/lib/... - persistent storage which will survive across reboots. Contains the actual data.
@@ -640,37 +646,11 @@ func providerSync(provider string) (bool, error) {
640646
return true, providerExtract(provider)
641647
}
642648

643-
// switchTunedHome changes "native" container's home directory as defined by the
644-
// Containerfile to the container's home directory on the host itself.
645-
func switchTunedHome() error {
646-
const (
647-
ocpTunedHomeHost = "/host" + ocpTunedHome
648-
)
649-
650-
// Create the container's home directory on the host.
651-
if err := os.MkdirAll(ocpTunedHomeHost, os.ModePerm); err != nil {
652-
return fmt.Errorf("failed to create directory %q: %v", ocpTunedHomeHost, err)
653-
}
654-
655-
// Delete the container's home directory. We need a recursive delete, because some cross-compiling environments
656-
// populate the directory with hidden cache directories.
657-
if err := os.RemoveAll(ocpTunedHome); err != nil {
658-
return fmt.Errorf("failed to delete: %q: %v", ocpTunedHome, err)
659-
}
660-
661-
if err := util.Symlink(ocpTunedHomeHost, ocpTunedHome); err != nil {
662-
return fmt.Errorf("failed to link %q -> %q: %v", ocpTunedHome, ocpTunedHomeHost, err)
663-
}
664-
665-
err := os.Chdir(ocpTunedHome)
666-
if err != nil {
649+
func prepareOpenShiftTunedDir() error {
650+
if err := TunedRsyncEtcToHost(); err != nil {
667651
return err
668652
}
669653

670-
return nil
671-
}
672-
673-
func prepareOpenShiftTunedDir() error {
674654
// Create the following directories unless they exist.
675655
dirs := []string{
676656
tunedRecommendDirHost,
@@ -1701,8 +1681,24 @@ func retryLoop(c *Controller) (err error) {
17011681
func RunInCluster(stopCh <-chan struct{}, version string) error {
17021682
klog.Infof("starting in-cluster %s %s", programName, version)
17031683

1704-
if err := switchTunedHome(); err != nil {
1705-
return err
1684+
dirs := []string{
1685+
ocpTunedHomeHost,
1686+
tunedPersistHomeHost,
1687+
}
1688+
for _, d := range dirs {
1689+
if err := os.MkdirAll(d, os.ModePerm); err != nil {
1690+
return fmt.Errorf("failed to create directory %q: %v", d, err)
1691+
}
1692+
}
1693+
1694+
links := map[string]string{
1695+
ocpTunedHomeHost: ocpTunedPersist,
1696+
tunedPersistHomeHost: tunedPersistHome,
1697+
}
1698+
for target, source := range links {
1699+
if err := util.Symlink(target, source); err != nil {
1700+
return fmt.Errorf("failed to link %q -> %q: %v", source, target, err)
1701+
}
17061702
}
17071703

17081704
if err := prepareOpenShiftTunedDir(); err != nil {

‎pkg/tuned/run.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,21 @@ func configDaemonMode() (func(), error) {
7272
return restoreF, nil
7373
}
7474

75+
func TunedRsyncEtcToHost() error {
76+
const (
77+
source = "/etc/tuned.orig/"
78+
target = ocpTunedHome + "/tuned"
79+
)
80+
81+
cmd := exec.Command("rsync", "--delete", "-av", source, target)
82+
out, err := cmd.CombinedOutput()
83+
if err != nil {
84+
return fmt.Errorf("rsync of %q to %q failed: %v\n%s", source, target, err, out)
85+
}
86+
87+
return nil
88+
}
89+
7590
func TunedRunNoDaemon(timeout time.Duration) error {
7691
var (
7792
cmd *exec.Cmd

‎test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-master_machineconfig.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,8 @@ spec:
195195
--security-opt label=disable \
196196
--log-driver=none \
197197
--volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
198-
--volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
198+
--volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \
199+
--volume /var/lib/tuned:/host/var/lib/tuned:rslave \
199200
--volume /etc/modprobe.d:/etc/modprobe.d:rslave \
200201
--volume /etc/sysconfig:/etc/sysconfig:rslave \
201202
--volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \

‎test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-ctrcfg/openshift-bootstrap-worker_machineconfig.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,8 @@ spec:
195195
--security-opt label=disable \
196196
--log-driver=none \
197197
--volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
198-
--volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
198+
--volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \
199+
--volume /var/lib/tuned:/host/var/lib/tuned:rslave \
199200
--volume /etc/modprobe.d:/etc/modprobe.d:rslave \
200201
--volume /etc/sysconfig:/etc/sysconfig:rslave \
201202
--volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \

‎test/e2e/performanceprofile/testdata/render-expected-output/bootstrap/extra-mcp/openshift-bootstrap-master_machineconfig.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,8 @@ spec:
195195
--security-opt label=disable \
196196
--log-driver=none \
197197
--volume /var/lib/kubelet:/var/lib/kubelet:rslave,ro \
198-
--volume /var/lib/ocp-tuned:/var/lib/ocp-tuned:rslave \
198+
--volume /var/lib/ocp-tuned:/host/var/lib/ocp-tuned:rslave \
199+
--volume /var/lib/tuned:/host/var/lib/tuned:rslave \
199200
--volume /etc/modprobe.d:/etc/modprobe.d:rslave \
200201
--volume /etc/sysconfig:/etc/sysconfig:rslave \
201202
--volume /etc/sysctl.d:/etc/sysctl.d:rslave,ro \

0 commit comments

Comments
 (0)