From 23026417a888535c84ec26f11a2b9da66068f0ad Mon Sep 17 00:00:00 2001 From: Norio Nomura Date: Sun, 7 Jul 2024 14:42:53 +0900 Subject: [PATCH 1/5] Create `rosetta.conf` for `binfmt.d(5)` Sometime `qemu-user-static` interpreters are registered with `systemd-binfmt.service(8)` by creating configuration files for `binfmt.d(5)`. By creating a `binfmt.d(5)` configuration for Rosetta, Rosetta will be prioritized over `qemu-user-static` regardless of the execution timing of `systemd-binfmt.service(8)`. In subsequent startups, Rosetta will already be registered with binfmt_misc at this stage, so a check has been added to prevent errors due to duplicate registrations. Signed-off-by: Norio Nomura --- .../cidata.TEMPLATE.d/boot/05-rosetta-volume.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/cidata/cidata.TEMPLATE.d/boot/05-rosetta-volume.sh b/pkg/cidata/cidata.TEMPLATE.d/boot/05-rosetta-volume.sh index 2ad4719f0ad..9420001799e 100755 --- a/pkg/cidata/cidata.TEMPLATE.d/boot/05-rosetta-volume.sh +++ b/pkg/cidata/cidata.TEMPLATE.d/boot/05-rosetta-volume.sh @@ -29,7 +29,13 @@ else fi if [ "$LIMA_CIDATA_ROSETTA_BINFMT" = "true" ]; then - echo \ - ':rosetta:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00:\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/mnt/lima-rosetta/rosetta:OCF' \ - >/proc/sys/fs/binfmt_misc/register + rosetta_binfmt=":rosetta:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00:\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/mnt/lima-rosetta/rosetta:OCF" + + # If rosetta is not registered in binfmt_misc, register it. + [ -f /proc/sys/fs/binfmt_misc/rosetta ] || echo "$rosetta_binfmt" >/proc/sys/fs/binfmt_misc/register + + # Create binfmt.d(5) configuration to prioritize rosetta even if qemu-user-static is installed on systemd based systems. + binfmtd_conf=/usr/lib/binfmt.d/rosetta.conf + # If the binfmt.d directory exists, consider systemd-binfmt.service(8) to be enabled and create the configuration file. + [ ! -d "$(dirname "$binfmtd_conf")" ] || [ -f "$binfmtd_conf" ] || echo "$rosetta_binfmt" >"$binfmtd_conf" fi From ba11ab58af8945220431ff226075973b461f1823 Mon Sep 17 00:00:00 2001 From: Norio Nomura Date: Sun, 7 Jul 2024 14:49:36 +0900 Subject: [PATCH 2/5] Mount the rosetta volume in `user-data` After creating the `binfmt.d(5)` configuration for Rosetta, `systemd-binfmt.service(8)` attempts to register at an earlier stage in subsequent boots. To prevent errors from not finding the Rosetta interpreter, `user-data` is used to mount the Rosetta volume earlier. Signed-off-by: Norio Nomura --- .../boot/05-rosetta-volume.sh | 39 +++++++++++-------- pkg/cidata/cidata.TEMPLATE.d/user-data | 12 ++++-- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/pkg/cidata/cidata.TEMPLATE.d/boot/05-rosetta-volume.sh b/pkg/cidata/cidata.TEMPLATE.d/boot/05-rosetta-volume.sh index 9420001799e..9488a4c56e2 100755 --- a/pkg/cidata/cidata.TEMPLATE.d/boot/05-rosetta-volume.sh +++ b/pkg/cidata/cidata.TEMPLATE.d/boot/05-rosetta-volume.sh @@ -10,26 +10,31 @@ if [ -f /etc/alpine-release ]; then rc-service qemu-binfmt stop --ifstarted fi -mkdir -p /mnt/lima-rosetta - -#Check selinux is enabled by kernel -if [ -d /sys/fs/selinux ]; then - ########################################################################################## - ## When using vz & virtiofs, initially container_file_t selinux label - ## was considered which works perfectly for container work loads - ## but it might break for other work loads if the process is running with - ## different label. Also these are the remote mounts from the host machine, - ## so keeping the label as nfs_t fits right. Package container-selinux by - ## default adds rules for nfs_t context which allows container workloads to work as well. - ## https://github.com/lima-vm/lima/pull/1965 - ########################################################################################## - mount -t virtiofs vz-rosetta /mnt/lima-rosetta -o context="system_u:object_r:nfs_t:s0" -else - mount -t virtiofs vz-rosetta /mnt/lima-rosetta +# Mount the rosetta volume for non cloud-init based images +rosetta_interpreter=/mnt/lima-rosetta/rosetta +if [ ! -f "$rosetta_interpreter" ]; then + rosetta_mountpoint=$(dirname "$rosetta_interpreter") + mkdir -p "$rosetta_mountpoint" + + #Check selinux is enabled by kernel + if [ -d /sys/fs/selinux ]; then + ########################################################################################## + ## When using vz & virtiofs, initially container_file_t selinux label + ## was considered which works perfectly for container work loads + ## but it might break for other work loads if the process is running with + ## different label. Also these are the remote mounts from the host machine, + ## so keeping the label as nfs_t fits right. Package container-selinux by + ## default adds rules for nfs_t context which allows container workloads to work as well. + ## https://github.com/lima-vm/lima/pull/1965 + ########################################################################################## + mount -t virtiofs vz-rosetta "$rosetta_mountpoint" -o context="system_u:object_r:nfs_t:s0" + else + mount -t virtiofs vz-rosetta "$rosetta_mountpoint" + fi fi if [ "$LIMA_CIDATA_ROSETTA_BINFMT" = "true" ]; then - rosetta_binfmt=":rosetta:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00:\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/mnt/lima-rosetta/rosetta:OCF" + rosetta_binfmt=":rosetta:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00:\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:$rosetta_interpreter:OCF" # If rosetta is not registered in binfmt_misc, register it. [ -f /proc/sys/fs/binfmt_misc/rosetta ] || echo "$rosetta_binfmt" >/proc/sys/fs/binfmt_misc/register diff --git a/pkg/cidata/cidata.TEMPLATE.d/user-data b/pkg/cidata/cidata.TEMPLATE.d/user-data index 26060b0c1c0..f389c26a9af 100644 --- a/pkg/cidata/cidata.TEMPLATE.d/user-data +++ b/pkg/cidata/cidata.TEMPLATE.d/user-data @@ -11,14 +11,18 @@ package_upgrade: true package_reboot_if_required: true {{- end }} -{{- if or (eq .MountType "9p") (eq .MountType "virtiofs") }} -{{- if .Mounts }} +{{- if or .RosettaEnabled (or (eq .MountType "9p") (eq .MountType "virtiofs")) }} mounts: - {{- range $m := $.Mounts}} + # Mount the rosetta volume before systemd-binfmt.service(8) starts + {{- if .RosettaEnabled }} +- ["vz-rosetta", "/mnt/lima-rosetta", "virtiofs", "context=\"system_u:object_r:nfs_t:s0\""] + {{- end }} + {{- if .Mounts }} + {{- range $m := $.Mounts}} - [{{$m.Tag}}, {{$m.MountPoint}}, {{$m.Type}}, "{{$m.Options}}", "0", "0"] + {{- end }} {{- end }} {{- end }} -{{- end }} {{- if .TimeZone }} timezone: {{.TimeZone}} From 50c10f3eb2fb6fee55041f467281a053b28558da Mon Sep 17 00:00:00 2001 From: Norio Nomura Date: Wed, 10 Jul 2024 10:37:35 +0900 Subject: [PATCH 3/5] Change comment to using template syntax Signed-off-by: Norio Nomura --- pkg/cidata/cidata.TEMPLATE.d/user-data | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/cidata/cidata.TEMPLATE.d/user-data b/pkg/cidata/cidata.TEMPLATE.d/user-data index f389c26a9af..de494d814cf 100644 --- a/pkg/cidata/cidata.TEMPLATE.d/user-data +++ b/pkg/cidata/cidata.TEMPLATE.d/user-data @@ -13,8 +13,7 @@ package_reboot_if_required: true {{- if or .RosettaEnabled (or (eq .MountType "9p") (eq .MountType "virtiofs")) }} mounts: - # Mount the rosetta volume before systemd-binfmt.service(8) starts - {{- if .RosettaEnabled }} + {{- if .RosettaEnabled }}{{/* Mount the rosetta volume before systemd-binfmt.service(8) starts */}} - ["vz-rosetta", "/mnt/lima-rosetta", "virtiofs", "context=\"system_u:object_r:nfs_t:s0\""] {{- end }} {{- if .Mounts }} From 7c13a72d62191f86c7361dc90c3b0715f098ffd3 Mon Sep 17 00:00:00 2001 From: Norio Nomura Date: Thu, 11 Jul 2024 19:14:01 +0900 Subject: [PATCH 4/5] Remove mounting vz-rosetta from 05-rosetta-volume.sh Since the mount option workaround for selinux to vz-rosetta will be covered in 05-lima-mounts.sh. Signed-off-by: Norio Nomura --- .../boot/05-rosetta-volume.sh | 25 +------------------ pkg/cidata/cidata.TEMPLATE.d/user-data | 2 +- 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/pkg/cidata/cidata.TEMPLATE.d/boot/05-rosetta-volume.sh b/pkg/cidata/cidata.TEMPLATE.d/boot/05-rosetta-volume.sh index 9488a4c56e2..bb8369bf445 100755 --- a/pkg/cidata/cidata.TEMPLATE.d/boot/05-rosetta-volume.sh +++ b/pkg/cidata/cidata.TEMPLATE.d/boot/05-rosetta-volume.sh @@ -10,31 +10,8 @@ if [ -f /etc/alpine-release ]; then rc-service qemu-binfmt stop --ifstarted fi -# Mount the rosetta volume for non cloud-init based images -rosetta_interpreter=/mnt/lima-rosetta/rosetta -if [ ! -f "$rosetta_interpreter" ]; then - rosetta_mountpoint=$(dirname "$rosetta_interpreter") - mkdir -p "$rosetta_mountpoint" - - #Check selinux is enabled by kernel - if [ -d /sys/fs/selinux ]; then - ########################################################################################## - ## When using vz & virtiofs, initially container_file_t selinux label - ## was considered which works perfectly for container work loads - ## but it might break for other work loads if the process is running with - ## different label. Also these are the remote mounts from the host machine, - ## so keeping the label as nfs_t fits right. Package container-selinux by - ## default adds rules for nfs_t context which allows container workloads to work as well. - ## https://github.com/lima-vm/lima/pull/1965 - ########################################################################################## - mount -t virtiofs vz-rosetta "$rosetta_mountpoint" -o context="system_u:object_r:nfs_t:s0" - else - mount -t virtiofs vz-rosetta "$rosetta_mountpoint" - fi -fi - if [ "$LIMA_CIDATA_ROSETTA_BINFMT" = "true" ]; then - rosetta_binfmt=":rosetta:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00:\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:$rosetta_interpreter:OCF" + rosetta_binfmt=":rosetta:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00:\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/mnt/lima-rosetta/rosetta:OCF" # If rosetta is not registered in binfmt_misc, register it. [ -f /proc/sys/fs/binfmt_misc/rosetta ] || echo "$rosetta_binfmt" >/proc/sys/fs/binfmt_misc/register diff --git a/pkg/cidata/cidata.TEMPLATE.d/user-data b/pkg/cidata/cidata.TEMPLATE.d/user-data index de494d814cf..32995307a87 100644 --- a/pkg/cidata/cidata.TEMPLATE.d/user-data +++ b/pkg/cidata/cidata.TEMPLATE.d/user-data @@ -14,7 +14,7 @@ package_reboot_if_required: true {{- if or .RosettaEnabled (or (eq .MountType "9p") (eq .MountType "virtiofs")) }} mounts: {{- if .RosettaEnabled }}{{/* Mount the rosetta volume before systemd-binfmt.service(8) starts */}} -- ["vz-rosetta", "/mnt/lima-rosetta", "virtiofs", "context=\"system_u:object_r:nfs_t:s0\""] +- [vz-rosetta, /mnt/lima-rosetta, virtiofs] {{- end }} {{- if .Mounts }} {{- range $m := $.Mounts}} From 5ef067b055fb8cfafd3b6f4cdc985d5040a54ea5 Mon Sep 17 00:00:00 2001 From: Norio Nomura Date: Sat, 13 Jul 2024 09:11:41 +0900 Subject: [PATCH 5/5] Unregister rosetta if `.rosetta.binfmt` is not true If `binfmt.d/rosetta.conf` already exists, Rosetta might already be registered by the time this script is called. To apply the settings in `lima.yml`, if Rosetta is already registered, the script will remove `binfmt.d/rosetta.conf` and unregister it from `binfmt_misc`. Signed-off-by: Norio Nomura --- pkg/cidata/cidata.TEMPLATE.d/boot/05-rosetta-volume.sh | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/cidata/cidata.TEMPLATE.d/boot/05-rosetta-volume.sh b/pkg/cidata/cidata.TEMPLATE.d/boot/05-rosetta-volume.sh index bb8369bf445..38e71b3b00b 100755 --- a/pkg/cidata/cidata.TEMPLATE.d/boot/05-rosetta-volume.sh +++ b/pkg/cidata/cidata.TEMPLATE.d/boot/05-rosetta-volume.sh @@ -10,14 +10,20 @@ if [ -f /etc/alpine-release ]; then rc-service qemu-binfmt stop --ifstarted fi +binfmt_entry=/proc/sys/fs/binfmt_misc/rosetta +binfmtd_conf=/usr/lib/binfmt.d/rosetta.conf if [ "$LIMA_CIDATA_ROSETTA_BINFMT" = "true" ]; then rosetta_binfmt=":rosetta:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x3e\x00:\xff\xff\xff\xff\xff\xfe\xfe\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/mnt/lima-rosetta/rosetta:OCF" # If rosetta is not registered in binfmt_misc, register it. - [ -f /proc/sys/fs/binfmt_misc/rosetta ] || echo "$rosetta_binfmt" >/proc/sys/fs/binfmt_misc/register + [ -f "$binfmt_entry" ] || echo "$rosetta_binfmt" >/proc/sys/fs/binfmt_misc/register # Create binfmt.d(5) configuration to prioritize rosetta even if qemu-user-static is installed on systemd based systems. - binfmtd_conf=/usr/lib/binfmt.d/rosetta.conf # If the binfmt.d directory exists, consider systemd-binfmt.service(8) to be enabled and create the configuration file. [ ! -d "$(dirname "$binfmtd_conf")" ] || [ -f "$binfmtd_conf" ] || echo "$rosetta_binfmt" >"$binfmtd_conf" +else + # unregister rosetta from binfmt_misc if it exists + [ ! -f "$binfmt_entry" ] || echo -1 | "$binfmt_entry" + # remove binfmt.d(5) configuration if it exists + [ ! -f "$binfmtd_conf" ] || rm "$binfmtd_conf" fi