2.3.3 staging prep

.github/PULL_REQUEST_TEMPLATE.md

@@ -27,6 +27,7 @@ https://openzfs.github.io/openzfs-docs/Developer%20Resources/Buildbot%20Options.
 - [ ] New feature (non-breaking change which adds functionality)
 - [ ] Performance enhancement (non-breaking change which improves efficiency)
 - [ ] Code cleanup (non-breaking change which makes code smaller or more readable)
+- [ ] Quality assurance (non-breaking change which makes the code more robust against bugs)
 - [ ] Breaking change (fix or feature that would cause existing functionality to change)
 - [ ] Library ABI change (libzfs, libzfs\_core, libnvpair, libuutil and libzfsbootenv)
 - [ ] Documentation (a change to man pages or other documentation)

.github/workflows/scripts/generate-ci-type.py

@@ -29,6 +29,7 @@
 Patterns of files that are considered to trigger full CI.
 """
 FULL_RUN_REGEX = list(map(re.compile, [
+    r'\.github/workflows/scripts/.*',
     r'cmd.*',
     r'configs/.*',
     r'META',

.github/workflows/scripts/qemu-1-setup.sh

@@ -10,36 +10,12 @@ set -eu
 export DEBIAN_FRONTEND="noninteractive"
 sudo apt-get -y update
 sudo apt-get install -y axel cloud-image-utils daemonize guestfs-tools \
-  ksmtuned virt-manager linux-modules-extra-$(uname -r) zfsutils-linux
+  virt-manager linux-modules-extra-$(uname -r) zfsutils-linux
 
 # generate ssh keys
 rm -f ~/.ssh/id_ed25519
 ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519 -q -N ""
 
-# we expect RAM shortage
-cat << EOF | sudo tee /etc/ksmtuned.conf > /dev/null
-# /etc/ksmtuned.conf - Configuration file for ksmtuned
-# https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/virtualization_tuning_and_optimization_guide/chap-ksm
-KSM_MONITOR_INTERVAL=60
-
-# Millisecond sleep between ksm scans for 16Gb server.
-# Smaller servers sleep more, bigger sleep less.
-KSM_SLEEP_MSEC=30
-
-KSM_NPAGES_BOOST=0
-KSM_NPAGES_DECAY=0
-KSM_NPAGES_MIN=1000
-KSM_NPAGES_MAX=25000
-
-KSM_THRES_COEF=80
-KSM_THRES_CONST=8192
-
-LOGFILE=/var/log/ksmtuned.log
-DEBUG=1
-EOF
-sudo systemctl restart ksm
-sudo systemctl restart ksmtuned
-
 # not needed
 sudo systemctl stop docker.socket
 sudo systemctl stop multipathd.socket
@@ -65,16 +41,14 @@ $DISK
 sync
 sleep 1
 
-# swap with same size as RAM
+# swap with same size as RAM (16GiB)
 sudo mkswap $DISK-part1
 sudo swapon $DISK-part1
 
-# 60GB data disk
+# JBOD 2xdisk for OpenZFS storage (test vm's)
 SSD1="$DISK-part2"
-
-# 10GB data disk on ext4
-sudo fallocate -l 10G /test.ssd1
-SSD2=$(sudo losetup -b 4096 -f /test.ssd1 --show)
+sudo fallocate -l 12G /test.ssd2
+SSD2=$(sudo losetup -b 4096 -f /test.ssd2 --show)
 
 # adjust zfs module parameter and create pool
 exec 1>/dev/null
@@ -83,11 +57,11 @@ ARC_MAX=$((1024*1024*512))
 echo $ARC_MIN | sudo tee /sys/module/zfs/parameters/zfs_arc_min
 echo $ARC_MAX | sudo tee /sys/module/zfs/parameters/zfs_arc_max
 echo 1 | sudo tee /sys/module/zfs/parameters/zvol_use_blk_mq
-sudo zpool create -f -o ashift=12 zpool $SSD1 $SSD2 \
-  -O relatime=off -O atime=off -O xattr=sa -O compression=lz4 \
-  -O mountpoint=/mnt/tests
+sudo zpool create -f -o ashift=12 zpool $SSD1 $SSD2 -O relatime=off \
+  -O atime=off -O xattr=sa -O compression=lz4 -O sync=disabled \
+  -O redundant_metadata=none -O mountpoint=/mnt/tests
 
 # no need for some scheduler
 for i in /sys/block/s*/queue/scheduler; do
-  echo "none" | sudo tee $i > /dev/null
+  echo "none" | sudo tee $i
 done

.github/workflows/scripts/qemu-2-start.sh

@@ -14,13 +14,13 @@ OSv=$OS
 
 # compressed with .zst extension
 REPO="https://github.com/mcmilk/openzfs-freebsd-images"
-FREEBSD="$REPO/releases/download/v2024-12-14"
+FREEBSD="$REPO/releases/download/v2025-04-13"
 URLzs=""
 
 # Ubuntu mirrors
-#UBMIRROR="https://cloud-images.ubuntu.com"
+UBMIRROR="https://cloud-images.ubuntu.com"
 #UBMIRROR="https://mirrors.cloud.tencent.com/ubuntu-cloud-images"
-UBMIRROR="https://mirror.citrahost.com/ubuntu-cloud-images"
+#UBMIRROR="https://mirror.citrahost.com/ubuntu-cloud-images"
 
 # default nic model for vm's
 NIC="virtio"
@@ -37,8 +37,6 @@ case "$OS" in
   archlinux)
     OSNAME="Archlinux"
     URL="https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-cloudimg.qcow2"
-    # dns sometimes fails with that url  :/
-    echo "89.187.191.12  geo.mirror.pkgbuild.com" | sudo tee /etc/hosts > /dev/null
     ;;
   centos-stream10)
     OSNAME="CentOS Stream 10"
@@ -80,16 +78,29 @@ case "$OS" in
     BASH="/usr/local/bin/bash"
     NIC="rtl8139"
     ;;
+  freebsd13-5r)
+    OSNAME="FreeBSD 13.5-RELEASE"
+    OSv="freebsd13.0"
+    URLzs="$FREEBSD/amd64-freebsd-13.5-RELEASE.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    NIC="rtl8139"
+    ;;
+  freebsd14-1r)
+    OSNAME="FreeBSD 14.1-RELEASE"
+    OSv="freebsd14.0"
+    URLzs="$FREEBSD/amd64-freebsd-14.1-RELEASE.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
   freebsd14-2r)
     OSNAME="FreeBSD 14.2-RELEASE"
     OSv="freebsd14.0"
     URLzs="$FREEBSD/amd64-freebsd-14.2-RELEASE.qcow2.zst"
     BASH="/usr/local/bin/bash"
     ;;
-  freebsd13-4s)
-    OSNAME="FreeBSD 13.4-STABLE"
+  freebsd13-5s)
+    OSNAME="FreeBSD 13.5-STABLE"
     OSv="freebsd13.0"
-    URLzs="$FREEBSD/amd64-freebsd-13.4-STABLE.qcow2.zst"
+    URLzs="$FREEBSD/amd64-freebsd-13.5-STABLE.qcow2.zst"
     BASH="/usr/local/bin/bash"
     NIC="rtl8139"
     ;;
@@ -139,7 +150,7 @@ echo "ENV=$ENV" >> $ENV
 # result path
 echo 'RESPATH="/var/tmp/test_results"' >> $ENV
 
-# FreeBSD 13 has problems with: e1000+virtio
+# FreeBSD 13 has problems with: e1000 and virtio
 echo "NIC=$NIC" >> $ENV
 
 # freebsd15 -> used in zfs-qemu.yml
@@ -151,6 +162,14 @@ echo "OSv=\"$OSv\"" >> $ENV
 # FreeBSD 15 (Current) -> used for summary
 echo "OSNAME=\"$OSNAME\"" >> $ENV
 
+# default vm count for testings
+VMs=2
+echo "VMs=\"$VMs\"" >> $ENV
+
+# default cpu count for testing vm's
+CPU=2
+echo "CPU=\"$CPU\"" >> $ENV
+
 sudo mkdir -p "/mnt/tests"
 sudo chown -R $(whoami) /mnt/tests
 
@@ -212,12 +231,19 @@ sudo virt-install \
   --disk $DISK,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \
   --import --noautoconsole >/dev/null
 
+# enable KSM on Linux
+if [ ${OS:0:7} != "freebsd" ]; then
+  sudo virsh dommemstat --domain "openzfs" --period 5
+  sudo virsh node-memory-tune 100 50 1
+  echo 1 | sudo tee /sys/kernel/mm/ksm/run > /dev/null
+fi
+
 # Give the VMs hostnames so we don't have to refer to them with
 # hardcoded IP addresses.
 #
 # vm0:          Initial VM we install dependencies and build ZFS on.
 # vm1..2        Testing VMs
-for i in {0..9} ; do
+for ((i=0; i<=VMs; i++)); do
   echo "192.168.122.1$i vm$i" | sudo tee -a /etc/hosts
 done
 

.github/workflows/scripts/qemu-3-deps-vm.sh

@@ -4,6 +4,8 @@
 # 3) install dependencies for compiling and loading
 #
 # $1: OS name (like 'fedora41')
+# $2: (optional) Experimental Fedora kernel version, like "6.14" to
+#     install instead of Fedora defaults.
 ######################################################################
 
 set -eu
@@ -55,6 +57,7 @@ function freebsd() {
     '^samba4[[:digit:]]+$' \
     '^py3[[:digit:]]+-cffi$' \
     '^py3[[:digit:]]+-sysctl$' \
+    '^py3[[:digit:]]+-setuptools$' \
     '^py3[[:digit:]]+-packaging$'
   echo "##[endgroup]"
 }
@@ -94,6 +97,25 @@ function tumbleweed() {
   echo "##[endgroup]"
 }
 
+# $1: Kernel version to install (like '6.14rc7')
+function install_fedora_experimental_kernel {
+
+  our_version="$1"
+  sudo dnf -y copr enable @kernel-vanilla/stable
+  sudo dnf -y copr enable @kernel-vanilla/mainline
+  all="$(sudo dnf list --showduplicates kernel-*)"
+  echo "Available versions:"
+  echo "$all"
+
+  # You can have a bunch of minor variants of the version we want '6.14'.
+  # Pick the newest variant (sorted by version number).
+  specific_version=$(echo "$all" | grep $our_version | awk '{print $2}' | sort -V | tail -n 1)
+  list="$(echo "$all" | grep $specific_version | grep -Ev 'kernel-rt|kernel-selftests|kernel-debuginfo' | sed 's/.x86_64//g' | awk '{print $1"-"$2}')"
+  sudo dnf install -y $list
+  sudo dnf -y copr disable @kernel-vanilla/stable
+  sudo dnf -y copr disable @kernel-vanilla/mainline
+}
+
 # Install dependencies
 case "$1" in
   almalinux8)
@@ -132,6 +154,11 @@ case "$1" in
 
     # Fedora 42+ moves /usr/bin/script from 'util-linux' to 'util-linux-script'
     sudo dnf install -y util-linux-script || true
+
+    # Optional: Install an experimental kernel ($2 = kernel version)
+    if [ -n "${2:-}" ] ; then
+      install_fedora_experimental_kernel "$2"
+    fi
     ;;
   freebsd*)
     freebsd

.github/workflows/scripts/qemu-3-deps.sh

@@ -3,12 +3,25 @@
 #    script on it.
 #
 # $1: OS name (like 'fedora41')
+# $2: (optional) Experimental kernel version to install on fedora,
+#     like "6.14".
 ######################################################################
 
 .github/workflows/scripts/qemu-wait-for-vm.sh vm0
+
+# SPECIAL CASE:
+#
+# If the user passed in an experimental kernel version to test on Fedora,
+# we need to update the kernel version in zfs's META file to allow the
+# build to happen.  We update our local copy of META here, since we know
+# it will be rsync'd up in the next step.
+if [ -n "${2:-}" ] ; then
+  sed -i -E 's/Linux-Maximum: .+/Linux-Maximum: 99.99/g' META
+fi
+
 scp .github/workflows/scripts/qemu-3-deps-vm.sh zfs@vm0:qemu-3-deps-vm.sh
 PID=`pidof /usr/bin/qemu-system-x86_64`
-ssh zfs@vm0 '$HOME/qemu-3-deps-vm.sh' $1
+ssh zfs@vm0 '$HOME/qemu-3-deps-vm.sh' "$@"
 # wait for poweroff to succeed
 tail --pid=$PID -f /dev/null
 sleep 5 # avoid this: "error: Domain is already active"

.github/workflows/scripts/qemu-5-setup.sh

@@ -14,39 +14,33 @@ PID=$(pidof /usr/bin/qemu-system-x86_64)
 tail --pid=$PID -f /dev/null
 sudo virsh undefine openzfs
 
-# default values per test vm:
-VMs=2
-CPU=2
-
 # cpu pinning
 CPUSET=("0,1" "2,3")
 
 case "$OS" in
   freebsd*)
-    # FreeBSD can't be optimized via ksmtuned
+    # FreeBSD needs only 6GiB
     RAM=6
     ;;
   *)
-    # Linux can be optimized via ksmtuned
+    # Linux needs more memory, but can be optimized to share it via KSM
     RAM=8
     ;;
 esac
 
-# this can be different for each distro
-echo "VMs=$VMs" >> $ENV
-
 # create snapshot we can clone later
 sudo zfs snapshot zpool/openzfs@now
 
 # setup the testing vm's
 PUBKEY=$(cat ~/.ssh/id_ed25519.pub)
-for i in $(seq 1 $VMs); do
 
+# start testing VMs
+for ((i=1; i<=VMs; i++)); do
   echo "Creating disk for vm$i..."
   DISK="/dev/zvol/zpool/vm$i"
   FORMAT="raw"
-  sudo zfs clone zpool/openzfs@now zpool/vm$i
-  sudo zfs create -ps -b 64k -V 80g zpool/vm$i-2
+  sudo zfs clone zpool/openzfs@now zpool/vm$i-system
+  sudo zfs create -ps -b 64k -V 64g zpool/vm$i-tests
 
   cat <<EOF > /tmp/user-data
 #cloud-config
@@ -83,23 +77,21 @@ EOF
     --graphics none \
     --cloud-init user-data=/tmp/user-data \
     --network bridge=virbr0,model=$NIC,mac="52:54:00:83:79:0$i" \
-    --disk $DISK,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \
-    --disk $DISK-2,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \
+    --disk $DISK-system,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \
+    --disk $DISK-tests,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \
     --import --noautoconsole >/dev/null
 done
 
-# check the memory state from time to time
+# generate some memory stats
 cat <<EOF > cronjob.sh
-# $OS
 exec 1>>/var/tmp/stats.txt
 exec 2>&1
-echo "*******************************************************"
-date
+echo "********************************************************************************"
 uptime
 free -m
-df -h /mnt/tests
 zfs list
 EOF
+
 sudo chmod +x cronjob.sh
 sudo mv -f cronjob.sh /root/cronjob.sh
 echo '*/5 * * * *  /root/cronjob.sh' > crontab.txt
@@ -108,15 +100,15 @@ rm crontab.txt
 
 # check if the machines are okay
 echo "Waiting for vm's to come up...  (${VMs}x CPU=$CPU RAM=$RAM)"
-for i in $(seq 1 $VMs); do
+for ((i=1; i<=VMs; i++)); do
   .github/workflows/scripts/qemu-wait-for-vm.sh vm$i
 done
 echo "All $VMs VMs are up now."
 
 # Save the VM's serial output (ttyS0) to /var/tmp/console.txt
 # - ttyS0 on the VM corresponds to a local /dev/pty/N entry
 # - use 'virsh ttyconsole' to lookup the /dev/pty/N entry
-for i in $(seq 1 $VMs); do
+for ((i=1; i<=VMs; i++)); do
   mkdir -p $RESPATH/vm$i
   read "pty" <<< $(sudo virsh ttyconsole vm$i)
   sudo nohup bash -c "cat $pty > $RESPATH/vm$i/console.txt" &