tests: Extend 'test_vfio' with block device passthrough

Fixes: cloud-hypervisor#2822

Signed-off-by: Bo Chen <[email protected]>

Author: likebreath

test_data/cloud-init/ubuntu/user-data

@@ -36,15 +36,18 @@ write_files:
         #!/bin/bash
 
         mount /dev/vdc /mnt
+        bash -c "echo 1af4 1042 > /sys/bus/pci/drivers/vfio-pci/new_id"
+        bash -c "echo 0000:00:05.0 > /sys/bus/pci/devices/0000\:00\:05.0/driver/unbind"
+        bash -c "echo 0000:00:05.0 > /sys/bus/pci/drivers/vfio-pci/bind"
         bash -c "echo 1af4 1041 > /sys/bus/pci/drivers/vfio-pci/new_id"
-        bash -c "echo 0000:00:06.0 > /sys/bus/pci/devices/0000\:00\:06.0/driver/unbind"
-        bash -c "echo 0000:00:06.0 > /sys/bus/pci/drivers/vfio-pci/bind"
         bash -c "echo 0000:00:07.0 > /sys/bus/pci/devices/0000\:00\:07.0/driver/unbind"
         bash -c "echo 0000:00:07.0 > /sys/bus/pci/drivers/vfio-pci/bind"
+        bash -c "echo 0000:00:08.0 > /sys/bus/pci/devices/0000\:00\:08.0/driver/unbind"
+        bash -c "echo 0000:00:08.0 > /sys/bus/pci/drivers/vfio-pci/bind"
         # 1G ram requires 512 pages
         echo 512 | sudo tee /proc/sys/vm/nr_hugepages
         sudo chmod a+rwX /dev/hugepages
-        /mnt/cloud-hypervisor --kernel /mnt/vmlinux --cmdline "console=hvc0 reboot=k panic=1 nomodules root=/dev/vda1 VFIOTAG" --disk path=/mnt/focal-server-cloudimg-amd64-custom-20210609-0.qcow2 path=/mnt/cloudinit.img --cpus boot=1 --memory size=512M,hotplug_size=1G,hugepages=on --device path=/sys/bus/pci/devices/0000:00:06.0/ path=/sys/bus/pci/devices/0000:00:07.0/ --api-socket /tmp/ch_api.sock
+        /mnt/cloud-hypervisor --kernel /mnt/vmlinux --cmdline "console=hvc0 reboot=k panic=1 nomodules root=/dev/vda1 VFIOTAG" --disk path=/mnt/focal-server-cloudimg-amd64-custom-20210609-0.qcow2 path=/mnt/cloudinit.img --cpus boot=1 --memory size=512M,hotplug_size=1G,hugepages=on --device path=/sys/bus/pci/devices/0000:00:05.0/ path=/sys/bus/pci/devices/0000:00:07.0/ path=/sys/bus/pci/devices/0000:00:08.0/ --api-socket /tmp/ch_api.sock
 
   -
     path: /etc/systemd/system/notify-booted.service

tests/integration.rs

@@ -3238,6 +3238,8 @@ mod tests {
         // line (We tag the command line from cloud-hypervisor for that purpose).
         // The third device is added to validate that hotplug works correctly since
         // it is being added to the L2 VM through hotplugging mechanism.
+        // Also, we pass-through a vitio-blk device to the L2 VM to test the 32-bit
+        // vfio device support
         fn test_vfio() {
             setup_vfio_network_interfaces();
 
@@ -3263,7 +3265,7 @@ mod tests {
             )
             .expect("copying of cloud-init disk failed");
 
-            let mut vfio_disk_path = workload_path;
+            let mut vfio_disk_path = workload_path.clone();
             vfio_disk_path.push("vfio.img");
 
             // Create the vfio disk image
@@ -3279,6 +3281,9 @@ mod tests {
                 panic!("mkfs.ext4 command generated an error");
             }
 
+            let mut blk_file_path = workload_path;
+            blk_file_path.push("blk.img");
+
             let vfio_tap0 = "vfio-tap0";
             let vfio_tap1 = "vfio-tap1";
             let vfio_tap2 = "vfio-tap2";
@@ -3301,6 +3306,7 @@ mod tests {
                     )
                     .as_str(),
                     format!("path={}", vfio_disk_path.to_str().unwrap()).as_str(),
+                    format!("path={},iommu=on", blk_file_path.to_str().unwrap()).as_str(),
                 ])
                 .args(&[
                     "--cmdline",
@@ -3374,27 +3380,38 @@ mod tests {
                         .trim()
                         .parse::<u32>()
                         .unwrap_or_default(),
-                    7,
+                    8,
+                );
+
+                // Check both if /dev/vdc exists and if the block size is 16M in L2 VM
+                assert_eq!(
+                    guest
+                        .ssh_command_l2_1("lsblk | grep vdc | grep -c 16M")
+                        .unwrap()
+                        .trim()
+                        .parse::<u32>()
+                        .unwrap_or_default(),
+                    1
                 );
 
                 // Hotplug an extra virtio-net device through L2 VM.
                 guest.ssh_command_l1(
-                    "echo 0000:00:08.0 | sudo tee /sys/bus/pci/devices/0000:00:08.0/driver/unbind",
+                    "echo 0000:00:09.0 | sudo tee /sys/bus/pci/devices/0000:00:09.0/driver/unbind",
                 ).unwrap();
                 guest
                     .ssh_command_l1(
-                        "echo 0000:00:08.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind",
+                        "echo 0000:00:09.0 | sudo tee /sys/bus/pci/drivers/vfio-pci/bind",
                     )
                     .unwrap();
                 let vfio_hotplug_output = guest
                     .ssh_command_l1(
                         "sudo /mnt/ch-remote \
                  --api-socket=/tmp/ch_api.sock \
-                 add-device path=/sys/bus/pci/devices/0000:00:08.0,id=vfio123",
+                 add-device path=/sys/bus/pci/devices/0000:00:09.0,id=vfio123",
                     )
                     .unwrap();
                 assert!(
-                    vfio_hotplug_output.contains("{\"id\":\"vfio123\",\"bdf\":\"0000:00:07.0\"}")
+                    vfio_hotplug_output.contains("{\"id\":\"vfio123\",\"bdf\":\"0000:00:08.0\"}")
                 );
 
                 thread::sleep(std::time::Duration::new(10, 0));
@@ -3414,15 +3431,15 @@ mod tests {
 
                 // Check the amount of PCI devices appearing in L2 VM.
                 // There should be one more device than before, raising the count
-                // up to 8 PCI devices.
+                // up to 9 PCI devices.
                 assert_eq!(
                     guest
                         .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l")
                         .unwrap()
                         .trim()
                         .parse::<u32>()
                         .unwrap_or_default(),
-                    8,
+                    9,
                 );
 
                 // Let's now verify that we can correctly remove the virtio-net
@@ -3438,15 +3455,15 @@ mod tests {
                 thread::sleep(std::time::Duration::new(10, 0));
 
                 // Check the amount of PCI devices appearing in L2 VM is back down
-                // to 7 devices.
+                // to 8 devices.
                 assert_eq!(
                     guest
                         .ssh_command_l2_1("ls /sys/bus/pci/devices | wc -l")
                         .unwrap()
                         .trim()
                         .parse::<u32>()
                         .unwrap_or_default(),
-                    7,
+                    8,
                 );
 
                 // Perform memory hotplug in L2 and validate the memory is showing