feat: support linux v6.12 balloon stat

When the guest runs a kernel >= v6.12, the balloon stats API
will not crash and will return additional metrics: oom_kill,
alloc_stall, async_scan, direct_scan, async_reclaim, direct_reclaim

these metrics can be found in the related kernel commit:
74c025c5d7e4ac7c7ad269c1ee64da4bdfe4770c

To avoid any misunderstanding, these metrics are omitted when
the guest kernel < v6.12.

Signed-off-by: 刘顺钰 <[email protected]>

Author: laushunyu

CHANGELOG.md

@@ -26,6 +26,9 @@ and this project adheres to
 - [#4028](https://github.com/firecracker-microvm/firecracker/pull/4028):
   Firecracker now creates the log and metrics files if they do not exist,
   simplifying the launch of Firecracker by removing a manual step.
+- [#5516](https://github.com/firecracker-microvm/firecracker/pull/5516): Balloon
+  stats now supports guest kernel >= 6.12, adding metrics on guest OOM kills,
+  memory allocation stalls, and memory scan/reclaim info.
 - [#5526](https://github.com/firecracker-microvm/firecracker/pull/5526): Specify
   IA32_MTRRdefType MSR on VM boot to allow it to set page attributes for memory
   regions.

docs/ballooning.md

@@ -196,6 +196,11 @@ curl --unix-socket $socket_location -i \
 This will update the target size of the balloon to `amount_mib` and the
 statistics polling interval to `polling_interval`.
 
+> [!NOTE] Balloon inflation instructs the guest to reclaim memory which may
+> cause performance issues in the guest. The balloon statistics defined
+> [below](#virtio-balloon-statistics) can be used to decide whether it's
+> necessary to reclaim memory.
+
 ## Virtio balloon statistics
 
 The statistics are enabled by setting the `stats_polling_interval_s` field in
@@ -249,6 +254,30 @@ device has support for the following statistics:
 - `VIRTIO_BALLOON_S_HTLB_PGFAIL`: The number of failed hugetlb page allocations
   in the guest.
 
+Since linux v6.12, following metrics added(omitted in < v6.12):
+
+- `VIRTIO_BALLOON_S_OOM_KILL`: OOM killer invocations, indicating critical
+  memory pressure.
+- `VIRTIO_BALLOON_S_ALLOC_STALL`: Counter of Allocation enter a slow path to
+  gain more memory page. The reclaim/scan metrics can reveal what is actually
+  happening.
+- `VIRTIO_BALLOON_S_ASYNC_SCAN`: Amount of memory scanned asynchronously.
+- `VIRTIO_BALLOON_S_DIRECT_SCAN`: Amount of memory scanned directly.
+- `VIRTIO_BALLOON_S_ASYNC_RECLAIM`: Amount of memory reclaimed asynchronously.
+- `VIRTIO_BALLOON_S_DIRECT_RECLAIM`: Amount of memory reclaimed directly.
+
+When the pages_high watermark is reached, Linux `kswapd` performs asynchronous
+page reclaim, which increases ASYNC_SCAN and ASYNC_RECLAIM.
+
+When a process allocates more memory than the kernel can provide, the process is
+stalled while pages are reclaimed directly, which increases DIRECT_SCAN and
+DIRECT_RECLAIM.
+
+> `man sar`: %vmeff Calculated as pgsteal(RECLAIM) / pgscan(SCAN), this is a
+> metric of the efficiency of page reclaim. If it is near 100% then almost every
+> page coming off the tail of the inactive list is being reaped. If it gets too
+> low (e.g. less than 30%) then the virtual memory is having some difficulty.
+
 The driver is queried for updated statistics every time the amount of time
 specified in that field passes. The driver may not provide all the statistics
 when queried, in which case the old values of the missing statistics are

src/firecracker/swagger/firecracker.yaml

@@ -967,6 +967,30 @@ definitions:
         description: The number of failed hugetlb page allocations in the guest.
         type: integer
         format: int64
+      oom_kill:
+        description: OOM killer invocations, indicating critical memory pressure.
+        type: integer
+        format: int64
+      alloc_stall:
+        description: Counter of Allocation enter a slow path to gain more memory page. The reclaim/scan metrics can reveal what is actually happening.
+        type: integer
+        format: int64
+      async_scan:
+        description: Amount of memory scanned asynchronously.
+        type: integer
+        format: int64
+      direct_scan:
+        description: Amount of memory scanned directly.
+        type: integer
+        format: int64
+      async_reclaim:
+        description: Amount of memory reclaimed asynchronously.
+        type: integer
+        format: int64
+      direct_reclaim:
+        description: Amount of memory reclaimed directly.
+        type: integer
+        format: int64
 
   BalloonStartCmd:
     type: object

src/vmm/src/devices/virtio/balloon/device.rs

@@ -20,10 +20,12 @@ use super::{
     FREE_PAGE_HINT_STOP, INFLATE_INDEX, MAX_PAGE_COMPACT_BUFFER, MAX_PAGES_IN_DESC,
     MIB_TO_4K_PAGES, STATS_INDEX, VIRTIO_BALLOON_F_DEFLATE_ON_OOM,
     VIRTIO_BALLOON_F_FREE_PAGE_HINTING, VIRTIO_BALLOON_F_FREE_PAGE_REPORTING,
-    VIRTIO_BALLOON_F_STATS_VQ, VIRTIO_BALLOON_PFN_SHIFT, VIRTIO_BALLOON_S_AVAIL,
-    VIRTIO_BALLOON_S_CACHES, VIRTIO_BALLOON_S_HTLB_PGALLOC, VIRTIO_BALLOON_S_HTLB_PGFAIL,
-    VIRTIO_BALLOON_S_MAJFLT, VIRTIO_BALLOON_S_MEMFREE, VIRTIO_BALLOON_S_MEMTOT,
-    VIRTIO_BALLOON_S_MINFLT, VIRTIO_BALLOON_S_SWAP_IN, VIRTIO_BALLOON_S_SWAP_OUT,
+    VIRTIO_BALLOON_F_STATS_VQ, VIRTIO_BALLOON_PFN_SHIFT, VIRTIO_BALLOON_S_ALLOC_STALL,
+    VIRTIO_BALLOON_S_ASYNC_RECLAIM, VIRTIO_BALLOON_S_ASYNC_SCAN, VIRTIO_BALLOON_S_AVAIL,
+    VIRTIO_BALLOON_S_CACHES, VIRTIO_BALLOON_S_DIRECT_RECLAIM, VIRTIO_BALLOON_S_DIRECT_SCAN,
+    VIRTIO_BALLOON_S_HTLB_PGALLOC, VIRTIO_BALLOON_S_HTLB_PGFAIL, VIRTIO_BALLOON_S_MAJFLT,
+    VIRTIO_BALLOON_S_MEMFREE, VIRTIO_BALLOON_S_MEMTOT, VIRTIO_BALLOON_S_MINFLT,
+    VIRTIO_BALLOON_S_OOM_KILL, VIRTIO_BALLOON_S_SWAP_IN, VIRTIO_BALLOON_S_SWAP_OUT,
 };
 use crate::devices::virtio::balloon::BalloonError;
 use crate::devices::virtio::device::ActiveState;
@@ -184,6 +186,24 @@ pub struct BalloonStats {
     /// in the guest.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub hugetlb_failures: Option<u64>,
+    /// OOM killer invocations. since linux v6.12.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub oom_kill: Option<u64>,
+    /// Stall count of memory allocatoin. since linux v6.12.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub alloc_stall: Option<u64>,
+    /// Amount of memory scanned asynchronously. since linux v6.12.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub async_scan: Option<u64>,
+    /// Amount of memory scanned directly. since linux v6.12.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub direct_scan: Option<u64>,
+    /// Amount of memory reclaimed asynchronously. since linux v6.12.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub async_reclaim: Option<u64>,
+    /// Amount of memory reclaimed directly. since linux v6.12.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub direct_reclaim: Option<u64>,
 }
 
 impl BalloonStats {
@@ -200,6 +220,12 @@ impl BalloonStats {
             VIRTIO_BALLOON_S_CACHES => self.disk_caches = val,
             VIRTIO_BALLOON_S_HTLB_PGALLOC => self.hugetlb_allocations = val,
             VIRTIO_BALLOON_S_HTLB_PGFAIL => self.hugetlb_failures = val,
+            VIRTIO_BALLOON_S_OOM_KILL => self.oom_kill = val,
+            VIRTIO_BALLOON_S_ALLOC_STALL => self.alloc_stall = val,
+            VIRTIO_BALLOON_S_ASYNC_SCAN => self.async_scan = val,
+            VIRTIO_BALLOON_S_DIRECT_SCAN => self.direct_scan = val,
+            VIRTIO_BALLOON_S_ASYNC_RECLAIM => self.async_reclaim = val,
+            VIRTIO_BALLOON_S_DIRECT_RECLAIM => self.direct_reclaim = val,
             _ => {
                 return Err(BalloonError::MalformedPayload);
             }
@@ -1035,6 +1061,12 @@ pub(crate) mod tests {
             disk_caches: Some(0),
             hugetlb_allocations: Some(0),
             hugetlb_failures: Some(0),
+            oom_kill: None,
+            alloc_stall: None,
+            async_scan: None,
+            direct_scan: None,
+            async_reclaim: None,
+            direct_reclaim: None,
         };
 
         let mut stat = BalloonStat {
@@ -1071,6 +1103,24 @@ pub(crate) mod tests {
         stat.tag = VIRTIO_BALLOON_S_HTLB_PGFAIL;
         stats.update_with_stat(&stat).unwrap();
         assert_eq!(stats.hugetlb_failures, Some(1));
+        stat.tag = VIRTIO_BALLOON_S_OOM_KILL;
+        stats.update_with_stat(&stat).unwrap();
+        assert_eq!(stats.oom_kill, Some(1));
+        stat.tag = VIRTIO_BALLOON_S_ALLOC_STALL;
+        stats.update_with_stat(&stat).unwrap();
+        assert_eq!(stats.alloc_stall, Some(1));
+        stat.tag = VIRTIO_BALLOON_S_ASYNC_SCAN;
+        stats.update_with_stat(&stat).unwrap();
+        assert_eq!(stats.async_scan, Some(1));
+        stat.tag = VIRTIO_BALLOON_S_DIRECT_SCAN;
+        stats.update_with_stat(&stat).unwrap();
+        assert_eq!(stats.direct_scan, Some(1));
+        stat.tag = VIRTIO_BALLOON_S_ASYNC_RECLAIM;
+        stats.update_with_stat(&stat).unwrap();
+        assert_eq!(stats.async_reclaim, Some(1));
+        stat.tag = VIRTIO_BALLOON_S_DIRECT_RECLAIM;
+        stats.update_with_stat(&stat).unwrap();
+        assert_eq!(stats.direct_reclaim, Some(1));
     }
 
     #[test]

src/vmm/src/devices/virtio/balloon/mod.rs

@@ -55,7 +55,7 @@ const VIRTIO_BALLOON_F_DEFLATE_ON_OOM: u32 = 2; // Deflate balloon on OOM.
 const VIRTIO_BALLOON_F_FREE_PAGE_HINTING: u32 = 3; // Enable free page hinting
 const VIRTIO_BALLOON_F_FREE_PAGE_REPORTING: u32 = 5; // Enable free page reporting
 
-// The statistics tags.
+// The statistics tags. defined in linux "include/uapi/linux/virtio_balloon.h".
 const VIRTIO_BALLOON_S_SWAP_IN: u16 = 0;
 const VIRTIO_BALLOON_S_SWAP_OUT: u16 = 1;
 const VIRTIO_BALLOON_S_MAJFLT: u16 = 2;
@@ -66,6 +66,12 @@ const VIRTIO_BALLOON_S_AVAIL: u16 = 6;
 const VIRTIO_BALLOON_S_CACHES: u16 = 7;
 const VIRTIO_BALLOON_S_HTLB_PGALLOC: u16 = 8;
 const VIRTIO_BALLOON_S_HTLB_PGFAIL: u16 = 9;
+const VIRTIO_BALLOON_S_OOM_KILL: u16 = 10;
+const VIRTIO_BALLOON_S_ALLOC_STALL: u16 = 11;
+const VIRTIO_BALLOON_S_ASYNC_SCAN: u16 = 12;
+const VIRTIO_BALLOON_S_DIRECT_SCAN: u16 = 13;
+const VIRTIO_BALLOON_S_ASYNC_RECLAIM: u16 = 14;
+const VIRTIO_BALLOON_S_DIRECT_RECLAIM: u16 = 15;
 
 /// Balloon device related errors.
 #[derive(Debug, thiserror::Error, displaydoc::Display)]

src/vmm/src/devices/virtio/balloon/persist.rs

@@ -41,6 +41,12 @@ pub struct BalloonStatsState {
     disk_caches: Option<u64>,
     hugetlb_allocations: Option<u64>,
     hugetlb_failures: Option<u64>,
+    oom_kill: Option<u64>,
+    alloc_stall: Option<u64>,
+    async_scan: Option<u64>,
+    direct_scan: Option<u64>,
+    async_reclaim: Option<u64>,
+    direct_reclaim: Option<u64>,
 }
 
 impl BalloonStatsState {
@@ -56,6 +62,12 @@ impl BalloonStatsState {
             disk_caches: stats.disk_caches,
             hugetlb_allocations: stats.hugetlb_allocations,
             hugetlb_failures: stats.hugetlb_failures,
+            oom_kill: stats.oom_kill,
+            alloc_stall: stats.alloc_stall,
+            async_scan: stats.async_scan,
+            direct_scan: stats.direct_scan,
+            async_reclaim: stats.async_reclaim,
+            direct_reclaim: stats.direct_reclaim,
         }
     }
 
@@ -75,6 +87,12 @@ impl BalloonStatsState {
             disk_caches: self.disk_caches,
             hugetlb_allocations: self.hugetlb_allocations,
             hugetlb_failures: self.hugetlb_failures,
+            oom_kill: self.oom_kill,
+            alloc_stall: self.alloc_stall,
+            async_scan: self.async_scan,
+            direct_scan: self.direct_scan,
+            async_reclaim: self.async_reclaim,
+            direct_reclaim: self.direct_reclaim,
         }
     }
 }
@@ -192,7 +210,6 @@ impl Persist<'_> for Balloon {
 
 #[cfg(test)]
 mod tests {
-
     use super::*;
     use crate::devices::virtio::device::VirtioDevice;
     use crate::devices::virtio::test_utils::{default_interrupt, default_mem};