From 0454ea01dda0c9e74f3625117bfec13700227001 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 8 Jul 2026 15:20:21 +0200 Subject: [PATCH] perf(gc): opt-in nursery-scoped valid-pointer set for non-moving minors (#6083) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every compiled program registers sync-only root scanners, which blocks the budgeted GC stepper — so nursery pressure takes the direct minor under force_full_scan (conservative, since a mid-construction value may live only in registers). That precludes the moving copying minor and falls to a non-moving mark-sweep whose ValidPointerSet build walks EVERY arena region: O(total heap) per minor, dominating GC time on any program with a large old generation. Mark and sweep are already nursery-scoped (minor traces never mark/reclaim old); only the ValidPointerSet build is whole-heap. For a fully non-moving minor it is equivalent to build it over the young generation only (eden + survivors + longlived, skipping old) — old objects are never marked-for-sweep or reclaimed in a minor and mark bits are per-cycle. The one subtlety: old remembered-set parents (dirty-page objects holding old->young pointers) are validated against the ValidPointerSet by mark_remembered_set_roots, so they're added to the exact lookup set only (not the arena runs, keeping young interior-pointer lookups correct). Gated behind PERRY_NURSERY_SCOPED_MINOR (default OFF), and only for minors with no old-page defrag selected (fully non-moving). Validated: byte-identical output flag on/off across churn + old->young retention + interior-pointer stress; evacuation verifier clean; forced-evacuate safe; ~26% faster wall-clock on a GC-heavy workload with a 400k-object old set, identical RSS. Default OFF pending broader soak. --- crates/perry-runtime/src/arena/walk.rs | 22 ++++++- crates/perry-runtime/src/gc/cycle.rs | 20 ++++++- crates/perry-runtime/src/gc/mod.rs | 18 ++++++ crates/perry-runtime/src/gc/trace.rs | 79 ++++++++++++++++++++++++++ 4 files changed, 135 insertions(+), 4 deletions(-) diff --git a/crates/perry-runtime/src/arena/walk.rs b/crates/perry-runtime/src/arena/walk.rs index e1426bd9ee3..419dfa20376 100644 --- a/crates/perry-runtime/src/arena/walk.rs +++ b/crates/perry-runtime/src/arena/walk.rs @@ -50,6 +50,10 @@ pub(crate) struct ArenaObjectCursorBuilder { region: usize, block_pos: usize, inspected_blocks: usize, + /// When set, the `old` arena region is skipped so the walk covers only the + /// young generation — used to build a nursery-scoped valid-pointer set for + /// non-moving minors (#6083). + young_only: bool, } const ARENA_CURSOR_REGION_COUNT: usize = 5; @@ -71,9 +75,20 @@ impl ArenaObjectCursorBuilder { region: 0, block_pos: 0, inspected_blocks: 0, + young_only: false, } } + /// Like `new`, but excludes the `old` arena region so the walk covers only + /// the young generation (eden + survivors + longlived). For a non-moving + /// minor this yields a valid-pointer set equivalent to the whole-heap one at + /// O(nursery) cost (#6083). + pub(crate) fn new_young_scoped(order: ArenaWalkOrder) -> Self { + let mut builder = Self::new(order); + builder.young_only = true; + builder + } + pub(crate) fn step(&mut self, remaining: &mut usize) -> Option { if *remaining == 0 { return None; @@ -123,7 +138,12 @@ impl ArenaObjectCursorBuilder { let survivor0_n = SURVIVOR_ARENA_0.with(|a| unsafe { (*a.get()).blocks.len() }); let survivor1_n = SURVIVOR_ARENA_1.with(|a| unsafe { (*a.get()).blocks.len() }); let longlived_n = LONGLIVED_ARENA.with(|a| unsafe { (*a.get()).blocks.len() }); - let old_n = OLD_ARENA.with(|a| unsafe { (*a.get()).blocks.len() }); + // #6083: a young-scoped build skips the old arena entirely. + let old_n = if self.young_only { + 0 + } else { + OLD_ARENA.with(|a| unsafe { (*a.get()).blocks.len() }) + }; self.region_lengths = [general_n, survivor0_n, survivor1_n, longlived_n, old_n]; self.region_bases = [ diff --git a/crates/perry-runtime/src/gc/cycle.rs b/crates/perry-runtime/src/gc/cycle.rs index 65dc28fa49b..e03f34ce8c9 100644 --- a/crates/perry-runtime/src/gc/cycle.rs +++ b/crates/perry-runtime/src/gc/cycle.rs @@ -1025,9 +1025,23 @@ impl GcCycleState { fn step_build_valid_pointer_set(&mut self, budget: GcWorkBudget) { let phase_start = trace_phase_start(&self.trace); - let builder = self - .valid_builder - .get_or_insert_with(ValidPointerSetBuilder::new); + // #6083: a minor with no old-page defrag selected can build the + // valid-pointer set over the young generation only — equivalent to the + // whole-heap set for a fully non-moving minor (old is never marked-for- + // sweep or reclaimed here), at O(nursery) instead of O(total heap). We + // additionally force this minor non-moving below. Gated OFF by default. + let young_scoped = nursery_scoped_minor_enabled() + && self + .minor + .as_ref() + .is_some_and(|m| m.old_page_selection.selected_pages == 0); + let builder = self.valid_builder.get_or_insert_with(|| { + if young_scoped { + ValidPointerSetBuilder::new_young_scoped() + } else { + ValidPointerSetBuilder::new() + } + }); if !builder.step(budget.work_units) { trace_phase_record(&mut self.trace, "build_valid_pointer_set", phase_start); return; diff --git a/crates/perry-runtime/src/gc/mod.rs b/crates/perry-runtime/src/gc/mod.rs index 4ac5bfc69ae..e71290fd739 100644 --- a/crates/perry-runtime/src/gc/mod.rs +++ b/crates/perry-runtime/src/gc/mod.rs @@ -189,6 +189,24 @@ pub fn gen_gc_evacuate_enabled() -> bool { }) } +/// #6083: opt-in nursery-scoped valid-pointer set for non-moving minors. +/// `PERRY_NURSERY_SCOPED_MINOR=1`/`on`/`true` builds a minor's valid-pointer set +/// over the young generation only (eden + survivors + longlived, skipping the +/// old arena) — O(nursery) instead of O(total heap). Equivalent for a fully +/// non-moving minor (old is never marked-for-sweep or reclaimed in a minor, and +/// mark bits are per-cycle). Default off while it soaks: a missed old→young edge +/// here would be a use-after-free. +pub(super) fn nursery_scoped_minor_enabled() -> bool { + use std::sync::OnceLock; + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| { + matches!( + std::env::var("PERRY_NURSERY_SCOPED_MINOR").as_deref(), + Ok("1") | Ok("on") | Ok("true") + ) + }) +} + fn gc_force_evacuate_enabled() -> bool { gen_gc_evacuate_enabled() && matches!( diff --git a/crates/perry-runtime/src/gc/trace.rs b/crates/perry-runtime/src/gc/trace.rs index 2b4c7650d73..6c70092d3be 100644 --- a/crates/perry-runtime/src/gc/trace.rs +++ b/crates/perry-runtime/src/gc/trace.rs @@ -75,6 +75,18 @@ impl ValidPointerSet { self.record_pointer_range(ptr); } + /// #6083: add an old remembered-set parent object to the exact-lookup set + /// (and range prefilter) only — NOT the arena runs. A young-scoped minor + /// build must include these so `mark_remembered_set_roots` validates the + /// old→young parent (barrier.rs) and marks its young children; but old + /// objects need no interior-pointer support in a minor (they are never + /// swept), and appending them out of address order would break the sorted + /// arena-run invariant `find_arena_floor` relies on for young objects. + pub(super) fn push_remembered_parent(&mut self, ptr: usize) { + self.lookup_set.insert(ptr); + self.record_pointer_range(ptr); + } + pub(super) fn record_tenured_nursery_bytes(&mut self, bytes: usize) { self.tenured_nursery_bytes += bytes; } @@ -181,6 +193,9 @@ pub(super) struct ValidPointerSetBuilder { arena_cursor_builder: Option, arena_cursor: Option, malloc_index: usize, + /// #6083: when the arena walk is young-scoped, an extra phase adds the old + /// remembered-set parents (objects on dirty pages) to the exact set. + young_scoped: bool, } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -188,6 +203,7 @@ pub(super) enum ValidPointerSetBuildPhase { ArenaCursorSetup, ArenaWalk, MallocWalk, + DirtyOldPageWalk, Finalize, Done, } @@ -213,6 +229,24 @@ impl ValidPointerSetBuilder { )), arena_cursor: None, malloc_index: 0, + young_scoped: false, + } + } + + /// #6083: build the valid-pointer set over the young generation only (the + /// arena walk skips the old region; the malloc walk is unchanged, since + /// young malloc objects are still swept). Equivalent to `new()` for a fully + /// non-moving minor. + pub(super) fn new_young_scoped() -> Self { + Self { + set: ValidPointerSet::new(), + phase: ValidPointerSetBuildPhase::ArenaCursorSetup, + arena_cursor_builder: Some(crate::arena::ArenaObjectCursorBuilder::new_young_scoped( + crate::arena::ArenaWalkOrder::Address, + )), + arena_cursor: None, + malloc_index: 0, + young_scoped: true, } } @@ -257,6 +291,17 @@ impl ValidPointerSetBuilder { if !self.step_malloc_walk(&mut remaining) { return false; } + self.phase = if self.young_scoped { + ValidPointerSetBuildPhase::DirtyOldPageWalk + } else { + ValidPointerSetBuildPhase::Finalize + }; + if !unbounded { + return false; + } + } + ValidPointerSetBuildPhase::DirtyOldPageWalk => { + self.step_dirty_old_page_walk(); self.phase = ValidPointerSetBuildPhase::Finalize; if !unbounded { return false; @@ -308,6 +353,40 @@ impl ValidPointerSetBuilder { false } + /// #6083: a young-scoped build excludes the old arena, but old objects that + /// hold old→young pointers (the remembered-set parents on dirty pages) must + /// stay exact-lookup-valid, or `mark_remembered_set_roots` skips them and + /// their young children are collected (UAF). Add those parents to the exact + /// set only (not the arena runs — old objects need no interior-pointer + /// support in a minor). Bounded by the dirty-page set, so the build stays + /// O(nursery + dirty-old) rather than O(total heap). + fn step_dirty_old_page_walk(&mut self) { + let snapshot = super::barrier::remembered_dirty_snapshot(); + let mut user_ptrs = Vec::new(); + // Must match EXACTLY the parent objects `mark_remembered_set_roots` + // validates against the set, across all three of its sources: + // (1) objects on dirty old pages — `dirty_pages` already folds in the + // external-entry pages, so this covers those too; + crate::arena::old_arena_walk_objects_on_pages(&snapshot.dirty_pages, |header_ptr| { + let user_ptr = unsafe { header_ptr.add(GC_HEADER_SIZE) as usize }; + user_ptrs.push(user_ptr); + }); + // (2) external-buffer dirty entries, by header (belt-and-suspenders — + // their pages are in `dirty_pages` above); + for &(_, header_addr) in &snapshot.external_dirty_entries { + user_ptrs.push(header_addr + GC_HEADER_SIZE); + } + // (3) the fallback per-header remembered set — NOT page-tracked, so the + // page walk above misses these; without them the mark skips these + // old→young parents and their young children are collected (UAF). + for &header_addr in &snapshot.fallback_headers { + user_ptrs.push(header_addr + GC_HEADER_SIZE); + } + for ptr in user_ptrs { + self.set.push_remembered_parent(ptr); + } + } + fn record_arena_header(&mut self, header_ptr: *mut u8) { let user_ptr = unsafe { header_ptr.add(GC_HEADER_SIZE) }; self.set.push_arena(user_ptr as usize);