From 72f25eea3398e9dda3db71a423baea334916d19c Mon Sep 17 00:00:00 2001 From: Daria Sukhonina Date: Mon, 20 Apr 2026 13:22:22 +0300 Subject: [PATCH 1/4] Replace `ShardedHashMap` method `insert` with debug-checked `insert_unique` --- compiler/rustc_data_structures/src/sharded.rs | 24 ++++++++++++------- .../rustc_middle/src/mir/interpret/mod.rs | 19 ++++++--------- compiler/rustc_middle/src/query/caches.rs | 4 +--- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/compiler/rustc_data_structures/src/sharded.rs b/compiler/rustc_data_structures/src/sharded.rs index e10ccccad5bb4..48b6da6ed48cb 100644 --- a/compiler/rustc_data_structures/src/sharded.rs +++ b/compiler/rustc_data_structures/src/sharded.rs @@ -1,6 +1,6 @@ use std::borrow::Borrow; use std::hash::{Hash, Hasher}; -use std::{iter, mem}; +use std::iter; use either::Either; use hashbrown::hash_table::{self, Entry, HashTable}; @@ -183,19 +183,25 @@ impl ShardedHashMap { } } + /// Insert value into the [`ShardedHashMap`] with unique key. + /// + /// Checks uniqness if debug_assertions enabled. #[inline] - pub fn insert(&self, key: K, value: V) -> Option { + pub fn insert_unique(&self, key: K, value: V) { let hash = make_hash(&key); let mut shard = self.lock_shard_by_hash(hash); - match table_entry(&mut shard, hash, &key) { - Entry::Occupied(e) => { - let previous = mem::replace(&mut e.into_mut().1, value); - Some(previous) + cfg_select! { + debug_assertions => match table_entry(&mut shard, hash, &key) { + Entry::Occupied(_) => { + panic!("tried to insert key that's already present"); + } + Entry::Vacant(e) => { + e.insert((key, value)); + } } - Entry::Vacant(e) => { - e.insert((key, value)); - None + _ => { + shard.insert_unique(hash, (key, value), |(k, _)| make_hash(k)); } } } diff --git a/compiler/rustc_middle/src/mir/interpret/mod.rs b/compiler/rustc_middle/src/mir/interpret/mod.rs index b3ec4439e3541..84a4e523d95c2 100644 --- a/compiler/rustc_middle/src/mir/interpret/mod.rs +++ b/compiler/rustc_middle/src/mir/interpret/mod.rs @@ -473,9 +473,8 @@ impl<'tcx> TyCtxt<'tcx> { } let id = self.alloc_map.reserve(); debug!("creating alloc {:?} with id {id:?}", alloc_salt.0); - let had_previous = self.alloc_map.to_alloc.insert(id, alloc_salt.0.clone()).is_some(); // We just reserved, so should always be unique. - assert!(!had_previous); + self.alloc_map.to_alloc.insert_unique(id, alloc_salt.0.clone()); dedup.insert(alloc_salt, id); id } @@ -548,21 +547,17 @@ impl<'tcx> TyCtxt<'tcx> { } /// Freezes an `AllocId` created with `reserve` by pointing it at an `Allocation`. Trying to - /// call this function twice, even with the same `Allocation` will ICE the compiler. + /// call this function twice, even with the same `Allocation` will ICE the compiler if + /// debug_assertions are enabled. pub fn set_alloc_id_memory(self, id: AllocId, mem: ConstAllocation<'tcx>) { - if let Some(old) = self.alloc_map.to_alloc.insert(id, GlobalAlloc::Memory(mem)) { - bug!("tried to set allocation ID {id:?}, but it was already existing as {old:#?}"); - } + self.alloc_map.to_alloc.insert_unique(id, GlobalAlloc::Memory(mem)) } /// Freezes an `AllocId` created with `reserve` by pointing it at a static item. Trying to - /// call this function twice, even with the same `DefId` will ICE the compiler. + /// call this function twice, even with the same `DefId` will ICE the compiler if + /// debug_assertions are enabled. pub fn set_nested_alloc_id_static(self, id: AllocId, def_id: LocalDefId) { - if let Some(old) = - self.alloc_map.to_alloc.insert(id, GlobalAlloc::Static(def_id.to_def_id())) - { - bug!("tried to set allocation ID {id:?}, but it was already existing as {old:#?}"); - } + self.alloc_map.to_alloc.insert_unique(id, GlobalAlloc::Static(def_id.to_def_id())) } } diff --git a/compiler/rustc_middle/src/query/caches.rs b/compiler/rustc_middle/src/query/caches.rs index 0c71a98b7fb29..665d4f4d7dfae 100644 --- a/compiler/rustc_middle/src/query/caches.rs +++ b/compiler/rustc_middle/src/query/caches.rs @@ -65,9 +65,7 @@ where #[inline] fn complete(&self, key: K, value: V, index: DepNodeIndex) { - // We may be overwriting another value. This is all right, since the dep-graph - // will check that the value fingerprint matches. - self.cache.insert(key, (value, index)); + self.cache.insert_unique(key, (value, index)); } fn for_each(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) { From 93d018695758deb1545bc2b8dfb64b541962915e Mon Sep 17 00:00:00 2001 From: Daria Sukhonina Date: Mon, 20 Apr 2026 13:51:01 +0300 Subject: [PATCH 2/4] Fix typo --- compiler/rustc_data_structures/src/sharded.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_data_structures/src/sharded.rs b/compiler/rustc_data_structures/src/sharded.rs index 48b6da6ed48cb..5cce53efff49e 100644 --- a/compiler/rustc_data_structures/src/sharded.rs +++ b/compiler/rustc_data_structures/src/sharded.rs @@ -185,7 +185,7 @@ impl ShardedHashMap { /// Insert value into the [`ShardedHashMap`] with unique key. /// - /// Checks uniqness if debug_assertions enabled. + /// Checks uniqueness if debug_assertions enabled. #[inline] pub fn insert_unique(&self, key: K, value: V) { let hash = make_hash(&key); From 259c65fe2259a1005fb0835497c85460acb7dcf9 Mon Sep 17 00:00:00 2001 From: Daria Sukhonina Date: Mon, 20 Apr 2026 13:54:56 +0300 Subject: [PATCH 3/4] Use `repr(packed(4))` for query cache entries --- .../rustc_data_structures/src/vec_cache.rs | 1 + compiler/rustc_middle/src/query/caches.rs | 48 +++++++++++++++---- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/compiler/rustc_data_structures/src/vec_cache.rs b/compiler/rustc_data_structures/src/vec_cache.rs index 6d026bb2c7f74..e645b38868b24 100644 --- a/compiler/rustc_data_structures/src/vec_cache.rs +++ b/compiler/rustc_data_structures/src/vec_cache.rs @@ -16,6 +16,7 @@ use rustc_index::Idx; #[cfg(test)] mod tests; +#[repr(packed(4))] struct Slot { // We never construct &Slot so it's fine for this to not be in an UnsafeCell. value: V, diff --git a/compiler/rustc_middle/src/query/caches.rs b/compiler/rustc_middle/src/query/caches.rs index 665d4f4d7dfae..2538ae82eca64 100644 --- a/compiler/rustc_middle/src/query/caches.rs +++ b/compiler/rustc_middle/src/query/caches.rs @@ -1,6 +1,9 @@ +use std::hash::{Hash, Hasher as _}; use std::sync::OnceLock; -use rustc_data_structures::sharded::ShardedHashMap; +use rustc_data_structures::fx::FxHasher; +use rustc_data_structures::hash_table::{self, HashTable}; +use rustc_data_structures::sharded::Sharded; pub use rustc_data_structures::vec_cache::VecCache; use rustc_hir::def_id::LOCAL_CRATE; use rustc_index::Idx; @@ -38,10 +41,25 @@ pub trait QueryCache: Sized { fn len(&self) -> usize; } +#[inline] +fn make_hash(val: &K) -> u64 { + let mut state = FxHasher::default(); + val.hash(&mut state); + state.finish() +} + +#[repr(packed(4))] +#[derive(Clone, Copy)] +struct PackedCacheEntry { + key: K, + value: V, + index: DepNodeIndex, +} + /// In-memory cache for queries whose keys aren't suitable for any of the /// more specialized kinds of cache. Backed by a sharded hashmap. pub struct DefaultCache { - cache: ShardedHashMap, + cache: Sharded>>, } impl Default for DefaultCache { @@ -58,26 +76,40 @@ where type Key = K; type Value = V; - #[inline(always)] + #[inline] fn lookup(&self, key: &K) -> Option<(V, DepNodeIndex)> { - self.cache.get(key) + let hash = make_hash(key); + let shard = self.cache.lock_shard_by_hash(hash); + shard.find(hash, |ent| { ent.key } == *key).map(|ent| (ent.value, ent.index)) } #[inline] fn complete(&self, key: K, value: V, index: DepNodeIndex) { - self.cache.insert_unique(key, (value, index)); + let hash = make_hash(&key); + let mut shard = self.cache.lock_shard_by_hash(hash); + cfg_select! { + debug_assertions => { + match shard.entry(hash, |ent| { ent.key } == key, |ent| make_hash(&{ ent.key })) { + hash_table::Entry::Occupied(_) => panic!("trying to complete query twice"), + hash_table::Entry::Vacant(entry) => entry.insert(PackedCacheEntry { key, value, index }), + }; + } + _ => { + shard.insert_unique(hash, PackedCacheEntry { key, value, index }, |ent| make_hash(&{ ent.key })); + } + } } fn for_each(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) { for shard in self.cache.lock_shards() { - for (k, v) in shard.iter() { - f(k, &v.0, v.1); + for PackedCacheEntry { key, value, index } in shard.iter().copied() { + f(&key, &value, index); } } } fn len(&self) -> usize { - self.cache.len() + self.cache.lock_shards().map(|shard| shard.len()).sum() } } From 7042b57bd71f3fa592cf694a1c9d9831bffb5aaa Mon Sep 17 00:00:00 2001 From: Daria Sukhonina Date: Mon, 20 Apr 2026 14:50:28 +0300 Subject: [PATCH 4/4] Fix warning --- compiler/rustc_middle/src/query/caches.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_middle/src/query/caches.rs b/compiler/rustc_middle/src/query/caches.rs index 2538ae82eca64..02970d0f1452a 100644 --- a/compiler/rustc_middle/src/query/caches.rs +++ b/compiler/rustc_middle/src/query/caches.rs @@ -2,7 +2,7 @@ use std::hash::{Hash, Hasher as _}; use std::sync::OnceLock; use rustc_data_structures::fx::FxHasher; -use rustc_data_structures::hash_table::{self, HashTable}; +use rustc_data_structures::hash_table::HashTable; use rustc_data_structures::sharded::Sharded; pub use rustc_data_structures::vec_cache::VecCache; use rustc_hir::def_id::LOCAL_CRATE; @@ -89,9 +89,10 @@ where let mut shard = self.cache.lock_shard_by_hash(hash); cfg_select! { debug_assertions => { + use rustc_data_structures::hash_table::Entry::*; match shard.entry(hash, |ent| { ent.key } == key, |ent| make_hash(&{ ent.key })) { - hash_table::Entry::Occupied(_) => panic!("trying to complete query twice"), - hash_table::Entry::Vacant(entry) => entry.insert(PackedCacheEntry { key, value, index }), + Occupied(_) => panic!("trying to complete query twice"), + Vacant(entry) => entry.insert(PackedCacheEntry { key, value, index }), }; } _ => {