Skip to content

make query caches use Sharded only for multiple threads #115162

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

make query caches use Sharded only for multiple threads #115162

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
16b9941
make sharded query caches only for multiple threads
SparrowLii Aug 24, 2023
cd6850c
use `RefCell` for query caches in single thread
SparrowLii Aug 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions compiler/rustc_interface/src/passes.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -731,6 +731,7 @@ pub fn create_global_ctxt<'tcx>(
extern_providers,
query_result_on_disk_cache,
incremental,
!rustc_data_structures::sync::is_dyn_thread_safe(),
),
)
})
Expand Down
106 changes: 95 additions & 11 deletions compiler/rustc_middle/src/query/plumbing.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ use crate::dep_graph::DepKind;
use crate::query::on_disk_cache::CacheEncoder;
use crate::query::on_disk_cache::EncodedDepNodeIndex;
use crate::query::on_disk_cache::OnDiskCache;
#[cfg(parallel_compiler)]
use crate::query::MtQueryCaches;
use crate::query::{
DynamicQueries, ExternProviders, Providers, QueryArenas, QueryCaches, QueryEngine, QueryStates,
};
Expand All @@ -11,6 +13,8 @@ use field_offset::FieldOffset;
use measureme::StringId;
use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::sync::AtomicU64;
#[cfg(parallel_compiler)]
use rustc_data_structures::sync::DynSync;
use rustc_hir::def::DefKind;
use rustc_hir::def_id::{DefId, LocalDefId};
use rustc_hir::hir_id::OwnerId;
Expand All @@ -20,6 +24,8 @@ pub(crate) use rustc_query_system::query::QueryJobId;
use rustc_query_system::query::*;
use rustc_query_system::HandleCycleError;
use rustc_span::{Span, DUMMY_SP};
#[cfg(not(parallel_compiler))]
use std::marker::PhantomData;
use std::ops::Deref;

pub struct QueryKeyStringCache {
Expand All @@ -32,13 +38,17 @@ impl QueryKeyStringCache {
}
}

pub struct DynamicQuery<'tcx, C: QueryCache> {
pub struct DynamicQuery<'tcx, C: QueryCache, C2: QueryCache<Key = C::Key, Value = C::Value>> {
pub name: &'static str,
pub eval_always: bool,
pub dep_kind: DepKind,
pub handle_cycle_error: HandleCycleError,
pub query_state: FieldOffset<QueryStates<'tcx>, QueryState<C::Key, DepKind>>,
pub query_cache: FieldOffset<QueryCaches<'tcx>, C>,
#[cfg(not(parallel_compiler))]
pub mt_query_cache: PhantomData<C2>,
#[cfg(parallel_compiler)]
pub mt_query_cache: FieldOffset<MtQueryCaches<'tcx>, C2>,
pub cache_on_disk: fn(tcx: TyCtxt<'tcx>, key: &C::Key) -> bool,
pub execute_query: fn(tcx: TyCtxt<'tcx>, k: C::Key) -> C::Value,
pub compute: fn(tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value,
Expand Down Expand Up @@ -72,6 +82,10 @@ pub struct QuerySystem<'tcx> {
pub states: QueryStates<'tcx>,
pub arenas: QueryArenas<'tcx>,
pub caches: QueryCaches<'tcx>,
#[cfg(parallel_compiler)]
pub single_thread: bool,
#[cfg(parallel_compiler)]
pub mt_caches: MtQueryCaches<'tcx>,
pub dynamic_queries: DynamicQueries<'tcx>,

/// This provides access to the incremental compilation on-disk cache for query results.
Expand All @@ -85,6 +99,10 @@ pub struct QuerySystem<'tcx> {
pub jobs: AtomicU64,
}

// It's thread safe since `QueryCaches` only used under single thread
#[cfg(parallel_compiler)]
unsafe impl<'tcx> DynSync for QuerySystem<'tcx> {}

#[derive(Copy, Clone)]
pub struct TyCtxtAt<'tcx> {
pub tcx: TyCtxt<'tcx>,
Expand Down Expand Up @@ -138,6 +156,36 @@ impl<'tcx> TyCtxt<'tcx> {
}
}

#[macro_export]
#[cfg(not(parallel_compiler))]
macro_rules! with_query_caches {
($func: ident($($params: expr,)* :$tcx: expr, $name:ident, $($rest: expr,)*)) => {
$func($($params,)* &$tcx.query_system.caches.$name, $($rest,)*)
};
($tcx: expr, $name:ident, $func: ident($($params: expr,)*)) => {
$tcx.query_system.caches.$name.$func($($params,)*)
}
}

#[macro_export]
#[cfg(parallel_compiler)]
macro_rules! with_query_caches {
($func: ident($($params: expr,)* :$tcx: expr, $name:ident, $($rest: expr,)*)) => {
if $tcx.query_system.single_thread {
$func($($params,)* &$tcx.query_system.caches.$name, $($rest,)*)
} else {
$func($($params,)* &$tcx.query_system.mt_caches.$name, $($rest,)*)
}
};
($tcx: expr, $name:ident, $func: ident($($params: expr,)*)) => {
if $tcx.query_system.single_thread {
$tcx.query_system.caches.$name.$func($($params,)*)
} else {
$tcx.query_system.mt_caches.$name.$func($($params,)*)
}
}
}

#[inline]
pub fn query_get_at<'tcx, Cache>(
tcx: TyCtxt<'tcx>,
Expand Down Expand Up @@ -286,6 +334,10 @@ macro_rules! define_callbacks {
<$($K)* as keys::Key>::CacheSelector as CacheSelector<'tcx, Erase<$V>>
>::Cache;

pub type MtStorage<'tcx> = <
<$($K)* as keys::Key>::CacheSelector as CacheSelector<'tcx, Erase<$V>>
>::MtCache;

// Ensure that keys grow no larger than 64 bytes
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
const _: () = {
Expand Down Expand Up @@ -339,31 +391,36 @@ macro_rules! define_callbacks {
$($(#[$attr])* pub $name: queries::$name::Storage<'tcx>,)*
}

#[derive(Default)]
pub struct MtQueryCaches<'tcx> {
$($(#[$attr])* pub $name: queries::$name::MtStorage<'tcx>,)*
}

impl<'tcx> TyCtxtEnsure<'tcx> {
$($(#[$attr])*
#[inline(always)]
pub fn $name(self, key: query_helper_param_ty!($($K)*)) {
query_ensure(
with_query_caches!(query_ensure(
self.tcx,
self.tcx.query_system.fns.engine.$name,
&self.tcx.query_system.caches.$name,
:self.tcx, $name,
key.into_query_param(),
false,
);
));
})*
}

impl<'tcx> TyCtxtEnsureWithValue<'tcx> {
$($(#[$attr])*
#[inline(always)]
pub fn $name(self, key: query_helper_param_ty!($($K)*)) {
query_ensure(
with_query_caches!(query_ensure(
self.tcx,
self.tcx.query_system.fns.engine.$name,
&self.tcx.query_system.caches.$name,
:self.tcx, $name,
key.into_query_param(),
true,
);
));
})*
}

Expand All @@ -377,8 +434,10 @@ macro_rules! define_callbacks {
})*
}

#[cfg(not(parallel_compiler))]
impl<'tcx> TyCtxtAt<'tcx> {
$($(#[$attr])*

#[inline(always)]
pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> $V
{
Expand All @@ -392,9 +451,35 @@ macro_rules! define_callbacks {
})*
}

#[cfg(parallel_compiler)]
impl<'tcx> TyCtxtAt<'tcx> {
$($(#[$attr])*
#[inline(always)]
pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> $V
{
if std::intrinsics::likely(self.tcx.query_system.single_thread) {
restore::<$V>(query_get_at(
self.tcx,
self.tcx.query_system.fns.engine.$name,
&self.tcx.query_system.caches.$name,
self.span,
key.into_query_param(),
))
} else {
restore::<$V>(query_get_at(
self.tcx,
self.tcx.query_system.fns.engine.$name,
&self.tcx.query_system.mt_caches.$name,
self.span,
key.into_query_param(),
))
}
})*
}

pub struct DynamicQueries<'tcx> {
$(
pub $name: DynamicQuery<'tcx, queries::$name::Storage<'tcx>>,
pub $name: DynamicQuery<'tcx, queries::$name::Storage<'tcx>, queries::$name::MtStorage<'tcx>>,
)*
}

Expand Down Expand Up @@ -484,10 +569,9 @@ macro_rules! define_feedable {
let tcx = self.tcx;
let erased = queries::$name::provided_to_erased(tcx, value);
let value = restore::<$V>(erased);
let cache = &tcx.query_system.caches.$name;

let hasher: Option<fn(&mut StableHashingContext<'_>, &_) -> _> = hash_result!([$($modifiers)*]);
match try_get_cached(tcx, cache, &key) {
match with_query_caches!(try_get_cached(tcx, :tcx, $name, &key,)) {
Some(old) => {
let old = restore::<$V>(old);
if let Some(hasher) = hasher {
Expand Down Expand Up @@ -523,7 +607,7 @@ macro_rules! define_feedable {
&value,
hash_result!([$($modifiers)*]),
);
cache.complete(key, erased, dep_node_index);
with_query_caches!(tcx, $name, complete(key, erased, dep_node_index,));
}
}
}
Expand Down
37 changes: 30 additions & 7 deletions compiler/rustc_query_impl/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ use rustc_middle::query::plumbing::{
DynamicQuery, QueryKeyStringCache, QuerySystem, QuerySystemFns,
};
use rustc_middle::query::AsLocalKey;
#[cfg(parallel_compiler)]
use rustc_middle::query::MtQueryCaches;
use rustc_middle::query::{
queries, DynamicQueries, ExternProviders, Providers, QueryCaches, QueryEngine, QueryStates,
};
Expand All @@ -53,33 +55,40 @@ pub use self::profiling_support::alloc_self_profile_query_strings;
struct DynamicConfig<
'tcx,
C: QueryCache,
C2: QueryCache<Key = C::Key, Value = C::Value>,
const ANON: bool,
const DEPTH_LIMIT: bool,
const FEEDABLE: bool,
> {
dynamic: &'tcx DynamicQuery<'tcx, C>,
dynamic: &'tcx DynamicQuery<'tcx, C, C2>,
}

impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Copy
for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE>
impl<'tcx, C: QueryCache, C2, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Copy
for DynamicConfig<'tcx, C, C2, ANON, DEPTH_LIMIT, FEEDABLE>
where
C2: QueryCache<Key = C::Key, Value = C::Value>,
{
}
impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Clone
for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE>
impl<'tcx, C: QueryCache, C2, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Clone
for DynamicConfig<'tcx, C, C2, ANON, DEPTH_LIMIT, FEEDABLE>
where
C2: QueryCache<Key = C::Key, Value = C::Value>,
{
fn clone(&self) -> Self {
DynamicConfig { dynamic: self.dynamic }
}
}

impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool>
QueryConfig<QueryCtxt<'tcx>> for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE>
impl<'tcx, C: QueryCache, C2, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool>
QueryConfig<QueryCtxt<'tcx>> for DynamicConfig<'tcx, C, C2, ANON, DEPTH_LIMIT, FEEDABLE>
where
C2: QueryCache<Key = C::Key, Value = C::Value>,
for<'a> C::Key: HashStable<StableHashingContext<'a>>,
{
type Key = C::Key;
type Value = C::Value;
type Cache = C;
type MtCache = C2;

#[inline(always)]
fn name(self) -> &'static str {
Expand Down Expand Up @@ -107,6 +116,15 @@ where
self.dynamic.query_cache.apply(&qcx.tcx.query_system.caches)
}

#[inline(always)]
#[cfg(parallel_compiler)]
fn mt_query_cache<'a>(self, qcx: QueryCtxt<'tcx>) -> &'a Self::MtCache
where
'tcx: 'a,
{
self.dynamic.mt_query_cache.apply(&qcx.tcx.query_system.mt_caches)
}

#[inline(always)]
fn execute_query(self, tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
(self.dynamic.execute_query)(tcx, key)
Expand Down Expand Up @@ -207,11 +225,16 @@ pub fn query_system<'tcx>(
extern_providers: ExternProviders,
on_disk_cache: Option<OnDiskCache<'tcx>>,
incremental: bool,
_single_thread: bool,
) -> QuerySystem<'tcx> {
QuerySystem {
states: Default::default(),
arenas: Default::default(),
caches: Default::default(),
#[cfg(parallel_compiler)]
single_thread: _single_thread,
#[cfg(parallel_compiler)]
mt_caches: Default::default(),
dynamic_queries: dynamic_queries(),
on_disk_cache,
fns: QuerySystemFns {
Expand Down
Loading