Use dynamic dispatch for query execution

Cargo.lock

@@ -1046,7 +1046,7 @@ dependencies = [
  "autocfg",
  "cfg-if",
  "crossbeam-utils",
- "memoffset",
+ "memoffset 0.7.1",
  "scopeguard",
 ]
 
@@ -2622,6 +2622,15 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "memoffset"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "minifier"
 version = "0.2.2"
@@ -4556,6 +4565,7 @@ name = "rustc_query_impl"
 version = "0.0.0"
 dependencies = [
  "measureme",
+ "memoffset 0.8.0",
  "rustc-rayon-core",
  "rustc_ast",
  "rustc_data_structures",

compiler/rustc_interface/src/passes.rs

@@ -17,7 +17,7 @@ use rustc_lint::{unerased_lint_store, BufferedEarlyLint, EarlyCheckNode, LintSto
 use rustc_metadata::creader::CStore;
 use rustc_middle::arena::Arena;
 use rustc_middle::dep_graph::DepGraph;
-use rustc_middle::ty::query::{ExternProviders, Providers};
+use rustc_middle::ty::query::{ExternProviders, Providers, QuerySystem};
 use rustc_middle::ty::{self, GlobalCtxt, RegisteredTools, TyCtxt};
 use rustc_mir_build as mir_build;
 use rustc_parse::{parse_crate_from_file, parse_crate_from_source_str, validate_attr};
@@ -687,9 +687,10 @@ pub fn create_global_ctxt<'tcx>(
         callback(sess, &mut local_providers, &mut extern_providers);
     }
 
-    let queries = queries.get_or_init(|| {
-        TcxQueries::new(local_providers, extern_providers, query_result_on_disk_cache)
-    });
+    let queries =
+        queries.get_or_init(|| TcxQueries::new(&local_providers, query_result_on_disk_cache));
+
+    let query_system = QuerySystem::new(local_providers, extern_providers);
 
     sess.time("setup_global_ctxt", || {
         gcx_cell.get_or_init(move || {
@@ -701,6 +702,7 @@ pub fn create_global_ctxt<'tcx>(
                 untracked,
                 dep_graph,
                 queries.on_disk_cache.as_ref().map(OnDiskCache::as_dyn),
+                query_system,
                 queries.as_dyn(),
                 rustc_query_impl::query_callbacks(arena),
             )

compiler/rustc_middle/src/ty/context.rs

@@ -78,6 +78,8 @@ use std::iter;
 use std::mem;
 use std::ops::{Bound, Deref};
 
+use super::query::QuerySystem;
+
 const TINY_CONST_EVAL_LIMIT: Limit = Limit(20);
 
 pub trait OnDiskCache<'tcx>: rustc_data_structures::sync::Sync {
@@ -681,6 +683,7 @@ impl<'tcx> TyCtxt<'tcx> {
         untracked: Untracked,
         dep_graph: DepGraph,
         on_disk_cache: Option<&'tcx dyn OnDiskCache<'tcx>>,
+        query_system: QuerySystem<'tcx>,
         queries: &'tcx dyn query::QueryEngine<'tcx>,
         query_kinds: &'tcx [DepKindStruct<'tcx>],
     ) -> GlobalCtxt<'tcx> {
@@ -706,7 +709,7 @@ impl<'tcx> TyCtxt<'tcx> {
             untracked,
             on_disk_cache,
             queries,
-            query_system: Default::default(),
+            query_system,
             query_kinds,
             ty_rcache: Default::default(),
             pred_rcache: Default::default(),

compiler/rustc_middle/src/ty/query.rs

@@ -71,12 +71,24 @@ use std::sync::Arc;
 pub(crate) use rustc_query_system::query::QueryJobId;
 use rustc_query_system::query::*;
 
-#[derive(Default)]
 pub struct QuerySystem<'tcx> {
+    pub local_providers: Box<Providers>,
+    pub extern_providers: Box<ExternProviders>,
     pub arenas: QueryArenas<'tcx>,
     pub caches: QueryCaches<'tcx>,
 }
 
+impl<'tcx> QuerySystem<'tcx> {
+    pub fn new(local_providers: Providers, extern_providers: ExternProviders) -> Self {
+        QuerySystem {
+            local_providers: Box::new(local_providers),
+            extern_providers: Box::new(extern_providers),
+            arenas: Default::default(),
+            caches: Default::default(),
+        }
+    }
+}
+
 #[derive(Copy, Clone)]
 pub struct TyCtxtAt<'tcx> {
     pub tcx: TyCtxt<'tcx>,

compiler/rustc_query_impl/Cargo.toml

@@ -7,6 +7,7 @@ edition = "2021"
 
 
 [dependencies]
+memoffset = { version = "0.8.0", features = ["unstable_const"] }
 measureme = "10.0.0"
 rustc_ast = { path = "../rustc_ast" }
 rustc_data_structures = { path = "../rustc_data_structures" }

compiler/rustc_query_impl/src/lib.rs

@@ -3,12 +3,13 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 // this shouldn't be necessary, but the check for `&mut _` is too naive and denies returning a function pointer that takes a mut ref
 #![feature(const_mut_refs)]
+#![feature(const_refs_to_cell)]
 #![feature(min_specialization)]
 #![feature(never_type)]
 #![feature(once_cell)]
 #![feature(rustc_attrs)]
 #![recursion_limit = "256"]
-#![allow(rustc::potential_query_instability)]
+#![allow(rustc::potential_query_instability, unused_parens)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 
@@ -17,37 +18,211 @@ extern crate rustc_macros;
 #[macro_use]
 extern crate rustc_middle;
 
-use rustc_data_structures::sync::AtomicU64;
+use memoffset::offset_of;
+use rustc_data_structures::fingerprint::Fingerprint;
+use rustc_data_structures::stable_hasher::HashStable;
 use rustc_middle::arena::Arena;
 use rustc_middle::dep_graph::{self, DepKindStruct};
 use rustc_middle::query::Key;
+use rustc_middle::ty::query::QueryCaches;
 use rustc_middle::ty::query::{
     query_keys, query_provided, query_provided_to_value, query_storage, query_values,
 };
-use rustc_middle::ty::query::{ExternProviders, Providers, QueryEngine};
+use rustc_middle::ty::query::{Providers, QueryEngine};
 use rustc_middle::ty::TyCtxt;
+use rustc_query_system::dep_graph::DepNodeParams;
+use rustc_query_system::ich::StableHashingContext;
 use rustc_span::Span;
+use std::fmt;
+use std::marker::PhantomData;
 
 #[macro_use]
 mod plumbing;
-pub use plumbing::QueryCtxt;
+pub use plumbing::{Queries, QueryCtxt};
 use rustc_query_system::query::*;
 #[cfg(parallel_compiler)]
 pub use rustc_query_system::query::{deadlock, QueryContext};
 
 pub use rustc_query_system::query::QueryConfig;
+use rustc_query_system::HandleCycleError;
 
 mod on_disk_cache;
 pub use on_disk_cache::OnDiskCache;
 
 mod profiling_support;
 pub use self::profiling_support::alloc_self_profile_query_strings;
 
-rustc_query_append! { define_queries! }
+trait Bool {
+    fn value() -> bool;
+}
+
+struct True;
+
+impl Bool for True {
+    fn value() -> bool {
+        true
+    }
+}
+struct False;
+
+impl Bool for False {
+    fn value() -> bool {
+        false
+    }
+}
+struct DynamicQuery<'tcx, C: QueryCache> {
+    name: &'static str,
+    cache_on_disk: fn(tcx: TyCtxt<'tcx>, key: &C::Key) -> bool,
+    execute_query: fn(tcx: TyCtxt<'tcx>, k: C::Key) -> C::Value,
+    compute: fn(tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value,
+    try_load_from_disk:
+        fn(qcx: QueryCtxt<'tcx>, idx: &C::Key) -> TryLoadFromDisk<QueryCtxt<'tcx>, C::Value>,
+    query_state: usize,
+    query_cache: usize,
+    eval_always: bool,
+    dep_kind: rustc_middle::dep_graph::DepKind,
+    handle_cycle_error: HandleCycleError,
+    hash_result: Option<fn(&mut StableHashingContext<'_>, &C::Value) -> Fingerprint>,
+}
+
+struct DynamicConfig<'tcx, C: QueryCache, Anon, DepthLimit, Feedable> {
+    dynamic: &'tcx DynamicQuery<'tcx, C>,
+    data: PhantomData<(Anon, DepthLimit, Feedable)>,
+}
+
+impl<'tcx, C: QueryCache, Anon, DepthLimit, Feedable> Copy
+    for DynamicConfig<'tcx, C, Anon, DepthLimit, Feedable>
+{
+}
+impl<'tcx, C: QueryCache, Anon, DepthLimit, Feedable> Clone
+    for DynamicConfig<'tcx, C, Anon, DepthLimit, Feedable>
+{
+    fn clone(&self) -> Self {
+        DynamicConfig { dynamic: self.dynamic, data: PhantomData }
+    }
+}
+
+impl<'tcx, C: QueryCache, Anon, DepthLimit, Feedable> fmt::Debug
+    for DynamicConfig<'tcx, C, Anon, DepthLimit, Feedable>
+{
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "DynamicConfig<{}>", self.dynamic.name)
+    }
+}
+
+impl<'tcx, C: QueryCache, Anon: Bool, DepthLimit: Bool, Feedable: Bool> QueryConfig<QueryCtxt<'tcx>>
+    for DynamicConfig<'tcx, C, Anon, DepthLimit, Feedable>
+where
+    for<'a> C::Key: HashStable<StableHashingContext<'a>>,
+{
+    type Key = C::Key;
+    type Value = C::Value;
+    type Cache = C;
+
+    #[inline(always)]
+    fn name(self) -> &'static str {
+        self.dynamic.name
+    }
+
+    #[inline(always)]
+    fn cache_on_disk(self, tcx: TyCtxt<'tcx>, key: &Self::Key) -> bool {
+        (self.dynamic.cache_on_disk)(tcx, key)
+    }
+
+    #[inline(always)]
+    fn query_state<'a>(
+        self,
+        tcx: QueryCtxt<'tcx>,
+    ) -> &'a QueryState<Self::Key, crate::dep_graph::DepKind>
+    where
+        QueryCtxt<'tcx>: 'a,
+    {
+        unsafe {
+            &*((&tcx.queries.query_states as *const QueryStates<'tcx> as *const u8)
+                .offset(self.dynamic.query_state as isize) as *const _)
+        }
+    }
 
-impl<'tcx> Queries<'tcx> {
-    // Force codegen in the dyn-trait transformation in this crate.
-    pub fn as_dyn(&'tcx self) -> &'tcx dyn QueryEngine<'tcx> {
-        self
+    #[inline(always)]
+    fn query_cache<'a>(self, tcx: QueryCtxt<'tcx>) -> &'a Self::Cache
+    where
+        'tcx: 'a,
+    {
+        unsafe {
+            &*((&tcx.query_system.caches as *const QueryCaches<'tcx> as *const u8)
+                .offset(self.dynamic.query_cache as isize) as *const _)
+        }
+    }
+
+    #[inline(always)]
+    fn execute_query(self, tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
+        (self.dynamic.execute_query)(tcx, key)
+    }
+
+    #[inline(always)]
+    fn compute(self, tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
+        (self.dynamic.compute)(tcx, key)
+    }
+
+    #[inline(always)]
+    fn try_load_from_disk(
+        self,
+        qcx: QueryCtxt<'tcx>,
+        key: &Self::Key,
+    ) -> rustc_query_system::query::TryLoadFromDisk<QueryCtxt<'tcx>, Self::Value> {
+        (self.dynamic.try_load_from_disk)(qcx, key)
+    }
+
+    #[inline(always)]
+    fn anon(self) -> bool {
+        Anon::value()
+    }
+
+    #[inline(always)]
+    fn eval_always(self) -> bool {
+        self.dynamic.eval_always
+    }
+
+    #[inline(always)]
+    fn depth_limit(self) -> bool {
+        DepthLimit::value()
+    }
+
+    #[inline(always)]
+    fn feedable(self) -> bool {
+        Feedable::value()
+    }
+
+    #[inline(always)]
+    fn dep_kind(self) -> rustc_middle::dep_graph::DepKind {
+        self.dynamic.dep_kind
+    }
+
+    #[inline(always)]
+    fn handle_cycle_error(self) -> rustc_query_system::HandleCycleError {
+        self.dynamic.handle_cycle_error
+    }
+
+    #[inline(always)]
+    fn hash_result(self) -> rustc_query_system::query::HashResult<Self::Value> {
+        self.dynamic.hash_result
     }
 }
+
+trait QueryToConfig<'tcx>: 'tcx
+where
+    for<'a> <Self::Cache as QueryCache>::Key: HashStable<StableHashingContext<'a>>,
+{
+    type Cache: QueryCache;
+    type Key: DepNodeParams<TyCtxt<'tcx>>;
+
+    type Anon: Bool;
+    type DepthLimit: Bool;
+    type Feedable: Bool;
+
+    fn config(
+        qcx: QueryCtxt<'tcx>,
+    ) -> DynamicConfig<'tcx, Self::Cache, Self::Anon, Self::DepthLimit, Self::Feedable>;
+}
+
+rustc_query_append! { define_queries! }