From 8684e9e47dcc52cc51dccdf1a74bac69deb38207 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Wed, 18 Nov 2020 16:53:39 +0100
Subject: [PATCH 01/27] Merge {get,ensure}_query.

---
 .../rustc_middle/src/ty/query/plumbing.rs     |  4 +-
 .../rustc_query_system/src/query/plumbing.rs  | 46 +++++++++----------
 2 files changed, 24 insertions(+), 26 deletions(-)

diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs
index 46addcdaead43..f6370452e80b2 100644
--- a/compiler/rustc_middle/src/ty/query/plumbing.rs
+++ b/compiler/rustc_middle/src/ty/query/plumbing.rs
@@ -401,7 +401,7 @@ macro_rules! define_queries {
             $($(#[$attr])*
             #[inline(always)]
             pub fn $name(self, key: query_helper_param_ty!($($K)*)) {
-                ensure_query::<queries::$name<'_>, _>(self.tcx, key.into_query_param())
+                get_query::<queries::$name<'_>, _>(self.tcx, DUMMY_SP, key.into_query_param(), QueryMode::Ensure);
             })*
         }
 
@@ -484,7 +484,7 @@ macro_rules! define_queries {
             pub fn $name(self, key: query_helper_param_ty!($($K)*))
                 -> <queries::$name<$tcx> as QueryConfig>::Stored
             {
-                get_query::<queries::$name<'_>, _>(self.tcx, self.span, key.into_query_param())
+                get_query::<queries::$name<'_>, _>(self.tcx, self.span, key.into_query_param(), QueryMode::Get).unwrap()
             })*
         }
 
diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs
index cbbb449b4f8ab..f2ebf8d7d3d08 100644
--- a/compiler/rustc_query_system/src/query/plumbing.rs
+++ b/compiler/rustc_query_system/src/query/plumbing.rs
@@ -17,7 +17,6 @@ use rustc_data_structures::sharded::Sharded;
 use rustc_data_structures::sync::{Lock, LockGuard};
 use rustc_data_structures::thin_vec::ThinVec;
 use rustc_errors::{Diagnostic, FatalError};
-use rustc_span::source_map::DUMMY_SP;
 use rustc_span::Span;
 use std::collections::hash_map::Entry;
 use std::fmt::Debug;
@@ -641,31 +640,26 @@ where
 
 /// Ensure that either this query has all green inputs or been executed.
 /// Executing `query::ensure(D)` is considered a read of the dep-node `D`.
+/// Returns true if the query should still run.
 ///
 /// This function is particularly useful when executing passes for their
 /// side-effects -- e.g., in order to report errors for erroneous programs.
 ///
 /// Note: The optimization is only available during incr. comp.
 #[inline(never)]
-fn ensure_query_impl<CTX, C>(
-    tcx: CTX,
-    state: &QueryState<CTX::DepKind, CTX::Query, C>,
-    key: C::Key,
-    query: &QueryVtable<CTX, C::Key, C::Value>,
-) where
-    C: QueryCache,
-    C::Key: crate::dep_graph::DepNodeParams<CTX>,
+fn ensure_must_run<CTX, K, V>(tcx: CTX, key: &K, query: &QueryVtable<CTX, K, V>) -> bool
+where
+    K: crate::dep_graph::DepNodeParams<CTX>,
     CTX: QueryContext,
 {
     if query.eval_always {
-        let _ = get_query_impl(tcx, state, DUMMY_SP, key, query);
-        return;
+        return true;
     }
 
     // Ensuring an anonymous query makes no sense
     assert!(!query.anon);
 
-    let dep_node = query.to_dep_node(tcx, &key);
+    let dep_node = query.to_dep_node(tcx, key);
 
     match tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node) {
         None => {
@@ -675,10 +669,11 @@ fn ensure_query_impl<CTX, C>(
             // DepNodeIndex. We must invoke the query itself. The performance cost
             // this introduces should be negligible as we'll immediately hit the
             // in-memory cache, or another query down the line will.
-            let _ = get_query_impl(tcx, state, DUMMY_SP, key, query);
+            true
         }
         Some((_, dep_node_index)) => {
             tcx.profiler().query_cache_hit(dep_node_index.into());
+            false
         }
     }
 }
@@ -720,24 +715,27 @@ fn force_query_impl<CTX, C>(
     );
 }
 
-pub fn get_query<Q, CTX>(tcx: CTX, span: Span, key: Q::Key) -> Q::Stored
-where
-    Q: QueryDescription<CTX>,
-    Q::Key: crate::dep_graph::DepNodeParams<CTX>,
-    CTX: QueryContext,
-{
-    debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span);
-
-    get_query_impl(tcx, Q::query_state(tcx), span, key, &Q::VTABLE)
+pub enum QueryMode {
+    Get,
+    Ensure,
 }
 
-pub fn ensure_query<Q, CTX>(tcx: CTX, key: Q::Key)
+pub fn get_query<Q, CTX>(tcx: CTX, span: Span, key: Q::Key, mode: QueryMode) -> Option<Q::Stored>
 where
     Q: QueryDescription<CTX>,
     Q::Key: crate::dep_graph::DepNodeParams<CTX>,
     CTX: QueryContext,
 {
-    ensure_query_impl(tcx, Q::query_state(tcx), key, &Q::VTABLE)
+    let query = &Q::VTABLE;
+    if let QueryMode::Ensure = mode {
+        if !ensure_must_run(tcx, &key, query) {
+            return None;
+        }
+    }
+
+    debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span);
+    let value = get_query_impl(tcx, Q::query_state(tcx), span, key, query);
+    Some(value)
 }
 
 pub fn force_query<Q, CTX>(tcx: CTX, key: Q::Key, span: Span, dep_node: DepNode<CTX::DepKind>)

From 4b42a6d90b850eb697a56bddb9e3239d7e5c72fb Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sun, 17 Jan 2021 14:57:07 +0100
Subject: [PATCH 02/27] Introduce query_stored module.

---
 .../rustc_middle/src/ty/query/plumbing.rs     | 20 +++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs
index f6370452e80b2..7a46bad0c1fd7 100644
--- a/compiler/rustc_middle/src/ty/query/plumbing.rs
+++ b/compiler/rustc_middle/src/ty/query/plumbing.rs
@@ -342,14 +342,20 @@ macro_rules! define_queries {
 
             $(pub type $name<$tcx> = $V;)*
         }
+        #[allow(nonstandard_style, unused_lifetimes)]
+        pub mod query_stored {
+            use super::*;
+
+            $(pub type $name<$tcx> = <
+                query_storage!([$($modifiers)*][$($K)*, $V])
+                as QueryStorage
+            >::Stored;)*
+        }
 
         $(impl<$tcx> QueryConfig for queries::$name<$tcx> {
             type Key = $($K)*;
             type Value = $V;
-            type Stored = <
-                query_storage!([$($modifiers)*][$($K)*, $V])
-                as QueryStorage
-            >::Stored;
+            type Stored = query_stored::$name<$tcx>;
             const NAME: &'static str = stringify!($name);
         }
 
@@ -442,8 +448,7 @@ macro_rules! define_queries {
             $($(#[$attr])*
             #[inline(always)]
             #[must_use]
-            pub fn $name(self, key: query_helper_param_ty!($($K)*))
-                -> <queries::$name<$tcx> as QueryConfig>::Stored
+            pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> query_stored::$name<$tcx>
             {
                 self.at(DUMMY_SP).$name(key.into_query_param())
             })*
@@ -481,8 +486,7 @@ macro_rules! define_queries {
         impl TyCtxtAt<$tcx> {
             $($(#[$attr])*
             #[inline(always)]
-            pub fn $name(self, key: query_helper_param_ty!($($K)*))
-                -> <queries::$name<$tcx> as QueryConfig>::Stored
+            pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> query_stored::$name<$tcx>
             {
                 get_query::<queries::$name<'_>, _>(self.tcx, self.span, key.into_query_param(), QueryMode::Get).unwrap()
             })*

From f8ab649dfd8866e35e3281e04534fe024e4095f7 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Tue, 19 Jan 2021 20:02:05 +0100
Subject: [PATCH 03/27] Introduce query_storage.

---
 compiler/rustc_middle/src/ty/query/plumbing.rs | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs
index 7a46bad0c1fd7..dcfc116585b9e 100644
--- a/compiler/rustc_middle/src/ty/query/plumbing.rs
+++ b/compiler/rustc_middle/src/ty/query/plumbing.rs
@@ -343,13 +343,16 @@ macro_rules! define_queries {
             $(pub type $name<$tcx> = $V;)*
         }
         #[allow(nonstandard_style, unused_lifetimes)]
+        pub mod query_storage {
+            use super::*;
+
+            $(pub type $name<$tcx> = query_storage!([$($modifiers)*][$($K)*, $V]);)*
+        }
+        #[allow(nonstandard_style, unused_lifetimes)]
         pub mod query_stored {
             use super::*;
 
-            $(pub type $name<$tcx> = <
-                query_storage!([$($modifiers)*][$($K)*, $V])
-                as QueryStorage
-            >::Stored;)*
+            $(pub type $name<$tcx> = <query_storage::$name<$tcx> as QueryStorage>::Stored;)*
         }
 
         $(impl<$tcx> QueryConfig for queries::$name<$tcx> {
@@ -364,7 +367,7 @@ macro_rules! define_queries {
             const EVAL_ALWAYS: bool = is_eval_always!([$($modifiers)*]);
             const DEP_KIND: dep_graph::DepKind = dep_graph::DepKind::$name;
 
-            type Cache = query_storage!([$($modifiers)*][$($K)*, $V]);
+            type Cache = query_storage::$name<$tcx>;
 
             #[inline(always)]
             fn query_state<'a>(tcx: TyCtxt<$tcx>) -> &'a QueryState<crate::dep_graph::DepKind, <TyCtxt<$tcx> as QueryContext>::Query, Self::Cache> {
@@ -523,7 +526,7 @@ macro_rules! define_queries_struct {
             $($(#[$attr])*  $name: QueryState<
                 crate::dep_graph::DepKind,
                 <TyCtxt<$tcx> as QueryContext>::Query,
-                <queries::$name<$tcx> as QueryAccessors<TyCtxt<'tcx>>>::Cache,
+                query_storage::$name<$tcx>,
             >,)*
         }
 

From 9f46259a7516f0bc453f9a0edb318be11c3d4a28 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Fri, 23 Oct 2020 22:34:32 +0200
Subject: [PATCH 04/27] Return a Result for query cache.

---
 .../rustc_query_system/src/query/caches.rs    | 56 +++++------
 .../rustc_query_system/src/query/plumbing.rs  | 99 ++++++++-----------
 2 files changed, 68 insertions(+), 87 deletions(-)

diff --git a/compiler/rustc_query_system/src/query/caches.rs b/compiler/rustc_query_system/src/query/caches.rs
index 1d2bc1a99a596..1ec32939d9f8d 100644
--- a/compiler/rustc_query_system/src/query/caches.rs
+++ b/compiler/rustc_query_system/src/query/caches.rs
@@ -31,17 +31,15 @@ pub trait QueryCache: QueryStorage {
     /// It returns the shard index and a lock guard to the shard,
     /// which will be used if the query is not in the cache and we need
     /// to compute it.
-    fn lookup<D, Q, R, OnHit, OnMiss>(
+    fn lookup<'s, D, Q, R, OnHit>(
         &self,
-        state: &QueryState<D, Q, Self>,
-        key: Self::Key,
+        state: &'s QueryState<D, Q, Self>,
+        key: &Self::Key,
         // `on_hit` can be called while holding a lock to the query state shard.
         on_hit: OnHit,
-        on_miss: OnMiss,
-    ) -> R
+    ) -> Result<R, QueryLookup<'s, D, Q, Self::Key, Self::Sharded>>
     where
-        OnHit: FnOnce(&Self::Stored, DepNodeIndex) -> R,
-        OnMiss: FnOnce(Self::Key, QueryLookup<'_, D, Q, Self::Key, Self::Sharded>) -> R;
+        OnHit: FnOnce(&Self::Stored, DepNodeIndex) -> R;
 
     fn complete(
         &self,
@@ -95,23 +93,24 @@ where
     type Sharded = FxHashMap<K, (V, DepNodeIndex)>;
 
     #[inline(always)]
-    fn lookup<D, Q, R, OnHit, OnMiss>(
+    fn lookup<'s, D, Q, R, OnHit>(
         &self,
-        state: &QueryState<D, Q, Self>,
-        key: K,
+        state: &'s QueryState<D, Q, Self>,
+        key: &K,
         on_hit: OnHit,
-        on_miss: OnMiss,
-    ) -> R
+    ) -> Result<R, QueryLookup<'s, D, Q, K, Self::Sharded>>
     where
         OnHit: FnOnce(&V, DepNodeIndex) -> R,
-        OnMiss: FnOnce(K, QueryLookup<'_, D, Q, K, Self::Sharded>) -> R,
     {
-        let mut lookup = state.get_lookup(&key);
-        let lock = &mut *lookup.lock;
+        let lookup = state.get_lookup(key);
+        let result = lookup.lock.cache.raw_entry().from_key_hashed_nocheck(lookup.key_hash, key);
 
-        let result = lock.cache.raw_entry().from_key_hashed_nocheck(lookup.key_hash, &key);
-
-        if let Some((_, value)) = result { on_hit(&value.0, value.1) } else { on_miss(key, lookup) }
+        if let Some((_, value)) = result {
+            let hit_result = on_hit(&value.0, value.1);
+            Ok(hit_result)
+        } else {
+            Err(lookup)
+        }
     }
 
     #[inline]
@@ -177,26 +176,23 @@ where
     type Sharded = FxHashMap<K, &'tcx (V, DepNodeIndex)>;
 
     #[inline(always)]
-    fn lookup<D, Q, R, OnHit, OnMiss>(
+    fn lookup<'s, D, Q, R, OnHit>(
         &self,
-        state: &QueryState<D, Q, Self>,
-        key: K,
+        state: &'s QueryState<D, Q, Self>,
+        key: &K,
         on_hit: OnHit,
-        on_miss: OnMiss,
-    ) -> R
+    ) -> Result<R, QueryLookup<'s, D, Q, K, Self::Sharded>>
     where
         OnHit: FnOnce(&&'tcx V, DepNodeIndex) -> R,
-        OnMiss: FnOnce(K, QueryLookup<'_, D, Q, K, Self::Sharded>) -> R,
     {
-        let mut lookup = state.get_lookup(&key);
-        let lock = &mut *lookup.lock;
-
-        let result = lock.cache.raw_entry().from_key_hashed_nocheck(lookup.key_hash, &key);
+        let lookup = state.get_lookup(key);
+        let result = lookup.lock.cache.raw_entry().from_key_hashed_nocheck(lookup.key_hash, key);
 
         if let Some((_, value)) = result {
-            on_hit(&&value.0, value.1)
+            let hit_result = on_hit(&&value.0, value.1);
+            Ok(hit_result)
         } else {
-            on_miss(key, lookup)
+            Err(lookup)
         }
     }
 
diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs
index f2ebf8d7d3d08..4f93017200f59 100644
--- a/compiler/rustc_query_system/src/query/plumbing.rs
+++ b/compiler/rustc_query_system/src/query/plumbing.rs
@@ -248,13 +248,8 @@ where
                 return TryGetJob::Cycle(value);
             }
 
-            let cached = try_get_cached(
-                tcx,
-                state,
-                (*key).clone(),
-                |value, index| (value.clone(), index),
-                |_, _| panic!("value must be in cache after waiting"),
-            );
+            let cached = try_get_cached(tcx, state, key, |value, index| (value.clone(), index))
+                .unwrap_or_else(|_| panic!("value must be in cache after waiting"));
 
             if let Some(prof_timer) = _query_blocked_prof_timer.take() {
                 prof_timer.finish_with_query_invocation_id(cached.1.into());
@@ -356,35 +351,28 @@ where
 /// It returns the shard index and a lock guard to the shard,
 /// which will be used if the query is not in the cache and we need
 /// to compute it.
-fn try_get_cached<CTX, C, R, OnHit, OnMiss>(
+fn try_get_cached<'a, CTX, C, R, OnHit>(
     tcx: CTX,
-    state: &QueryState<CTX::DepKind, CTX::Query, C>,
-    key: C::Key,
+    state: &'a QueryState<CTX::DepKind, CTX::Query, C>,
+    key: &C::Key,
     // `on_hit` can be called while holding a lock to the query cache
     on_hit: OnHit,
-    on_miss: OnMiss,
-) -> R
+) -> Result<R, QueryLookup<'a, CTX::DepKind, CTX::Query, C::Key, C::Sharded>>
 where
     C: QueryCache,
     CTX: QueryContext,
     OnHit: FnOnce(&C::Stored, DepNodeIndex) -> R,
-    OnMiss: FnOnce(C::Key, QueryLookup<'_, CTX::DepKind, CTX::Query, C::Key, C::Sharded>) -> R,
 {
-    state.cache.lookup(
-        state,
-        key,
-        |value, index| {
-            if unlikely!(tcx.profiler().enabled()) {
-                tcx.profiler().query_cache_hit(index.into());
-            }
-            #[cfg(debug_assertions)]
-            {
-                state.cache_hits.fetch_add(1, Ordering::Relaxed);
-            }
-            on_hit(value, index)
-        },
-        on_miss,
-    )
+    state.cache.lookup(state, &key, |value, index| {
+        if unlikely!(tcx.profiler().enabled()) {
+            tcx.profiler().query_cache_hit(index.into());
+        }
+        #[cfg(debug_assertions)]
+        {
+            state.cache_hits.fetch_add(1, Ordering::Relaxed);
+        }
+        on_hit(value, index)
+    })
 }
 
 fn try_execute_query<CTX, C>(
@@ -626,16 +614,14 @@ where
     C: QueryCache,
     C::Key: crate::dep_graph::DepNodeParams<CTX>,
 {
-    try_get_cached(
-        tcx,
-        state,
-        key,
-        |value, index| {
-            tcx.dep_graph().read_index(index);
-            value.clone()
-        },
-        |key, lookup| try_execute_query(tcx, state, span, key, lookup, query),
-    )
+    let cached = try_get_cached(tcx, state, &key, |value, index| {
+        tcx.dep_graph().read_index(index);
+        value.clone()
+    });
+    match cached {
+        Ok(value) => value,
+        Err(lookup) => try_execute_query(tcx, state, span, key, lookup, query),
+    }
 }
 
 /// Ensure that either this query has all green inputs or been executed.
@@ -694,25 +680,24 @@ fn force_query_impl<CTX, C>(
     // We may be concurrently trying both execute and force a query.
     // Ensure that only one of them runs the query.
 
-    try_get_cached(
-        tcx,
-        state,
-        key,
-        |_, _| {
-            // Cache hit, do nothing
-        },
-        |key, lookup| {
-            let job = match JobOwner::<'_, CTX::DepKind, CTX::Query, C>::try_start(
-                tcx, state, span, &key, lookup, query,
-            ) {
-                TryGetJob::NotYetStarted(job) => job,
-                TryGetJob::Cycle(_) => return,
-                #[cfg(parallel_compiler)]
-                TryGetJob::JobCompleted(_) => return,
-            };
-            force_query_with_job(tcx, key, job, dep_node, query);
-        },
-    );
+    let cached = try_get_cached(tcx, state, &key, |_, _| {
+        // Cache hit, do nothing
+    });
+
+    let lookup = match cached {
+        Ok(()) => return,
+        Err(lookup) => lookup,
+    };
+
+    let job = match JobOwner::<'_, CTX::DepKind, CTX::Query, C>::try_start(
+        tcx, state, span, &key, lookup, query,
+    ) {
+        TryGetJob::NotYetStarted(job) => job,
+        TryGetJob::Cycle(_) => return,
+        #[cfg(parallel_compiler)]
+        TryGetJob::JobCompleted(_) => return,
+    };
+    force_query_with_job(tcx, key, job, dep_node, query);
 }
 
 pub enum QueryMode {

From 15b0bc6b8380942fb45f1839b9fd91e66fad8045 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 6 Feb 2021 13:49:08 +0100
Subject: [PATCH 05/27] Separate the query cache from the query state.

---
 compiler/rustc_data_structures/src/sharded.rs |  30 +--
 compiler/rustc_middle/src/ty/context.rs       |   2 +
 .../src/ty/query/on_disk_cache.rs             |   7 +-
 .../rustc_middle/src/ty/query/plumbing.rs     |  20 +-
 .../src/ty/query/profiling_support.rs         |   8 +-
 compiler/rustc_middle/src/ty/query/stats.rs   |  11 +-
 .../rustc_query_system/src/query/caches.rs    |  24 +--
 .../rustc_query_system/src/query/config.rs    |   9 +-
 .../rustc_query_system/src/query/plumbing.rs  | 197 ++++++++++--------
 9 files changed, 173 insertions(+), 135 deletions(-)

diff --git a/compiler/rustc_data_structures/src/sharded.rs b/compiler/rustc_data_structures/src/sharded.rs
index 485719c517564..14db71cb8f070 100644
--- a/compiler/rustc_data_structures/src/sharded.rs
+++ b/compiler/rustc_data_structures/src/sharded.rs
@@ -63,23 +63,9 @@ impl<T> Sharded<T> {
         if SHARDS == 1 { &self.shards[0].0 } else { self.get_shard_by_hash(make_hash(val)) }
     }
 
-    /// Get a shard with a pre-computed hash value. If `get_shard_by_value` is
-    /// ever used in combination with `get_shard_by_hash` on a single `Sharded`
-    /// instance, then `hash` must be computed with `FxHasher`. Otherwise,
-    /// `hash` can be computed with any hasher, so long as that hasher is used
-    /// consistently for each `Sharded` instance.
-    #[inline]
-    pub fn get_shard_index_by_hash(&self, hash: u64) -> usize {
-        let hash_len = mem::size_of::<usize>();
-        // Ignore the top 7 bits as hashbrown uses these and get the next SHARD_BITS highest bits.
-        // hashbrown also uses the lowest bits, so we can't use those
-        let bits = (hash >> (hash_len * 8 - 7 - SHARD_BITS)) as usize;
-        bits % SHARDS
-    }
-
     #[inline]
     pub fn get_shard_by_hash(&self, hash: u64) -> &Lock<T> {
-        &self.shards[self.get_shard_index_by_hash(hash)].0
+        &self.shards[get_shard_index_by_hash(hash)].0
     }
 
     #[inline]
@@ -166,3 +152,17 @@ fn make_hash<K: Hash + ?Sized>(val: &K) -> u64 {
     val.hash(&mut state);
     state.finish()
 }
+
+/// Get a shard with a pre-computed hash value. If `get_shard_by_value` is
+/// ever used in combination with `get_shard_by_hash` on a single `Sharded`
+/// instance, then `hash` must be computed with `FxHasher`. Otherwise,
+/// `hash` can be computed with any hasher, so long as that hasher is used
+/// consistently for each `Sharded` instance.
+#[inline]
+pub fn get_shard_index_by_hash(hash: u64) -> usize {
+    let hash_len = mem::size_of::<usize>();
+    // Ignore the top 7 bits as hashbrown uses these and get the next SHARD_BITS highest bits.
+    // hashbrown also uses the lowest bits, so we can't use those
+    let bits = (hash >> (hash_len * 8 - 7 - SHARD_BITS)) as usize;
+    bits % SHARDS
+}
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index f83056ebe2a45..4654a8424706d 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -963,6 +963,7 @@ pub struct GlobalCtxt<'tcx> {
     pub(crate) definitions: &'tcx Definitions,
 
     pub queries: query::Queries<'tcx>,
+    pub query_caches: query::QueryCaches<'tcx>,
 
     maybe_unused_trait_imports: FxHashSet<LocalDefId>,
     maybe_unused_extern_crates: Vec<(LocalDefId, Span)>,
@@ -1154,6 +1155,7 @@ impl<'tcx> TyCtxt<'tcx> {
             untracked_crate: krate,
             definitions,
             queries: query::Queries::new(providers, extern_providers, on_disk_query_result_cache),
+            query_caches: query::QueryCaches::default(),
             ty_rcache: Default::default(),
             pred_rcache: Default::default(),
             selection_cache: Default::default(),
diff --git a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs
index cfe47004e01b6..b41edb5deeb2c 100644
--- a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs
+++ b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs
@@ -1244,10 +1244,9 @@ where
         .prof
         .extra_verbose_generic_activity("encode_query_results_for", std::any::type_name::<Q>());
 
-    let state = Q::query_state(tcx);
-    assert!(state.all_inactive());
-
-    state.iter_results(|results| {
+    assert!(Q::query_state(tcx).all_inactive());
+    let cache = Q::query_cache(tcx);
+    cache.iter_results(|results| {
         for (key, value, dep_node) in results {
             if Q::cache_on_disk(tcx, &key, Some(value)) {
                 let dep_node = SerializedDepNodeIndex::new(dep_node.index());
diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs
index dcfc116585b9e..9a011846fd62d 100644
--- a/compiler/rustc_middle/src/ty/query/plumbing.rs
+++ b/compiler/rustc_middle/src/ty/query/plumbing.rs
@@ -355,6 +355,11 @@ macro_rules! define_queries {
             $(pub type $name<$tcx> = <query_storage::$name<$tcx> as QueryStorage>::Stored;)*
         }
 
+        #[derive(Default)]
+        pub struct QueryCaches<$tcx> {
+            $($(#[$attr])* $name: QueryCacheStore<query_storage::$name<$tcx>>,)*
+        }
+
         $(impl<$tcx> QueryConfig for queries::$name<$tcx> {
             type Key = $($K)*;
             type Value = $V;
@@ -370,10 +375,17 @@ macro_rules! define_queries {
             type Cache = query_storage::$name<$tcx>;
 
             #[inline(always)]
-            fn query_state<'a>(tcx: TyCtxt<$tcx>) -> &'a QueryState<crate::dep_graph::DepKind, <TyCtxt<$tcx> as QueryContext>::Query, Self::Cache> {
+            fn query_state<'a>(tcx: TyCtxt<$tcx>) -> &'a QueryState<crate::dep_graph::DepKind, Query<$tcx>, Self::Key> {
                 &tcx.queries.$name
             }
 
+            #[inline(always)]
+            fn query_cache<'a>(tcx: TyCtxt<$tcx>) -> &'a QueryCacheStore<Self::Cache>
+                where 'tcx:'a
+            {
+                &tcx.query_caches.$name
+            }
+
             #[inline]
             fn compute(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
                 let provider = tcx.queries.providers.get(key.query_crate())
@@ -479,7 +491,7 @@ macro_rules! define_queries {
                     alloc_self_profile_query_strings_for_query_cache(
                         self,
                         stringify!($name),
-                        &self.queries.$name,
+                        &self.query_caches.$name,
                         &mut string_cache,
                     );
                 })*
@@ -525,8 +537,8 @@ macro_rules! define_queries_struct {
 
             $($(#[$attr])*  $name: QueryState<
                 crate::dep_graph::DepKind,
-                <TyCtxt<$tcx> as QueryContext>::Query,
-                query_storage::$name<$tcx>,
+                Query<$tcx>,
+                query_keys::$name<$tcx>,
             >,)*
         }
 
diff --git a/compiler/rustc_middle/src/ty/query/profiling_support.rs b/compiler/rustc_middle/src/ty/query/profiling_support.rs
index cbcecb8849188..9976e7885090c 100644
--- a/compiler/rustc_middle/src/ty/query/profiling_support.rs
+++ b/compiler/rustc_middle/src/ty/query/profiling_support.rs
@@ -5,7 +5,7 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::profiling::SelfProfiler;
 use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, CRATE_DEF_INDEX, LOCAL_CRATE};
 use rustc_hir::definitions::DefPathData;
-use rustc_query_system::query::{QueryCache, QueryContext, QueryState};
+use rustc_query_system::query::{QueryCache, QueryCacheStore};
 use std::fmt::Debug;
 use std::io::Write;
 
@@ -230,7 +230,7 @@ where
 pub(super) fn alloc_self_profile_query_strings_for_query_cache<'tcx, C>(
     tcx: TyCtxt<'tcx>,
     query_name: &'static str,
-    query_state: &QueryState<crate::dep_graph::DepKind, <TyCtxt<'tcx> as QueryContext>::Query, C>,
+    query_cache: &QueryCacheStore<C>,
     string_cache: &mut QueryKeyStringCache,
 ) where
     C: QueryCache,
@@ -251,7 +251,7 @@ pub(super) fn alloc_self_profile_query_strings_for_query_cache<'tcx, C>(
             // need to invoke queries itself, we cannot keep the query caches
             // locked while doing so. Instead we copy out the
             // `(query_key, dep_node_index)` pairs and release the lock again.
-            let query_keys_and_indices: Vec<_> = query_state
+            let query_keys_and_indices: Vec<_> = query_cache
                 .iter_results(|results| results.map(|(k, _, i)| (k.clone(), i)).collect());
 
             // Now actually allocate the strings. If allocating the strings
@@ -276,7 +276,7 @@ pub(super) fn alloc_self_profile_query_strings_for_query_cache<'tcx, C>(
             let query_name = profiler.get_or_alloc_cached_string(query_name);
             let event_id = event_id_builder.from_label(query_name).to_string_id();
 
-            query_state.iter_results(|results| {
+            query_cache.iter_results(|results| {
                 let query_invocation_ids: Vec<_> = results.map(|v| v.2.into()).collect();
 
                 profiler.bulk_map_query_invocation_id_to_single_string(
diff --git a/compiler/rustc_middle/src/ty/query/stats.rs b/compiler/rustc_middle/src/ty/query/stats.rs
index e0b44ce23c912..c885a10f80595 100644
--- a/compiler/rustc_middle/src/ty/query/stats.rs
+++ b/compiler/rustc_middle/src/ty/query/stats.rs
@@ -1,10 +1,9 @@
 use crate::ty::query::queries;
 use crate::ty::TyCtxt;
 use rustc_hir::def_id::{DefId, LOCAL_CRATE};
-use rustc_query_system::query::{QueryAccessors, QueryCache, QueryContext, QueryState};
+use rustc_query_system::query::{QueryAccessors, QueryCache, QueryCacheStore};
 
 use std::any::type_name;
-use std::hash::Hash;
 use std::mem;
 #[cfg(debug_assertions)]
 use std::sync::atomic::Ordering;
@@ -37,10 +36,8 @@ struct QueryStats {
     local_def_id_keys: Option<usize>,
 }
 
-fn stats<D, Q, C>(name: &'static str, map: &QueryState<D, Q, C>) -> QueryStats
+fn stats<C>(name: &'static str, map: &QueryCacheStore<C>) -> QueryStats
 where
-    D: Copy + Clone + Eq + Hash,
-    Q: Clone,
     C: QueryCache,
 {
     let mut stats = QueryStats {
@@ -128,12 +125,10 @@ macro_rules! print_stats {
 
             $(
                 queries.push(stats::<
-                    crate::dep_graph::DepKind,
-                    <TyCtxt<'_> as QueryContext>::Query,
                     <queries::$name<'_> as QueryAccessors<TyCtxt<'_>>>::Cache,
                 >(
                     stringify!($name),
-                    &tcx.queries.$name,
+                    &tcx.query_caches.$name,
                 ));
             )*
 
diff --git a/compiler/rustc_query_system/src/query/caches.rs b/compiler/rustc_query_system/src/query/caches.rs
index 1ec32939d9f8d..d589c90fa7b12 100644
--- a/compiler/rustc_query_system/src/query/caches.rs
+++ b/compiler/rustc_query_system/src/query/caches.rs
@@ -1,5 +1,5 @@
 use crate::dep_graph::DepNodeIndex;
-use crate::query::plumbing::{QueryLookup, QueryState};
+use crate::query::plumbing::{QueryCacheStore, QueryLookup};
 
 use rustc_arena::TypedArena;
 use rustc_data_structures::fx::FxHashMap;
@@ -31,13 +31,13 @@ pub trait QueryCache: QueryStorage {
     /// It returns the shard index and a lock guard to the shard,
     /// which will be used if the query is not in the cache and we need
     /// to compute it.
-    fn lookup<'s, D, Q, R, OnHit>(
+    fn lookup<'s, R, OnHit>(
         &self,
-        state: &'s QueryState<D, Q, Self>,
+        state: &'s QueryCacheStore<Self>,
         key: &Self::Key,
         // `on_hit` can be called while holding a lock to the query state shard.
         on_hit: OnHit,
-    ) -> Result<R, QueryLookup<'s, D, Q, Self::Key, Self::Sharded>>
+    ) -> Result<R, QueryLookup<'s, Self::Sharded>>
     where
         OnHit: FnOnce(&Self::Stored, DepNodeIndex) -> R;
 
@@ -93,17 +93,17 @@ where
     type Sharded = FxHashMap<K, (V, DepNodeIndex)>;
 
     #[inline(always)]
-    fn lookup<'s, D, Q, R, OnHit>(
+    fn lookup<'s, R, OnHit>(
         &self,
-        state: &'s QueryState<D, Q, Self>,
+        state: &'s QueryCacheStore<Self>,
         key: &K,
         on_hit: OnHit,
-    ) -> Result<R, QueryLookup<'s, D, Q, K, Self::Sharded>>
+    ) -> Result<R, QueryLookup<'s, Self::Sharded>>
     where
         OnHit: FnOnce(&V, DepNodeIndex) -> R,
     {
         let lookup = state.get_lookup(key);
-        let result = lookup.lock.cache.raw_entry().from_key_hashed_nocheck(lookup.key_hash, key);
+        let result = lookup.lock.raw_entry().from_key_hashed_nocheck(lookup.key_hash, key);
 
         if let Some((_, value)) = result {
             let hit_result = on_hit(&value.0, value.1);
@@ -176,17 +176,17 @@ where
     type Sharded = FxHashMap<K, &'tcx (V, DepNodeIndex)>;
 
     #[inline(always)]
-    fn lookup<'s, D, Q, R, OnHit>(
+    fn lookup<'s, R, OnHit>(
         &self,
-        state: &'s QueryState<D, Q, Self>,
+        state: &'s QueryCacheStore<Self>,
         key: &K,
         on_hit: OnHit,
-    ) -> Result<R, QueryLookup<'s, D, Q, K, Self::Sharded>>
+    ) -> Result<R, QueryLookup<'s, Self::Sharded>>
     where
         OnHit: FnOnce(&&'tcx V, DepNodeIndex) -> R,
     {
         let lookup = state.get_lookup(key);
-        let result = lookup.lock.cache.raw_entry().from_key_hashed_nocheck(lookup.key_hash, key);
+        let result = lookup.lock.raw_entry().from_key_hashed_nocheck(lookup.key_hash, key);
 
         if let Some((_, value)) = result {
             let hit_result = on_hit(&&value.0, value.1);
diff --git a/compiler/rustc_query_system/src/query/config.rs b/compiler/rustc_query_system/src/query/config.rs
index 94e906fc433d5..fecd75049fb7a 100644
--- a/compiler/rustc_query_system/src/query/config.rs
+++ b/compiler/rustc_query_system/src/query/config.rs
@@ -4,7 +4,7 @@ use crate::dep_graph::DepNode;
 use crate::dep_graph::SerializedDepNodeIndex;
 use crate::query::caches::QueryCache;
 use crate::query::plumbing::CycleError;
-use crate::query::{QueryContext, QueryState};
+use crate::query::{QueryCacheStore, QueryContext, QueryState};
 
 use rustc_data_structures::fingerprint::Fingerprint;
 use std::fmt::Debug;
@@ -73,7 +73,12 @@ pub trait QueryAccessors<CTX: QueryContext>: QueryConfig {
     type Cache: QueryCache<Key = Self::Key, Stored = Self::Stored, Value = Self::Value>;
 
     // Don't use this method to access query results, instead use the methods on TyCtxt
-    fn query_state<'a>(tcx: CTX) -> &'a QueryState<CTX::DepKind, CTX::Query, Self::Cache>;
+    fn query_state<'a>(tcx: CTX) -> &'a QueryState<CTX::DepKind, CTX::Query, Self::Key>;
+
+    // Don't use this method to access query results, instead use the methods on TyCtxt
+    fn query_cache<'a>(tcx: CTX) -> &'a QueryCacheStore<Self::Cache>
+    where
+        CTX: 'a;
 
     fn to_dep_node(tcx: CTX, key: &Self::Key) -> DepNode<CTX::DepKind>
     where
diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs
index 4f93017200f59..51a72594b5e0c 100644
--- a/compiler/rustc_query_system/src/query/plumbing.rs
+++ b/compiler/rustc_query_system/src/query/plumbing.rs
@@ -13,7 +13,7 @@ use crate::query::{QueryContext, QueryMap};
 use rustc_data_structures::cold_path;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::{FxHashMap, FxHasher};
-use rustc_data_structures::sharded::Sharded;
+use rustc_data_structures::sharded::{get_shard_index_by_hash, Sharded};
 use rustc_data_structures::sync::{Lock, LockGuard};
 use rustc_data_structures::thin_vec::ThinVec;
 use rustc_errors::{Diagnostic, FatalError};
@@ -27,43 +27,73 @@ use std::ptr;
 #[cfg(debug_assertions)]
 use std::sync::atomic::{AtomicUsize, Ordering};
 
-pub(super) struct QueryStateShard<D, Q, K, C> {
-    pub(super) cache: C,
-    active: FxHashMap<K, QueryResult<D, Q>>,
-
-    /// Used to generate unique ids for active jobs.
-    jobs: u32,
+pub struct QueryCacheStore<C: QueryCache> {
+    cache: C,
+    shards: Sharded<C::Sharded>,
+    #[cfg(debug_assertions)]
+    pub cache_hits: AtomicUsize,
 }
 
-impl<D, Q, K, C: Default> Default for QueryStateShard<D, Q, K, C> {
-    fn default() -> QueryStateShard<D, Q, K, C> {
-        QueryStateShard { cache: Default::default(), active: Default::default(), jobs: 0 }
+impl<C: QueryCache> Default for QueryCacheStore<C> {
+    fn default() -> Self {
+        Self {
+            cache: C::default(),
+            shards: Default::default(),
+            #[cfg(debug_assertions)]
+            cache_hits: AtomicUsize::new(0),
+        }
     }
 }
 
-pub struct QueryState<D, Q, C: QueryCache> {
-    cache: C,
-    shards: Sharded<QueryStateShard<D, Q, C::Key, C::Sharded>>,
-    #[cfg(debug_assertions)]
-    pub cache_hits: AtomicUsize,
+/// Values used when checking a query cache which can be reused on a cache-miss to execute the query.
+pub struct QueryLookup<'tcx, C> {
+    pub(super) key_hash: u64,
+    shard: usize,
+    pub(super) lock: LockGuard<'tcx, C>,
 }
 
-impl<D, Q, C: QueryCache> QueryState<D, Q, C> {
-    pub(super) fn get_lookup<'tcx>(
-        &'tcx self,
-        key: &C::Key,
-    ) -> QueryLookup<'tcx, D, Q, C::Key, C::Sharded> {
-        // We compute the key's hash once and then use it for both the
-        // shard lookup and the hashmap lookup. This relies on the fact
-        // that both of them use `FxHasher`.
-        let mut hasher = FxHasher::default();
-        key.hash(&mut hasher);
-        let key_hash = hasher.finish();
-
-        let shard = self.shards.get_shard_index_by_hash(key_hash);
+// We compute the key's hash once and then use it for both the
+// shard lookup and the hashmap lookup. This relies on the fact
+// that both of them use `FxHasher`.
+fn hash_for_shard<K: Hash>(key: &K) -> u64 {
+    let mut hasher = FxHasher::default();
+    key.hash(&mut hasher);
+    hasher.finish()
+}
+
+impl<C: QueryCache> QueryCacheStore<C> {
+    pub(super) fn get_lookup<'tcx>(&'tcx self, key: &C::Key) -> QueryLookup<'tcx, C::Sharded> {
+        let key_hash = hash_for_shard(key);
+        let shard = get_shard_index_by_hash(key_hash);
         let lock = self.shards.get_shard_by_index(shard).lock();
         QueryLookup { key_hash, shard, lock }
     }
+
+    pub fn iter_results<R>(
+        &self,
+        f: impl for<'a> FnOnce(
+            Box<dyn Iterator<Item = (&'a C::Key, &'a C::Value, DepNodeIndex)> + 'a>,
+        ) -> R,
+    ) -> R {
+        self.cache.iter(&self.shards, |shard| &mut *shard, f)
+    }
+}
+
+struct QueryStateShard<D, Q, K> {
+    active: FxHashMap<K, QueryResult<D, Q>>,
+
+    /// Used to generate unique ids for active jobs.
+    jobs: u32,
+}
+
+impl<D, Q, K> Default for QueryStateShard<D, Q, K> {
+    fn default() -> QueryStateShard<D, Q, K> {
+        QueryStateShard { active: Default::default(), jobs: 0 }
+    }
+}
+
+pub struct QueryState<D, Q, K> {
+    shards: Sharded<QueryStateShard<D, Q, K>>,
 }
 
 /// Indicates the state of a query for a given key in a query map.
@@ -76,21 +106,12 @@ enum QueryResult<D, Q> {
     Poisoned,
 }
 
-impl<D, Q, C> QueryState<D, Q, C>
+impl<D, Q, K> QueryState<D, Q, K>
 where
     D: Copy + Clone + Eq + Hash,
     Q: Clone,
-    C: QueryCache,
+    K: Eq + Hash + Clone + Debug,
 {
-    pub fn iter_results<R>(
-        &self,
-        f: impl for<'a> FnOnce(
-            Box<dyn Iterator<Item = (&'a C::Key, &'a C::Value, DepNodeIndex)> + 'a>,
-        ) -> R,
-    ) -> R {
-        self.cache.iter(&self.shards, |shard| &mut shard.cache, f)
-    }
-
     pub fn all_inactive(&self) -> bool {
         let shards = self.shards.lock_shards();
         shards.iter().all(|shard| shard.active.is_empty())
@@ -99,7 +120,7 @@ where
     pub fn try_collect_active_jobs(
         &self,
         kind: D,
-        make_query: fn(C::Key) -> Q,
+        make_query: fn(K) -> Q,
         jobs: &mut QueryMap<D, Q>,
     ) -> Option<()> {
         // We use try_lock_shards here since we are called from the
@@ -122,24 +143,12 @@ where
     }
 }
 
-impl<D, Q, C: QueryCache> Default for QueryState<D, Q, C> {
-    fn default() -> QueryState<D, Q, C> {
-        QueryState {
-            cache: C::default(),
-            shards: Default::default(),
-            #[cfg(debug_assertions)]
-            cache_hits: AtomicUsize::new(0),
-        }
+impl<D, Q, K> Default for QueryState<D, Q, K> {
+    fn default() -> QueryState<D, Q, K> {
+        QueryState { shards: Default::default() }
     }
 }
 
-/// Values used when checking a query cache which can be reused on a cache-miss to execute the query.
-pub struct QueryLookup<'tcx, D, Q, K, C> {
-    pub(super) key_hash: u64,
-    shard: usize,
-    pub(super) lock: LockGuard<'tcx, QueryStateShard<D, Q, K, C>>,
-}
-
 /// A type representing the responsibility to execute the job in the `job` field.
 /// This will poison the relevant query if dropped.
 struct JobOwner<'tcx, D, Q, C>
@@ -148,7 +157,8 @@ where
     Q: Clone,
     C: QueryCache,
 {
-    state: &'tcx QueryState<D, Q, C>,
+    state: &'tcx QueryState<D, Q, C::Key>,
+    cache: &'tcx QueryCacheStore<C>,
     key: C::Key,
     id: QueryJobId<D>,
 }
@@ -170,16 +180,20 @@ where
     #[inline(always)]
     fn try_start<'a, 'b, CTX>(
         tcx: CTX,
-        state: &'b QueryState<CTX::DepKind, CTX::Query, C>,
+        state: &'b QueryState<CTX::DepKind, CTX::Query, C::Key>,
+        cache: &'b QueryCacheStore<C>,
         span: Span,
         key: &C::Key,
-        mut lookup: QueryLookup<'a, CTX::DepKind, CTX::Query, C::Key, C::Sharded>,
+        lookup: QueryLookup<'a, C::Sharded>,
         query: &QueryVtable<CTX, C::Key, C::Value>,
     ) -> TryGetJob<'b, CTX::DepKind, CTX::Query, C>
     where
         CTX: QueryContext,
     {
-        let lock = &mut *lookup.lock;
+        mem::drop(lookup.lock);
+        let shard = lookup.shard;
+        let mut state_lock = state.shards.get_shard_by_index(shard).lock();
+        let lock = &mut *state_lock;
 
         let (latch, mut _query_blocked_prof_timer) = match lock.active.entry((*key).clone()) {
             Entry::Occupied(mut entry) => {
@@ -195,7 +209,7 @@ where
                         };
 
                         // Create the id of the job we're waiting for
-                        let id = QueryJobId::new(job.id, lookup.shard, query.dep_kind);
+                        let id = QueryJobId::new(job.id, shard, query.dep_kind);
 
                         (job.latch(id), _query_blocked_prof_timer)
                     }
@@ -210,18 +224,18 @@ where
                 lock.jobs = id;
                 let id = QueryShardJobId(NonZeroU32::new(id).unwrap());
 
-                let global_id = QueryJobId::new(id, lookup.shard, query.dep_kind);
+                let global_id = QueryJobId::new(id, shard, query.dep_kind);
 
                 let job = tcx.current_query_job();
                 let job = QueryJob::new(id, span, job);
 
                 entry.insert(QueryResult::Started(job));
 
-                let owner = JobOwner { state, id: global_id, key: (*key).clone() };
+                let owner = JobOwner { state, cache, id: global_id, key: (*key).clone() };
                 return TryGetJob::NotYetStarted(owner);
             }
         };
-        mem::drop(lookup.lock);
+        mem::drop(state_lock);
 
         // If we are single-threaded we know that we have cycle error,
         // so we just return the error.
@@ -233,7 +247,7 @@ where
                 span,
             );
             let value = query.handle_cycle_error(tcx, error);
-            state.cache.store_nocache(value)
+            cache.cache.store_nocache(value)
         }));
 
         // With parallel queries we might just have to wait on some other
@@ -244,11 +258,11 @@ where
 
             if let Err(cycle) = result {
                 let value = query.handle_cycle_error(tcx, cycle);
-                let value = state.cache.store_nocache(value);
+                let value = cache.cache.store_nocache(value);
                 return TryGetJob::Cycle(value);
             }
 
-            let cached = try_get_cached(tcx, state, key, |value, index| (value.clone(), index))
+            let cached = try_get_cached(tcx, cache, key, |value, index| (value.clone(), index))
                 .unwrap_or_else(|_| panic!("value must be in cache after waiting"));
 
             if let Some(prof_timer) = _query_blocked_prof_timer.take() {
@@ -265,17 +279,25 @@ where
         // We can move out of `self` here because we `mem::forget` it below
         let key = unsafe { ptr::read(&self.key) };
         let state = self.state;
+        let cache = self.cache;
 
         // Forget ourself so our destructor won't poison the query
         mem::forget(self);
 
         let (job, result) = {
-            let mut lock = state.shards.get_shard_by_value(&key).lock();
-            let job = match lock.active.remove(&key).unwrap() {
-                QueryResult::Started(job) => job,
-                QueryResult::Poisoned => panic!(),
+            let key_hash = hash_for_shard(&key);
+            let shard = get_shard_index_by_hash(key_hash);
+            let job = {
+                let mut lock = state.shards.get_shard_by_index(shard).lock();
+                match lock.active.remove(&key).unwrap() {
+                    QueryResult::Started(job) => job,
+                    QueryResult::Poisoned => panic!(),
+                }
+            };
+            let result = {
+                let mut lock = cache.shards.get_shard_by_index(shard).lock();
+                cache.cache.complete(&mut lock, key, result, dep_node_index)
             };
-            let result = state.cache.complete(&mut lock.cache, key, result, dep_node_index);
             (job, result)
         };
 
@@ -353,23 +375,23 @@ where
 /// to compute it.
 fn try_get_cached<'a, CTX, C, R, OnHit>(
     tcx: CTX,
-    state: &'a QueryState<CTX::DepKind, CTX::Query, C>,
+    cache: &'a QueryCacheStore<C>,
     key: &C::Key,
     // `on_hit` can be called while holding a lock to the query cache
     on_hit: OnHit,
-) -> Result<R, QueryLookup<'a, CTX::DepKind, CTX::Query, C::Key, C::Sharded>>
+) -> Result<R, QueryLookup<'a, C::Sharded>>
 where
     C: QueryCache,
     CTX: QueryContext,
     OnHit: FnOnce(&C::Stored, DepNodeIndex) -> R,
 {
-    state.cache.lookup(state, &key, |value, index| {
+    cache.cache.lookup(cache, &key, |value, index| {
         if unlikely!(tcx.profiler().enabled()) {
             tcx.profiler().query_cache_hit(index.into());
         }
         #[cfg(debug_assertions)]
         {
-            state.cache_hits.fetch_add(1, Ordering::Relaxed);
+            cache.cache_hits.fetch_add(1, Ordering::Relaxed);
         }
         on_hit(value, index)
     })
@@ -377,10 +399,11 @@ where
 
 fn try_execute_query<CTX, C>(
     tcx: CTX,
-    state: &QueryState<CTX::DepKind, CTX::Query, C>,
+    state: &QueryState<CTX::DepKind, CTX::Query, C::Key>,
+    cache: &QueryCacheStore<C>,
     span: Span,
     key: C::Key,
-    lookup: QueryLookup<'_, CTX::DepKind, CTX::Query, C::Key, C::Sharded>,
+    lookup: QueryLookup<'_, C::Sharded>,
     query: &QueryVtable<CTX, C::Key, C::Value>,
 ) -> C::Stored
 where
@@ -389,7 +412,7 @@ where
     CTX: QueryContext,
 {
     let job = match JobOwner::<'_, CTX::DepKind, CTX::Query, C>::try_start(
-        tcx, state, span, &key, lookup, query,
+        tcx, state, cache, span, &key, lookup, query,
     ) {
         TryGetJob::NotYetStarted(job) => job,
         TryGetJob::Cycle(result) => return result,
@@ -604,7 +627,8 @@ where
 #[inline(never)]
 fn get_query_impl<CTX, C>(
     tcx: CTX,
-    state: &QueryState<CTX::DepKind, CTX::Query, C>,
+    state: &QueryState<CTX::DepKind, CTX::Query, C::Key>,
+    cache: &QueryCacheStore<C>,
     span: Span,
     key: C::Key,
     query: &QueryVtable<CTX, C::Key, C::Value>,
@@ -614,13 +638,13 @@ where
     C: QueryCache,
     C::Key: crate::dep_graph::DepNodeParams<CTX>,
 {
-    let cached = try_get_cached(tcx, state, &key, |value, index| {
+    let cached = try_get_cached(tcx, cache, &key, |value, index| {
         tcx.dep_graph().read_index(index);
         value.clone()
     });
     match cached {
         Ok(value) => value,
-        Err(lookup) => try_execute_query(tcx, state, span, key, lookup, query),
+        Err(lookup) => try_execute_query(tcx, state, cache, span, key, lookup, query),
     }
 }
 
@@ -667,7 +691,8 @@ where
 #[inline(never)]
 fn force_query_impl<CTX, C>(
     tcx: CTX,
-    state: &QueryState<CTX::DepKind, CTX::Query, C>,
+    state: &QueryState<CTX::DepKind, CTX::Query, C::Key>,
+    cache: &QueryCacheStore<C>,
     key: C::Key,
     span: Span,
     dep_node: DepNode<CTX::DepKind>,
@@ -680,7 +705,7 @@ fn force_query_impl<CTX, C>(
     // We may be concurrently trying both execute and force a query.
     // Ensure that only one of them runs the query.
 
-    let cached = try_get_cached(tcx, state, &key, |_, _| {
+    let cached = try_get_cached(tcx, cache, &key, |_, _| {
         // Cache hit, do nothing
     });
 
@@ -690,7 +715,7 @@ fn force_query_impl<CTX, C>(
     };
 
     let job = match JobOwner::<'_, CTX::DepKind, CTX::Query, C>::try_start(
-        tcx, state, span, &key, lookup, query,
+        tcx, state, cache, span, &key, lookup, query,
     ) {
         TryGetJob::NotYetStarted(job) => job,
         TryGetJob::Cycle(_) => return,
@@ -719,7 +744,7 @@ where
     }
 
     debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span);
-    let value = get_query_impl(tcx, Q::query_state(tcx), span, key, query);
+    let value = get_query_impl(tcx, Q::query_state(tcx), Q::query_cache(tcx), span, key, query);
     Some(value)
 }
 
@@ -729,5 +754,5 @@ where
     Q::Key: crate::dep_graph::DepNodeParams<CTX>,
     CTX: QueryContext,
 {
-    force_query_impl(tcx, Q::query_state(tcx), key, span, dep_node, &Q::VTABLE)
+    force_query_impl(tcx, Q::query_state(tcx), Q::query_cache(tcx), key, span, dep_node, &Q::VTABLE)
 }

From 280a2866d502747b51bd81390be760973c54e719 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 6 Feb 2021 14:04:20 +0100
Subject: [PATCH 06/27] Drop the cache lock earlier.

---
 .../rustc_query_system/src/query/caches.rs    | 14 +++++++-------
 .../rustc_query_system/src/query/plumbing.rs  | 19 ++++++++++---------
 2 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/compiler/rustc_query_system/src/query/caches.rs b/compiler/rustc_query_system/src/query/caches.rs
index d589c90fa7b12..ec71c8685804f 100644
--- a/compiler/rustc_query_system/src/query/caches.rs
+++ b/compiler/rustc_query_system/src/query/caches.rs
@@ -37,7 +37,7 @@ pub trait QueryCache: QueryStorage {
         key: &Self::Key,
         // `on_hit` can be called while holding a lock to the query state shard.
         on_hit: OnHit,
-    ) -> Result<R, QueryLookup<'s, Self::Sharded>>
+    ) -> Result<R, QueryLookup>
     where
         OnHit: FnOnce(&Self::Stored, DepNodeIndex) -> R;
 
@@ -98,12 +98,12 @@ where
         state: &'s QueryCacheStore<Self>,
         key: &K,
         on_hit: OnHit,
-    ) -> Result<R, QueryLookup<'s, Self::Sharded>>
+    ) -> Result<R, QueryLookup>
     where
         OnHit: FnOnce(&V, DepNodeIndex) -> R,
     {
-        let lookup = state.get_lookup(key);
-        let result = lookup.lock.raw_entry().from_key_hashed_nocheck(lookup.key_hash, key);
+        let (lookup, lock) = state.get_lookup(key);
+        let result = lock.raw_entry().from_key_hashed_nocheck(lookup.key_hash, key);
 
         if let Some((_, value)) = result {
             let hit_result = on_hit(&value.0, value.1);
@@ -181,12 +181,12 @@ where
         state: &'s QueryCacheStore<Self>,
         key: &K,
         on_hit: OnHit,
-    ) -> Result<R, QueryLookup<'s, Self::Sharded>>
+    ) -> Result<R, QueryLookup>
     where
         OnHit: FnOnce(&&'tcx V, DepNodeIndex) -> R,
     {
-        let lookup = state.get_lookup(key);
-        let result = lookup.lock.raw_entry().from_key_hashed_nocheck(lookup.key_hash, key);
+        let (lookup, lock) = state.get_lookup(key);
+        let result = lock.raw_entry().from_key_hashed_nocheck(lookup.key_hash, key);
 
         if let Some((_, value)) = result {
             let hit_result = on_hit(&&value.0, value.1);
diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs
index 51a72594b5e0c..c2e89e131b3fe 100644
--- a/compiler/rustc_query_system/src/query/plumbing.rs
+++ b/compiler/rustc_query_system/src/query/plumbing.rs
@@ -46,10 +46,9 @@ impl<C: QueryCache> Default for QueryCacheStore<C> {
 }
 
 /// Values used when checking a query cache which can be reused on a cache-miss to execute the query.
-pub struct QueryLookup<'tcx, C> {
+pub struct QueryLookup {
     pub(super) key_hash: u64,
     shard: usize,
-    pub(super) lock: LockGuard<'tcx, C>,
 }
 
 // We compute the key's hash once and then use it for both the
@@ -62,11 +61,14 @@ fn hash_for_shard<K: Hash>(key: &K) -> u64 {
 }
 
 impl<C: QueryCache> QueryCacheStore<C> {
-    pub(super) fn get_lookup<'tcx>(&'tcx self, key: &C::Key) -> QueryLookup<'tcx, C::Sharded> {
+    pub(super) fn get_lookup<'tcx>(
+        &'tcx self,
+        key: &C::Key,
+    ) -> (QueryLookup, LockGuard<'tcx, C::Sharded>) {
         let key_hash = hash_for_shard(key);
         let shard = get_shard_index_by_hash(key_hash);
         let lock = self.shards.get_shard_by_index(shard).lock();
-        QueryLookup { key_hash, shard, lock }
+        (QueryLookup { key_hash, shard }, lock)
     }
 
     pub fn iter_results<R>(
@@ -178,19 +180,18 @@ where
     /// This function is inlined because that results in a noticeable speed-up
     /// for some compile-time benchmarks.
     #[inline(always)]
-    fn try_start<'a, 'b, CTX>(
+    fn try_start<'b, CTX>(
         tcx: CTX,
         state: &'b QueryState<CTX::DepKind, CTX::Query, C::Key>,
         cache: &'b QueryCacheStore<C>,
         span: Span,
         key: &C::Key,
-        lookup: QueryLookup<'a, C::Sharded>,
+        lookup: QueryLookup,
         query: &QueryVtable<CTX, C::Key, C::Value>,
     ) -> TryGetJob<'b, CTX::DepKind, CTX::Query, C>
     where
         CTX: QueryContext,
     {
-        mem::drop(lookup.lock);
         let shard = lookup.shard;
         let mut state_lock = state.shards.get_shard_by_index(shard).lock();
         let lock = &mut *state_lock;
@@ -379,7 +380,7 @@ fn try_get_cached<'a, CTX, C, R, OnHit>(
     key: &C::Key,
     // `on_hit` can be called while holding a lock to the query cache
     on_hit: OnHit,
-) -> Result<R, QueryLookup<'a, C::Sharded>>
+) -> Result<R, QueryLookup>
 where
     C: QueryCache,
     CTX: QueryContext,
@@ -403,7 +404,7 @@ fn try_execute_query<CTX, C>(
     cache: &QueryCacheStore<C>,
     span: Span,
     key: C::Key,
-    lookup: QueryLookup<'_, C::Sharded>,
+    lookup: QueryLookup,
     query: &QueryVtable<CTX, C::Key, C::Value>,
 ) -> C::Stored
 where

From 3fc8ed68e99034ad5410cef47e8cd94828ef8946 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 6 Feb 2021 14:52:04 +0100
Subject: [PATCH 07/27] Check query cache before calling into the query engine.

---
 .../rustc_middle/src/ty/query/plumbing.rs     | 24 +++++++--
 .../rustc_query_system/src/query/plumbing.rs  | 52 +++++++++++++------
 2 files changed, 56 insertions(+), 20 deletions(-)

diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs
index 9a011846fd62d..0961d4d0091d0 100644
--- a/compiler/rustc_middle/src/ty/query/plumbing.rs
+++ b/compiler/rustc_middle/src/ty/query/plumbing.rs
@@ -422,7 +422,15 @@ macro_rules! define_queries {
             $($(#[$attr])*
             #[inline(always)]
             pub fn $name(self, key: query_helper_param_ty!($($K)*)) {
-                get_query::<queries::$name<'_>, _>(self.tcx, DUMMY_SP, key.into_query_param(), QueryMode::Ensure);
+                let key = key.into_query_param();
+                let cached = try_get_cached(self.tcx, &self.tcx.query_caches.$name, &key, |_| {});
+
+                let lookup = match cached {
+                    Ok(()) => return,
+                    Err(lookup) => lookup,
+                };
+
+                get_query::<queries::$name<'_>, _>(self.tcx, DUMMY_SP, key, lookup, QueryMode::Ensure);
             })*
         }
 
@@ -465,7 +473,7 @@ macro_rules! define_queries {
             #[must_use]
             pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> query_stored::$name<$tcx>
             {
-                self.at(DUMMY_SP).$name(key.into_query_param())
+                self.at(DUMMY_SP).$name(key)
             })*
 
             /// All self-profiling events generated by the query engine use
@@ -503,7 +511,17 @@ macro_rules! define_queries {
             #[inline(always)]
             pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> query_stored::$name<$tcx>
             {
-                get_query::<queries::$name<'_>, _>(self.tcx, self.span, key.into_query_param(), QueryMode::Get).unwrap()
+                let key = key.into_query_param();
+                let cached = try_get_cached(self.tcx, &self.tcx.query_caches.$name, &key, |value| {
+                    value.clone()
+                });
+
+                let lookup = match cached {
+                    Ok(value) => return value,
+                    Err(lookup) => lookup,
+                };
+
+                get_query::<queries::$name<'_>, _>(self.tcx, self.span, key, lookup, QueryMode::Get).unwrap()
             })*
         }
 
diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs
index c2e89e131b3fe..2610ce83e4d3e 100644
--- a/compiler/rustc_query_system/src/query/plumbing.rs
+++ b/compiler/rustc_query_system/src/query/plumbing.rs
@@ -263,7 +263,18 @@ where
                 return TryGetJob::Cycle(value);
             }
 
-            let cached = try_get_cached(tcx, cache, key, |value, index| (value.clone(), index))
+            let cached = cache
+                .cache
+                .lookup(cache, &key, |value, index| {
+                    if unlikely!(tcx.profiler().enabled()) {
+                        tcx.profiler().query_cache_hit(index.into());
+                    }
+                    #[cfg(debug_assertions)]
+                    {
+                        cache.cache_hits.fetch_add(1, Ordering::Relaxed);
+                    }
+                    (value.clone(), index)
+                })
                 .unwrap_or_else(|_| panic!("value must be in cache after waiting"));
 
             if let Some(prof_timer) = _query_blocked_prof_timer.take() {
@@ -374,7 +385,7 @@ where
 /// It returns the shard index and a lock guard to the shard,
 /// which will be used if the query is not in the cache and we need
 /// to compute it.
-fn try_get_cached<'a, CTX, C, R, OnHit>(
+pub fn try_get_cached<'a, CTX, C, R, OnHit>(
     tcx: CTX,
     cache: &'a QueryCacheStore<C>,
     key: &C::Key,
@@ -384,7 +395,7 @@ fn try_get_cached<'a, CTX, C, R, OnHit>(
 where
     C: QueryCache,
     CTX: QueryContext,
-    OnHit: FnOnce(&C::Stored, DepNodeIndex) -> R,
+    OnHit: FnOnce(&C::Stored) -> R,
 {
     cache.cache.lookup(cache, &key, |value, index| {
         if unlikely!(tcx.profiler().enabled()) {
@@ -394,7 +405,8 @@ where
         {
             cache.cache_hits.fetch_add(1, Ordering::Relaxed);
         }
-        on_hit(value, index)
+        tcx.dep_graph().read_index(index);
+        on_hit(value)
     })
 }
 
@@ -632,6 +644,7 @@ fn get_query_impl<CTX, C>(
     cache: &QueryCacheStore<C>,
     span: Span,
     key: C::Key,
+    lookup: QueryLookup,
     query: &QueryVtable<CTX, C::Key, C::Value>,
 ) -> C::Stored
 where
@@ -639,14 +652,7 @@ where
     C: QueryCache,
     C::Key: crate::dep_graph::DepNodeParams<CTX>,
 {
-    let cached = try_get_cached(tcx, cache, &key, |value, index| {
-        tcx.dep_graph().read_index(index);
-        value.clone()
-    });
-    match cached {
-        Ok(value) => value,
-        Err(lookup) => try_execute_query(tcx, state, cache, span, key, lookup, query),
-    }
+    try_execute_query(tcx, state, cache, span, key, lookup, query)
 }
 
 /// Ensure that either this query has all green inputs or been executed.
@@ -705,9 +711,14 @@ fn force_query_impl<CTX, C>(
 {
     // We may be concurrently trying both execute and force a query.
     // Ensure that only one of them runs the query.
-
-    let cached = try_get_cached(tcx, cache, &key, |_, _| {
-        // Cache hit, do nothing
+    let cached = cache.cache.lookup(cache, &key, |_, index| {
+        if unlikely!(tcx.profiler().enabled()) {
+            tcx.profiler().query_cache_hit(index.into());
+        }
+        #[cfg(debug_assertions)]
+        {
+            cache.cache_hits.fetch_add(1, Ordering::Relaxed);
+        }
     });
 
     let lookup = match cached {
@@ -731,7 +742,13 @@ pub enum QueryMode {
     Ensure,
 }
 
-pub fn get_query<Q, CTX>(tcx: CTX, span: Span, key: Q::Key, mode: QueryMode) -> Option<Q::Stored>
+pub fn get_query<Q, CTX>(
+    tcx: CTX,
+    span: Span,
+    key: Q::Key,
+    lookup: QueryLookup,
+    mode: QueryMode,
+) -> Option<Q::Stored>
 where
     Q: QueryDescription<CTX>,
     Q::Key: crate::dep_graph::DepNodeParams<CTX>,
@@ -745,7 +762,8 @@ where
     }
 
     debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span);
-    let value = get_query_impl(tcx, Q::query_state(tcx), Q::query_cache(tcx), span, key, query);
+    let value =
+        get_query_impl(tcx, Q::query_state(tcx), Q::query_cache(tcx), span, key, lookup, query);
     Some(value)
 }
 

From 749573227b4e80cf1f52c48f37238b87ec3fb9f9 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Thu, 9 Apr 2020 09:06:11 +0200
Subject: [PATCH 08/27] Group logic about the Providers struct.

---
 compiler/rustc_middle/src/ty/query/plumbing.rs | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs
index 0961d4d0091d0..1cb705c251c8b 100644
--- a/compiler/rustc_middle/src/ty/query/plumbing.rs
+++ b/compiler/rustc_middle/src/ty/query/plumbing.rs
@@ -529,11 +529,6 @@ macro_rules! define_queries {
             tcx: $tcx,
             input: ($(([$($modifiers)*] [$name] [$($K)*] [$V]))*)
         }
-
-        impl Copy for Providers {}
-        impl Clone for Providers {
-            fn clone(&self) -> Self { *self }
-        }
     }
 }
 
@@ -609,5 +604,10 @@ macro_rules! define_provider_struct {
                 Providers { $($name),* }
             }
         }
+
+        impl Copy for Providers {}
+        impl Clone for Providers {
+            fn clone(&self) -> Self { *self }
+        }
     };
 }

From 63058358186f2184b06164625aad13078fb76894 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sun, 11 Oct 2020 10:34:50 +0200
Subject: [PATCH 09/27] Don't require a QueryContext to access the DepGraph.

---
 compiler/rustc_middle/src/dep_graph/mod.rs            | 5 +++++
 compiler/rustc_middle/src/ty/query/plumbing.rs        | 5 -----
 compiler/rustc_query_system/src/cache.rs              | 7 +++----
 compiler/rustc_query_system/src/dep_graph/dep_node.rs | 2 +-
 compiler/rustc_query_system/src/dep_graph/mod.rs      | 3 +++
 compiler/rustc_query_system/src/query/mod.rs          | 5 +----
 6 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/compiler/rustc_middle/src/dep_graph/mod.rs b/compiler/rustc_middle/src/dep_graph/mod.rs
index b88ffa2bb7347..eaf414ccaa69d 100644
--- a/compiler/rustc_middle/src/dep_graph/mod.rs
+++ b/compiler/rustc_middle/src/dep_graph/mod.rs
@@ -111,6 +111,11 @@ impl<'tcx> DepContext for TyCtxt<'tcx> {
             || self.sess.opts.debugging_opts.query_dep_graph
     }
 
+    #[inline]
+    fn dep_graph(&self) -> &DepGraph {
+        &self.dep_graph
+    }
+
     fn try_force_from_dep_node(&self, dep_node: &DepNode) -> bool {
         // FIXME: This match is just a workaround for incremental bugs and should
         // be removed. https://github.com/rust-lang/rust/issues/62649 is one such
diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs
index 1cb705c251c8b..891da797b34ec 100644
--- a/compiler/rustc_middle/src/ty/query/plumbing.rs
+++ b/compiler/rustc_middle/src/ty/query/plumbing.rs
@@ -2,7 +2,6 @@
 //! generate the actual methods on tcx which find and execute the provider,
 //! manage the caches, and so forth.
 
-use crate::dep_graph::DepGraph;
 use crate::ty::query::Query;
 use crate::ty::tls::{self, ImplicitCtxt};
 use crate::ty::{self, TyCtxt};
@@ -30,10 +29,6 @@ impl QueryContext for TyCtxt<'tcx> {
         TyCtxt::def_path_str(*self, def_id)
     }
 
-    fn dep_graph(&self) -> &DepGraph {
-        &self.dep_graph
-    }
-
     fn current_query_job(&self) -> Option<QueryJobId<Self::DepKind>> {
         tls::with_related_context(*self, |icx| icx.query)
     }
diff --git a/compiler/rustc_query_system/src/cache.rs b/compiler/rustc_query_system/src/cache.rs
index be3d360772854..c6dc7b4fe2851 100644
--- a/compiler/rustc_query_system/src/cache.rs
+++ b/compiler/rustc_query_system/src/cache.rs
@@ -1,7 +1,6 @@
 //! Cache for candidate selection.
 
-use crate::dep_graph::DepNodeIndex;
-use crate::query::QueryContext;
+use crate::dep_graph::{DepContext, DepNodeIndex};
 
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::HashMapExt;
@@ -28,7 +27,7 @@ impl<Key, Value> Cache<Key, Value> {
 }
 
 impl<Key: Eq + Hash, Value: Clone> Cache<Key, Value> {
-    pub fn get<CTX: QueryContext>(&self, key: &Key, tcx: CTX) -> Option<Value> {
+    pub fn get<CTX: DepContext>(&self, key: &Key, tcx: CTX) -> Option<Value> {
         Some(self.hashmap.borrow().get(key)?.get(tcx))
     }
 
@@ -55,7 +54,7 @@ impl<T: Clone> WithDepNode<T> {
         WithDepNode { dep_node, cached_value }
     }
 
-    pub fn get<CTX: QueryContext>(&self, tcx: CTX) -> T {
+    pub fn get<CTX: DepContext>(&self, tcx: CTX) -> T {
         tcx.dep_graph().read_index(self.dep_node);
         self.cached_value.clone()
     }
diff --git a/compiler/rustc_query_system/src/dep_graph/dep_node.rs b/compiler/rustc_query_system/src/dep_graph/dep_node.rs
index 64aba870502c7..1319a31b8f54d 100644
--- a/compiler/rustc_query_system/src/dep_graph/dep_node.rs
+++ b/compiler/rustc_query_system/src/dep_graph/dep_node.rs
@@ -79,7 +79,7 @@ impl<K: DepKind> DepNode<K> {
 
     pub fn construct<Ctxt, Key>(tcx: Ctxt, kind: K, arg: &Key) -> DepNode<K>
     where
-        Ctxt: crate::query::QueryContext<DepKind = K>,
+        Ctxt: super::DepContext<DepKind = K>,
         Key: DepNodeParams<Ctxt>,
     {
         let hash = arg.to_fingerprint(tcx);
diff --git a/compiler/rustc_query_system/src/dep_graph/mod.rs b/compiler/rustc_query_system/src/dep_graph/mod.rs
index b1c901633a71b..5c1444ad6c4ca 100644
--- a/compiler/rustc_query_system/src/dep_graph/mod.rs
+++ b/compiler/rustc_query_system/src/dep_graph/mod.rs
@@ -29,6 +29,9 @@ pub trait DepContext: Copy {
     fn debug_dep_tasks(&self) -> bool;
     fn debug_dep_node(&self) -> bool;
 
+    /// Access the DepGraph.
+    fn dep_graph(&self) -> &DepGraph<Self::DepKind>;
+
     /// Try to force a dep node to execute and see if it's green.
     fn try_force_from_dep_node(&self, dep_node: &DepNode<Self::DepKind>) -> bool;
 
diff --git a/compiler/rustc_query_system/src/query/mod.rs b/compiler/rustc_query_system/src/query/mod.rs
index da45565dbe6bd..84d4b406c84e1 100644
--- a/compiler/rustc_query_system/src/query/mod.rs
+++ b/compiler/rustc_query_system/src/query/mod.rs
@@ -14,7 +14,7 @@ pub use self::caches::{
 mod config;
 pub use self::config::{QueryAccessors, QueryConfig, QueryDescription};
 
-use crate::dep_graph::{DepContext, DepGraph};
+use crate::dep_graph::DepContext;
 use crate::query::job::QueryMap;
 
 use rustc_data_structures::stable_hasher::HashStable;
@@ -32,9 +32,6 @@ pub trait QueryContext: DepContext {
     /// Get string representation from DefPath.
     fn def_path_str(&self, def_id: DefId) -> String;
 
-    /// Access the DepGraph.
-    fn dep_graph(&self) -> &DepGraph<Self::DepKind>;
-
     /// Get the query information from the TLS context.
     fn current_query_job(&self) -> Option<QueryJobId<Self::DepKind>>;
 

From fa4d3bea27500f900e2a41e7fb81b4400421964d Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sun, 18 Oct 2020 21:00:24 +0200
Subject: [PATCH 10/27] Remove QueryAccessors::to_dep_node.

---
 compiler/rustc_query_system/src/query/config.rs | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/compiler/rustc_query_system/src/query/config.rs b/compiler/rustc_query_system/src/query/config.rs
index fecd75049fb7a..23b1ab09722a1 100644
--- a/compiler/rustc_query_system/src/query/config.rs
+++ b/compiler/rustc_query_system/src/query/config.rs
@@ -80,13 +80,6 @@ pub trait QueryAccessors<CTX: QueryContext>: QueryConfig {
     where
         CTX: 'a;
 
-    fn to_dep_node(tcx: CTX, key: &Self::Key) -> DepNode<CTX::DepKind>
-    where
-        Self::Key: crate::dep_graph::DepNodeParams<CTX>,
-    {
-        DepNode::construct(tcx, Self::DEP_KIND, key)
-    }
-
     // Don't use this method to compute query results, instead use the methods on TyCtxt
     fn compute(tcx: CTX, key: Self::Key) -> Self::Value;
 

From 59b96b400c640acc5a529a61665cd6d160412184 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sun, 18 Oct 2020 21:01:36 +0200
Subject: [PATCH 11/27] Decouple QueryContext from DepContext.

---
 .../rustc_middle/src/ty/query/plumbing.rs     |  4 +-
 .../rustc_query_system/src/dep_graph/graph.rs | 13 +--
 .../rustc_query_system/src/dep_graph/mod.rs   | 21 +++++
 .../rustc_query_system/src/query/config.rs    |  4 +-
 compiler/rustc_query_system/src/query/job.rs  |  5 +-
 compiler/rustc_query_system/src/query/mod.rs  |  6 +-
 .../rustc_query_system/src/query/plumbing.rs  | 87 +++++++++++--------
 7 files changed, 87 insertions(+), 53 deletions(-)

diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs
index 891da797b34ec..ea889549e43da 100644
--- a/compiler/rustc_middle/src/ty/query/plumbing.rs
+++ b/compiler/rustc_middle/src/ty/query/plumbing.rs
@@ -48,7 +48,7 @@ impl QueryContext for TyCtxt<'tcx> {
         &self,
         token: QueryJobId<Self::DepKind>,
         diagnostics: Option<&Lock<ThinVec<Diagnostic>>>,
-        compute: impl FnOnce(Self) -> R,
+        compute: impl FnOnce() -> R,
     ) -> R {
         // The `TyCtxt` stored in TLS has the same global interner lifetime
         // as `self`, so we use `with_related_context` to relate the 'tcx lifetimes
@@ -65,7 +65,7 @@ impl QueryContext for TyCtxt<'tcx> {
 
             // Use the `ImplicitCtxt` while we execute the query.
             tls::enter_context(&new_icx, |_| {
-                rustc_data_structures::stack::ensure_sufficient_stack(|| compute(*self))
+                rustc_data_structures::stack::ensure_sufficient_stack(compute)
             })
         })
     }
diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs
index 4fb3a683ea225..9c441bf036434 100644
--- a/compiler/rustc_query_system/src/dep_graph/graph.rs
+++ b/compiler/rustc_query_system/src/dep_graph/graph.rs
@@ -23,7 +23,7 @@ use super::debug::EdgeFilter;
 use super::prev::PreviousDepGraph;
 use super::query::DepGraphQuery;
 use super::serialized::SerializedDepNodeIndex;
-use super::{DepContext, DepKind, DepNode, WorkProductId};
+use super::{DepContext, DepKind, DepNode, HasDepContext, WorkProductId};
 
 #[derive(Clone)]
 pub struct DepGraph<K: DepKind> {
@@ -235,7 +235,7 @@ impl<K: DepKind> DepGraph<K> {
     ///   `arg` parameter.
     ///
     /// [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/incremental-compilation.html
-    pub fn with_task<Ctxt: DepContext<DepKind = K>, A, R>(
+    pub fn with_task<Ctxt: HasDepContext<DepKind = K>, A, R>(
         &self,
         key: DepNode<K>,
         cx: Ctxt,
@@ -261,7 +261,7 @@ impl<K: DepKind> DepGraph<K> {
         )
     }
 
-    fn with_task_impl<Ctxt: DepContext<DepKind = K>, A, R>(
+    fn with_task_impl<Ctxt: HasDepContext<DepKind = K>, A, R>(
         &self,
         key: DepNode<K>,
         cx: Ctxt,
@@ -271,14 +271,15 @@ impl<K: DepKind> DepGraph<K> {
         hash_result: impl FnOnce(&mut Ctxt::StableHashingContext, &R) -> Option<Fingerprint>,
     ) -> (R, DepNodeIndex) {
         if let Some(ref data) = self.data {
+            let dcx = cx.dep_context();
             let task_deps = create_task(key).map(Lock::new);
             let result = K::with_deps(task_deps.as_ref(), || task(cx, arg));
             let edges = task_deps.map_or_else(|| smallvec![], |lock| lock.into_inner().reads);
 
-            let mut hcx = cx.create_stable_hashing_context();
+            let mut hcx = dcx.create_stable_hashing_context();
             let current_fingerprint = hash_result(&mut hcx, &result);
 
-            let print_status = cfg!(debug_assertions) && cx.debug_dep_tasks();
+            let print_status = cfg!(debug_assertions) && dcx.debug_dep_tasks();
 
             // Intern the new `DepNode`.
             let dep_node_index = if let Some(prev_index) = data.previous.node_to_index_opt(&key) {
@@ -408,7 +409,7 @@ impl<K: DepKind> DepGraph<K> {
 
     /// Executes something within an "eval-always" task which is a task
     /// that runs whenever anything changes.
-    pub fn with_eval_always_task<Ctxt: DepContext<DepKind = K>, A, R>(
+    pub fn with_eval_always_task<Ctxt: HasDepContext<DepKind = K>, A, R>(
         &self,
         key: DepNode<K>,
         cx: Ctxt,
diff --git a/compiler/rustc_query_system/src/dep_graph/mod.rs b/compiler/rustc_query_system/src/dep_graph/mod.rs
index 5c1444ad6c4ca..db192d1cfe77a 100644
--- a/compiler/rustc_query_system/src/dep_graph/mod.rs
+++ b/compiler/rustc_query_system/src/dep_graph/mod.rs
@@ -63,6 +63,27 @@ pub trait DepContext: Copy {
     fn profiler(&self) -> &SelfProfilerRef;
 }
 
+pub trait HasDepContext: Copy {
+    type DepKind: self::DepKind;
+    type StableHashingContext;
+    type DepContext: self::DepContext<
+        DepKind = Self::DepKind,
+        StableHashingContext = Self::StableHashingContext,
+    >;
+
+    fn dep_context(&self) -> &Self::DepContext;
+}
+
+impl<T: DepContext> HasDepContext for T {
+    type DepKind = T::DepKind;
+    type StableHashingContext = T::StableHashingContext;
+    type DepContext = Self;
+
+    fn dep_context(&self) -> &Self::DepContext {
+        self
+    }
+}
+
 /// Describe the different families of dependency nodes.
 pub trait DepKind: Copy + fmt::Debug + Eq + Hash {
     const NULL: Self;
diff --git a/compiler/rustc_query_system/src/query/config.rs b/compiler/rustc_query_system/src/query/config.rs
index 23b1ab09722a1..3873b47d4d40b 100644
--- a/compiler/rustc_query_system/src/query/config.rs
+++ b/compiler/rustc_query_system/src/query/config.rs
@@ -33,9 +33,9 @@ pub(crate) struct QueryVtable<CTX: QueryContext, K, V> {
 }
 
 impl<CTX: QueryContext, K, V> QueryVtable<CTX, K, V> {
-    pub(crate) fn to_dep_node(&self, tcx: CTX, key: &K) -> DepNode<CTX::DepKind>
+    pub(crate) fn to_dep_node(&self, tcx: CTX::DepContext, key: &K) -> DepNode<CTX::DepKind>
     where
-        K: crate::dep_graph::DepNodeParams<CTX>,
+        K: crate::dep_graph::DepNodeParams<CTX::DepContext>,
     {
         DepNode::construct(tcx, self.dep_kind, key)
     }
diff --git a/compiler/rustc_query_system/src/query/job.rs b/compiler/rustc_query_system/src/query/job.rs
index 5fed500390b68..0ecc2694a7906 100644
--- a/compiler/rustc_query_system/src/query/job.rs
+++ b/compiler/rustc_query_system/src/query/job.rs
@@ -10,7 +10,8 @@ use std::num::NonZeroU32;
 
 #[cfg(parallel_compiler)]
 use {
-    super::QueryContext,
+    crate::dep_graph::DepContext,
+    crate::query::QueryContext,
     parking_lot::{Condvar, Mutex},
     rustc_data_structures::fx::FxHashSet,
     rustc_data_structures::stable_hasher::{HashStable, StableHasher},
@@ -432,7 +433,7 @@ where
 {
     // Deterministically pick an entry point
     // FIXME: Sort this instead
-    let mut hcx = tcx.create_stable_hashing_context();
+    let mut hcx = tcx.dep_context().create_stable_hashing_context();
     queries
         .iter()
         .min_by_key(|v| {
diff --git a/compiler/rustc_query_system/src/query/mod.rs b/compiler/rustc_query_system/src/query/mod.rs
index 84d4b406c84e1..2d678035d4d51 100644
--- a/compiler/rustc_query_system/src/query/mod.rs
+++ b/compiler/rustc_query_system/src/query/mod.rs
@@ -14,7 +14,7 @@ pub use self::caches::{
 mod config;
 pub use self::config::{QueryAccessors, QueryConfig, QueryDescription};
 
-use crate::dep_graph::DepContext;
+use crate::dep_graph::HasDepContext;
 use crate::query::job::QueryMap;
 
 use rustc_data_structures::stable_hasher::HashStable;
@@ -23,7 +23,7 @@ use rustc_data_structures::thin_vec::ThinVec;
 use rustc_errors::Diagnostic;
 use rustc_span::def_id::DefId;
 
-pub trait QueryContext: DepContext {
+pub trait QueryContext: HasDepContext {
     type Query: Clone + HashStable<Self::StableHashingContext>;
 
     fn incremental_verify_ich(&self) -> bool;
@@ -44,6 +44,6 @@ pub trait QueryContext: DepContext {
         &self,
         token: QueryJobId<Self::DepKind>,
         diagnostics: Option<&Lock<ThinVec<Diagnostic>>>,
-        compute: impl FnOnce(Self) -> R,
+        compute: impl FnOnce() -> R,
     ) -> R;
 }
diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs
index 2610ce83e4d3e..eaaba10344a21 100644
--- a/compiler/rustc_query_system/src/query/plumbing.rs
+++ b/compiler/rustc_query_system/src/query/plumbing.rs
@@ -2,7 +2,7 @@
 //! generate the actual methods on tcx which find and execute the provider,
 //! manage the caches, and so forth.
 
-use crate::dep_graph::{DepKind, DepNode};
+use crate::dep_graph::{DepContext, DepKind, DepNode};
 use crate::dep_graph::{DepNodeIndex, SerializedDepNodeIndex};
 use crate::query::caches::QueryCache;
 use crate::query::config::{QueryDescription, QueryVtable, QueryVtableExt};
@@ -204,7 +204,7 @@ where
                         // in another thread has completed. Record how long we wait in the
                         // self-profiler.
                         let _query_blocked_prof_timer = if cfg!(parallel_compiler) {
-                            Some(tcx.profiler().query_blocked())
+                            Some(tcx.dep_context().profiler().query_blocked())
                         } else {
                             None
                         };
@@ -266,8 +266,8 @@ where
             let cached = cache
                 .cache
                 .lookup(cache, &key, |value, index| {
-                    if unlikely!(tcx.profiler().enabled()) {
-                        tcx.profiler().query_cache_hit(index.into());
+                    if unlikely!(tcx.dep_context().profiler().enabled()) {
+                        tcx.dep_context().profiler().query_cache_hit(index.into());
                     }
                     #[cfg(debug_assertions)]
                     {
@@ -394,7 +394,7 @@ pub fn try_get_cached<'a, CTX, C, R, OnHit>(
 ) -> Result<R, QueryLookup>
 where
     C: QueryCache,
-    CTX: QueryContext,
+    CTX: DepContext,
     OnHit: FnOnce(&C::Stored) -> R,
 {
     cache.cache.lookup(cache, &key, |value, index| {
@@ -421,7 +421,7 @@ fn try_execute_query<CTX, C>(
 ) -> C::Stored
 where
     C: QueryCache,
-    C::Key: crate::dep_graph::DepNodeParams<CTX>,
+    C::Key: crate::dep_graph::DepNodeParams<CTX::DepContext>,
     CTX: QueryContext,
 {
     let job = match JobOwner::<'_, CTX::DepKind, CTX::Query, C>::try_start(
@@ -431,46 +431,51 @@ where
         TryGetJob::Cycle(result) => return result,
         #[cfg(parallel_compiler)]
         TryGetJob::JobCompleted((v, index)) => {
-            tcx.dep_graph().read_index(index);
+            tcx.dep_context().dep_graph().read_index(index);
             return v;
         }
     };
 
     // Fast path for when incr. comp. is off. `to_dep_node` is
     // expensive for some `DepKind`s.
-    if !tcx.dep_graph().is_fully_enabled() {
+    if !tcx.dep_context().dep_graph().is_fully_enabled() {
         let null_dep_node = DepNode::new_no_params(DepKind::NULL);
         return force_query_with_job(tcx, key, job, null_dep_node, query).0;
     }
 
     if query.anon {
-        let prof_timer = tcx.profiler().query_provider();
+        let prof_timer = tcx.dep_context().profiler().query_provider();
 
         let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
-            tcx.start_query(job.id, diagnostics, |tcx| {
-                tcx.dep_graph().with_anon_task(query.dep_kind, || query.compute(tcx, key))
+            tcx.start_query(job.id, diagnostics, || {
+                tcx.dep_context()
+                    .dep_graph()
+                    .with_anon_task(query.dep_kind, || query.compute(tcx, key))
             })
         });
 
         prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 
-        tcx.dep_graph().read_index(dep_node_index);
+        tcx.dep_context().dep_graph().read_index(dep_node_index);
 
         if unlikely!(!diagnostics.is_empty()) {
-            tcx.store_diagnostics_for_anon_node(dep_node_index, diagnostics);
+            tcx.dep_context().store_diagnostics_for_anon_node(dep_node_index, diagnostics);
         }
 
         return job.complete(result, dep_node_index);
     }
 
-    let dep_node = query.to_dep_node(tcx, &key);
+    let dep_node = query.to_dep_node(*tcx.dep_context(), &key);
 
     if !query.eval_always {
         // The diagnostics for this query will be
         // promoted to the current session during
         // `try_mark_green()`, so we can ignore them here.
-        let loaded = tcx.start_query(job.id, None, |tcx| {
-            let marked = tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node);
+        let loaded = tcx.start_query(job.id, None, || {
+            let marked = tcx
+                .dep_context()
+                .dep_graph()
+                .try_mark_green_and_read(*tcx.dep_context(), &dep_node);
             marked.map(|(prev_dep_node_index, dep_node_index)| {
                 (
                     load_from_disk_and_cache_in_memory(
@@ -491,7 +496,7 @@ where
     }
 
     let (result, dep_node_index) = force_query_with_job(tcx, key, job, dep_node, query);
-    tcx.dep_graph().read_index(dep_node_index);
+    tcx.dep_context().dep_graph().read_index(dep_node_index);
     result
 }
 
@@ -509,11 +514,11 @@ where
     // Note this function can be called concurrently from the same query
     // We must ensure that this is handled correctly.
 
-    debug_assert!(tcx.dep_graph().is_green(dep_node));
+    debug_assert!(tcx.dep_context().dep_graph().is_green(dep_node));
 
     // First we try to load the result from the on-disk cache.
     let result = if query.cache_on_disk(tcx, &key, None) {
-        let prof_timer = tcx.profiler().incr_cache_loading();
+        let prof_timer = tcx.dep_context().profiler().incr_cache_loading();
         let result = query.try_load_from_disk(tcx, prev_dep_node_index);
         prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 
@@ -535,10 +540,10 @@ where
     } else {
         // We could not load a result from the on-disk cache, so
         // recompute.
-        let prof_timer = tcx.profiler().query_provider();
+        let prof_timer = tcx.dep_context().profiler().query_provider();
 
         // The dep-graph for this computation is already in-place.
-        let result = tcx.dep_graph().with_ignore(|| query.compute(tcx, key));
+        let result = tcx.dep_context().dep_graph().with_ignore(|| query.compute(tcx, key));
 
         prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 
@@ -548,7 +553,7 @@ where
     // If `-Zincremental-verify-ich` is specified, re-hash results from
     // the cache and make sure that they have the expected fingerprint.
     if unlikely!(tcx.incremental_verify_ich()) {
-        incremental_verify_ich(tcx, &result, dep_node, dep_node_index, query);
+        incremental_verify_ich(*tcx.dep_context(), &result, dep_node, dep_node_index, query);
     }
 
     result
@@ -557,7 +562,7 @@ where
 #[inline(never)]
 #[cold]
 fn incremental_verify_ich<CTX, K, V: Debug>(
-    tcx: CTX,
+    tcx: CTX::DepContext,
     result: &V,
     dep_node: &DepNode<CTX::DepKind>,
     dep_node_index: DepNodeIndex,
@@ -600,7 +605,7 @@ where
     // 2. Two distinct query keys get mapped to the same `DepNode`
     //    (see for example #48923).
     assert!(
-        !tcx.dep_graph().dep_node_exists(&dep_node),
+        !tcx.dep_context().dep_graph().dep_node_exists(&dep_node),
         "forcing query with already existing `DepNode`\n\
                  - query-key: {:?}\n\
                  - dep-node: {:?}",
@@ -608,12 +613,12 @@ where
         dep_node
     );
 
-    let prof_timer = tcx.profiler().query_provider();
+    let prof_timer = tcx.dep_context().profiler().query_provider();
 
     let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| {
-        tcx.start_query(job.id, diagnostics, |tcx| {
+        tcx.start_query(job.id, diagnostics, || {
             if query.eval_always {
-                tcx.dep_graph().with_eval_always_task(
+                tcx.dep_context().dep_graph().with_eval_always_task(
                     dep_node,
                     tcx,
                     key,
@@ -621,7 +626,13 @@ where
                     query.hash_result,
                 )
             } else {
-                tcx.dep_graph().with_task(dep_node, tcx, key, query.compute, query.hash_result)
+                tcx.dep_context().dep_graph().with_task(
+                    dep_node,
+                    tcx,
+                    key,
+                    query.compute,
+                    query.hash_result,
+                )
             }
         })
     });
@@ -629,7 +640,7 @@ where
     prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 
     if unlikely!(!diagnostics.is_empty()) && dep_node.kind != DepKind::NULL {
-        tcx.store_diagnostics(dep_node_index, diagnostics);
+        tcx.dep_context().store_diagnostics(dep_node_index, diagnostics);
     }
 
     let result = job.complete(result, dep_node_index);
@@ -650,7 +661,7 @@ fn get_query_impl<CTX, C>(
 where
     CTX: QueryContext,
     C: QueryCache,
-    C::Key: crate::dep_graph::DepNodeParams<CTX>,
+    C::Key: crate::dep_graph::DepNodeParams<CTX::DepContext>,
 {
     try_execute_query(tcx, state, cache, span, key, lookup, query)
 }
@@ -664,9 +675,9 @@ where
 ///
 /// Note: The optimization is only available during incr. comp.
 #[inline(never)]
-fn ensure_must_run<CTX, K, V>(tcx: CTX, key: &K, query: &QueryVtable<CTX, K, V>) -> bool
+fn ensure_must_run<CTX, K, V>(tcx: CTX::DepContext, key: &K, query: &QueryVtable<CTX, K, V>) -> bool
 where
-    K: crate::dep_graph::DepNodeParams<CTX>,
+    K: crate::dep_graph::DepNodeParams<CTX::DepContext>,
     CTX: QueryContext,
 {
     if query.eval_always {
@@ -706,14 +717,14 @@ fn force_query_impl<CTX, C>(
     query: &QueryVtable<CTX, C::Key, C::Value>,
 ) where
     C: QueryCache,
-    C::Key: crate::dep_graph::DepNodeParams<CTX>,
+    C::Key: crate::dep_graph::DepNodeParams<CTX::DepContext>,
     CTX: QueryContext,
 {
     // We may be concurrently trying both execute and force a query.
     // Ensure that only one of them runs the query.
     let cached = cache.cache.lookup(cache, &key, |_, index| {
-        if unlikely!(tcx.profiler().enabled()) {
-            tcx.profiler().query_cache_hit(index.into());
+        if unlikely!(tcx.dep_context().profiler().enabled()) {
+            tcx.dep_context().profiler().query_cache_hit(index.into());
         }
         #[cfg(debug_assertions)]
         {
@@ -751,12 +762,12 @@ pub fn get_query<Q, CTX>(
 ) -> Option<Q::Stored>
 where
     Q: QueryDescription<CTX>,
-    Q::Key: crate::dep_graph::DepNodeParams<CTX>,
+    Q::Key: crate::dep_graph::DepNodeParams<CTX::DepContext>,
     CTX: QueryContext,
 {
     let query = &Q::VTABLE;
     if let QueryMode::Ensure = mode {
-        if !ensure_must_run(tcx, &key, query) {
+        if !ensure_must_run(*tcx.dep_context(), &key, query) {
             return None;
         }
     }
@@ -770,7 +781,7 @@ where
 pub fn force_query<Q, CTX>(tcx: CTX, key: Q::Key, span: Span, dep_node: DepNode<CTX::DepKind>)
 where
     Q: QueryDescription<CTX>,
-    Q::Key: crate::dep_graph::DepNodeParams<CTX>,
+    Q::Key: crate::dep_graph::DepNodeParams<CTX::DepContext>,
     CTX: QueryContext,
 {
     force_query_impl(tcx, Q::query_state(tcx), Q::query_cache(tcx), key, span, dep_node, &Q::VTABLE)

From 152266cf887b3d532fd90633a2e261f3debbcdef Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Mon, 4 Jan 2021 23:38:20 +0100
Subject: [PATCH 12/27] Decouple the on-disk cache from the query engine.

---
 compiler/rustc_macros/src/query.rs                |  4 ++--
 compiler/rustc_middle/src/dep_graph/dep_node.rs   |  7 ++-----
 compiler/rustc_middle/src/dep_graph/mod.rs        |  9 ++++-----
 compiler/rustc_middle/src/query/mod.rs            |  7 +------
 compiler/rustc_middle/src/ty/context.rs           | 15 +++++++++++----
 .../rustc_middle/src/ty/query/on_disk_cache.rs    |  1 -
 compiler/rustc_middle/src/ty/query/plumbing.rs    |  8 --------
 7 files changed, 20 insertions(+), 31 deletions(-)

diff --git a/compiler/rustc_macros/src/query.rs b/compiler/rustc_macros/src/query.rs
index bd20c7689ea2e..48b74bdaf1eee 100644
--- a/compiler/rustc_macros/src/query.rs
+++ b/compiler/rustc_macros/src/query.rs
@@ -97,7 +97,7 @@ impl Parse for QueryModifier {
             Ok(QueryModifier::Cache(args, block))
         } else if modifier == "load_cached" {
             // Parse a load_cached modifier like:
-            // `load_cached(tcx, id) { tcx.queries.on_disk_cache.try_load_query_result(tcx, id) }`
+            // `load_cached(tcx, id) { tcx.on_disk_cache.try_load_query_result(tcx, id) }`
             let args;
             parenthesized!(args in input);
             let tcx = args.parse()?;
@@ -368,7 +368,7 @@ fn add_query_description_impl(
                     tcx: TyCtxt<'tcx>,
                     id: SerializedDepNodeIndex
                 ) -> Option<Self::Value> {
-                    tcx.queries.on_disk_cache.as_ref().and_then(|c| c.try_load_query_result(tcx, id))
+                    tcx.on_disk_cache.as_ref()?.try_load_query_result(tcx, id)
                 }
             }
         };
diff --git a/compiler/rustc_middle/src/dep_graph/dep_node.rs b/compiler/rustc_middle/src/dep_graph/dep_node.rs
index 1cb75757379c9..669d7ea8d3641 100644
--- a/compiler/rustc_middle/src/dep_graph/dep_node.rs
+++ b/compiler/rustc_middle/src/dep_graph/dep_node.rs
@@ -414,10 +414,7 @@ impl DepNodeExt for DepNode {
     /// has been removed.
     fn extract_def_id(&self, tcx: TyCtxt<'tcx>) -> Option<DefId> {
         if self.kind.can_reconstruct_query_key() {
-            tcx.queries
-                .on_disk_cache
-                .as_ref()?
-                .def_path_hash_to_def_id(tcx, DefPathHash(self.hash.into()))
+            tcx.on_disk_cache.as_ref()?.def_path_hash_to_def_id(tcx, DefPathHash(self.hash.into()))
         } else {
             None
         }
@@ -472,7 +469,7 @@ impl<'tcx> DepNodeParams<TyCtxt<'tcx>> for DefId {
         // we will use the old DefIndex as an initial guess for
         // a lookup into the crate metadata.
         if !self.is_local() {
-            if let Some(cache) = &tcx.queries.on_disk_cache {
+            if let Some(cache) = &tcx.on_disk_cache {
                 cache.store_foreign_def_id_hash(*self, hash);
             }
         }
diff --git a/compiler/rustc_middle/src/dep_graph/mod.rs b/compiler/rustc_middle/src/dep_graph/mod.rs
index eaf414ccaa69d..4746004cfca23 100644
--- a/compiler/rustc_middle/src/dep_graph/mod.rs
+++ b/compiler/rustc_middle/src/dep_graph/mod.rs
@@ -94,7 +94,7 @@ impl<'tcx> DepContext for TyCtxt<'tcx> {
     type StableHashingContext = StableHashingContext<'tcx>;
 
     fn register_reused_dep_node(&self, dep_node: &DepNode) {
-        if let Some(cache) = self.queries.on_disk_cache.as_ref() {
+        if let Some(cache) = self.on_disk_cache.as_ref() {
             cache.register_reused_dep_node(*self, dep_node)
         }
     }
@@ -185,15 +185,14 @@ impl<'tcx> DepContext for TyCtxt<'tcx> {
     }
 
     fn load_diagnostics(&self, prev_dep_node_index: SerializedDepNodeIndex) -> Vec<Diagnostic> {
-        self.queries
-            .on_disk_cache
+        self.on_disk_cache
             .as_ref()
             .map(|c| c.load_diagnostics(*self, prev_dep_node_index))
             .unwrap_or_default()
     }
 
     fn store_diagnostics(&self, dep_node_index: DepNodeIndex, diagnostics: ThinVec<Diagnostic>) {
-        if let Some(c) = self.queries.on_disk_cache.as_ref() {
+        if let Some(c) = self.on_disk_cache.as_ref() {
             c.store_diagnostics(dep_node_index, diagnostics)
         }
     }
@@ -203,7 +202,7 @@ impl<'tcx> DepContext for TyCtxt<'tcx> {
         dep_node_index: DepNodeIndex,
         diagnostics: ThinVec<Diagnostic>,
     ) {
-        if let Some(c) = self.queries.on_disk_cache.as_ref() {
+        if let Some(c) = self.on_disk_cache.as_ref() {
             c.store_diagnostics_for_anon_node(dep_node_index, diagnostics)
         }
     }
diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs
index f0166ec21672b..f60ada5bac981 100644
--- a/compiler/rustc_middle/src/query/mod.rs
+++ b/compiler/rustc_middle/src/query/mod.rs
@@ -125,11 +125,6 @@ rustc_queries! {
         desc { |tcx| "computing generics of `{}`", tcx.def_path_str(key) }
         storage(ArenaCacheSelector<'tcx>)
         cache_on_disk_if { key.is_local() }
-        load_cached(tcx, id) {
-            let generics: Option<ty::Generics> = tcx.queries.on_disk_cache.as_ref()
-                                                    .and_then(|c| c.try_load_query_result(tcx, id));
-            generics
-        }
     }
 
     /// Maps from the `DefId` of an item (trait/struct/enum/fn) to the
@@ -702,7 +697,7 @@ rustc_queries! {
         cache_on_disk_if { true }
         load_cached(tcx, id) {
             let typeck_results: Option<ty::TypeckResults<'tcx>> = tcx
-                .queries.on_disk_cache.as_ref()
+                .on_disk_cache.as_ref()
                 .and_then(|c| c.try_load_query_result(tcx, id));
 
             typeck_results.map(|x| &*tcx.arena.alloc(x))
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index 4654a8424706d..fa176193a93ee 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -14,7 +14,7 @@ use crate::middle::stability;
 use crate::mir::interpret::{self, Allocation, ConstValue, Scalar};
 use crate::mir::{Body, Field, Local, Place, PlaceElem, ProjectionKind, Promoted};
 use crate::traits;
-use crate::ty::query::{self, TyCtxtAt};
+use crate::ty::query::{self, OnDiskCache, TyCtxtAt};
 use crate::ty::subst::{GenericArg, GenericArgKind, InternalSubsts, Subst, SubstsRef, UserSubsts};
 use crate::ty::TyKind::*;
 use crate::ty::{
@@ -962,6 +962,12 @@ pub struct GlobalCtxt<'tcx> {
     pub(crate) untracked_crate: &'tcx hir::Crate<'tcx>,
     pub(crate) definitions: &'tcx Definitions,
 
+    /// This provides access to the incremental compilation on-disk cache for query results.
+    /// Do not access this directly. It is only meant to be used by
+    /// `DepGraph::try_mark_green()` and the query infrastructure.
+    /// This is `None` if we are not incremental compilation mode
+    pub(crate) on_disk_cache: Option<OnDiskCache<'tcx>>,
+
     pub queries: query::Queries<'tcx>,
     pub query_caches: query::QueryCaches<'tcx>,
 
@@ -1110,7 +1116,7 @@ impl<'tcx> TyCtxt<'tcx> {
         krate: &'tcx hir::Crate<'tcx>,
         definitions: &'tcx Definitions,
         dep_graph: DepGraph,
-        on_disk_query_result_cache: Option<query::OnDiskCache<'tcx>>,
+        on_disk_cache: Option<query::OnDiskCache<'tcx>>,
         crate_name: &str,
         output_filenames: &OutputFilenames,
     ) -> GlobalCtxt<'tcx> {
@@ -1154,7 +1160,8 @@ impl<'tcx> TyCtxt<'tcx> {
             extern_prelude: resolutions.extern_prelude,
             untracked_crate: krate,
             definitions,
-            queries: query::Queries::new(providers, extern_providers, on_disk_query_result_cache),
+            on_disk_cache,
+            queries: query::Queries::new(providers, extern_providers),
             query_caches: query::QueryCaches::default(),
             ty_rcache: Default::default(),
             pred_rcache: Default::default(),
@@ -1320,7 +1327,7 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     pub fn serialize_query_result_cache(self, encoder: &mut FileEncoder) -> FileEncodeResult {
-        self.queries.on_disk_cache.as_ref().map_or(Ok(()), |c| c.serialize(self, encoder))
+        self.on_disk_cache.as_ref().map_or(Ok(()), |c| c.serialize(self, encoder))
     }
 
     /// If `true`, we should use the MIR-based borrowck, but also
diff --git a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs
index b41edb5deeb2c..d5ed5039d320a 100644
--- a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs
+++ b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs
@@ -918,7 +918,6 @@ impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for DefId {
         // which means that the definition with this hash is guaranteed to
         // still exist in the current compilation session.
         Ok(d.tcx()
-            .queries
             .on_disk_cache
             .as_ref()
             .unwrap()
diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs
index ea889549e43da..8eb060a39a501 100644
--- a/compiler/rustc_middle/src/ty/query/plumbing.rs
+++ b/compiler/rustc_middle/src/ty/query/plumbing.rs
@@ -534,12 +534,6 @@ macro_rules! define_queries_struct {
     (tcx: $tcx:tt,
      input: ($(([$($modifiers:tt)*] [$($attr:tt)*] [$name:ident]))*)) => {
         pub struct Queries<$tcx> {
-            /// This provides access to the incremental compilation on-disk cache for query results.
-            /// Do not access this directly. It is only meant to be used by
-            /// `DepGraph::try_mark_green()` and the query infrastructure.
-            /// This is `None` if we are not incremental compilation mode
-            pub(crate) on_disk_cache: Option<OnDiskCache<'tcx>>,
-
             providers: IndexVec<CrateNum, Providers>,
             fallback_extern_providers: Box<Providers>,
 
@@ -554,12 +548,10 @@ macro_rules! define_queries_struct {
             pub(crate) fn new(
                 providers: IndexVec<CrateNum, Providers>,
                 fallback_extern_providers: Providers,
-                on_disk_cache: Option<OnDiskCache<'tcx>>,
             ) -> Self {
                 Queries {
                     providers,
                     fallback_extern_providers: Box::new(fallback_extern_providers),
-                    on_disk_cache,
                     $($name: Default::default()),*
                 }
             }

From c783ecbfab4b0f3a49eb723d72e32782d8d3718a Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Wed, 8 Apr 2020 17:03:34 +0200
Subject: [PATCH 13/27] Wrap TyCtxt inside a QueryCtxt for queries.

---
 compiler/rustc_macros/src/query.rs            | 26 ++++----
 .../rustc_middle/src/dep_graph/dep_node.rs    |  5 +-
 compiler/rustc_middle/src/query/mod.rs        |  1 +
 compiler/rustc_middle/src/ty/query/job.rs     |  5 +-
 compiler/rustc_middle/src/ty/query/mod.rs     |  1 +
 .../src/ty/query/on_disk_cache.rs             |  7 +-
 .../rustc_middle/src/ty/query/plumbing.rs     | 65 +++++++++++++------
 compiler/rustc_middle/src/ty/query/stats.rs   |  6 +-
 compiler/rustc_middle/src/ty/query/values.rs  | 15 +++--
 9 files changed, 82 insertions(+), 49 deletions(-)

diff --git a/compiler/rustc_macros/src/query.rs b/compiler/rustc_macros/src/query.rs
index 48b74bdaf1eee..1ac71824faf8b 100644
--- a/compiler/rustc_macros/src/query.rs
+++ b/compiler/rustc_macros/src/query.rs
@@ -354,9 +354,10 @@ fn add_query_description_impl(
             quote! {
                 #[inline]
                 fn try_load_from_disk(
-                    #tcx: TyCtxt<'tcx>,
-                    #id: SerializedDepNodeIndex
+                    tcx: QueryCtxt<'tcx>,
+                    id: SerializedDepNodeIndex
                 ) -> Option<Self::Value> {
+                    let (#tcx, #id) = (*tcx, id);
                     #block
                 }
             }
@@ -365,10 +366,10 @@ fn add_query_description_impl(
             quote! {
                 #[inline]
                 fn try_load_from_disk(
-                    tcx: TyCtxt<'tcx>,
+                    tcx: QueryCtxt<'tcx>,
                     id: SerializedDepNodeIndex
                 ) -> Option<Self::Value> {
-                    tcx.on_disk_cache.as_ref()?.try_load_query_result(tcx, id)
+                    tcx.on_disk_cache.as_ref()?.try_load_query_result(*tcx, id)
                 }
             }
         };
@@ -393,10 +394,11 @@ fn add_query_description_impl(
             #[inline]
             #[allow(unused_variables, unused_braces)]
             fn cache_on_disk(
-                #tcx: TyCtxt<'tcx>,
-                #key: &Self::Key,
-                #value: Option<&Self::Value>
+                tcx: QueryCtxt<'tcx>,
+                key: &Self::Key,
+                value: Option<&Self::Value>
             ) -> bool {
+                let (#tcx, #key, #value) = (*tcx, key, value);
                 #expr
             }
 
@@ -414,16 +416,14 @@ fn add_query_description_impl(
 
     let desc = quote! {
         #[allow(unused_variables)]
-        fn describe(
-            #tcx: TyCtxt<'tcx>,
-            #key: #arg,
-        ) -> String {
-            ::rustc_middle::ty::print::with_no_trimmed_paths(|| format!(#desc))
+        fn describe(tcx: QueryCtxt<'tcx>, key: #arg) -> String {
+            let (#tcx, #key) = (*tcx, key);
+            ::rustc_middle::ty::print::with_no_trimmed_paths(|| format!(#desc).into())
         }
     };
 
     impls.extend(quote! {
-        impl<'tcx> QueryDescription<TyCtxt<'tcx>> for queries::#name<'tcx> {
+        impl<'tcx> QueryDescription<QueryCtxt<'tcx>> for queries::#name<'tcx> {
             #desc
             #cache
         }
diff --git a/compiler/rustc_middle/src/dep_graph/dep_node.rs b/compiler/rustc_middle/src/dep_graph/dep_node.rs
index 669d7ea8d3641..ea1376e41c98f 100644
--- a/compiler/rustc_middle/src/dep_graph/dep_node.rs
+++ b/compiler/rustc_middle/src/dep_graph/dep_node.rs
@@ -55,6 +55,7 @@
 //!
 //! [dependency graph]: https://rustc-dev-guide.rust-lang.org/query.html
 
+use crate::ty::query::QueryCtxt;
 use crate::ty::TyCtxt;
 
 use rustc_data_structures::fingerprint::Fingerprint;
@@ -261,7 +262,7 @@ pub mod dep_kind {
 
                     if let Some(key) = recover(tcx, dep_node) {
                         force_query::<queries::$variant<'_>, _>(
-                            tcx,
+                            QueryCtxt(tcx),
                             key,
                             DUMMY_SP,
                             *dep_node
@@ -287,7 +288,7 @@ pub mod dep_kind {
                                      .unwrap_or(false));
 
                     let key = recover(tcx, dep_node).unwrap_or_else(|| panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash));
-                    if queries::$variant::cache_on_disk(tcx, &key, None) {
+                    if queries::$variant::cache_on_disk(QueryCtxt(tcx), &key, None) {
                         let _ = tcx.$variant(key);
                     }
                 }
diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs
index f60ada5bac981..77de3b216ee65 100644
--- a/compiler/rustc_middle/src/query/mod.rs
+++ b/compiler/rustc_middle/src/query/mod.rs
@@ -7,6 +7,7 @@ use crate::traits::query::{
     CanonicalTypeOpProvePredicateGoal, CanonicalTypeOpSubtypeGoal,
 };
 use crate::ty::query::queries;
+use crate::ty::query::QueryCtxt;
 use crate::ty::subst::{GenericArg, SubstsRef};
 use crate::ty::{self, ParamEnvAnd, Ty, TyCtxt};
 use rustc_hir::def_id::{CrateNum, DefId, LocalDefId};
diff --git a/compiler/rustc_middle/src/ty/query/job.rs b/compiler/rustc_middle/src/ty/query/job.rs
index bd2e7747b7db8..0ea8bcc9d9d56 100644
--- a/compiler/rustc_middle/src/ty/query/job.rs
+++ b/compiler/rustc_middle/src/ty/query/job.rs
@@ -1,3 +1,4 @@
+use crate::ty::query::QueryCtxt;
 use crate::ty::tls;
 
 use rustc_query_system::query::deadlock;
@@ -20,7 +21,7 @@ pub unsafe fn handle_deadlock() {
     thread::spawn(move || {
         tls::enter_context(icx, |_| {
             rustc_span::SESSION_GLOBALS
-                .set(session_globals, || tls::with(|tcx| deadlock(tcx, &registry)))
-        })
+                .set(session_globals, || tls::with(|tcx| deadlock(QueryCtxt(tcx), &registry)))
+        });
     });
 }
diff --git a/compiler/rustc_middle/src/ty/query/mod.rs b/compiler/rustc_middle/src/ty/query/mod.rs
index 804c045a69010..88d1efffea7cf 100644
--- a/compiler/rustc_middle/src/ty/query/mod.rs
+++ b/compiler/rustc_middle/src/ty/query/mod.rs
@@ -60,6 +60,7 @@ use std::sync::Arc;
 
 #[macro_use]
 mod plumbing;
+pub use plumbing::QueryCtxt;
 pub(crate) use rustc_query_system::query::CycleError;
 use rustc_query_system::query::*;
 
diff --git a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs
index d5ed5039d320a..49d2602f8d4aa 100644
--- a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs
+++ b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs
@@ -3,6 +3,7 @@ use crate::mir::interpret::{AllocDecodingSession, AllocDecodingState};
 use crate::mir::{self, interpret};
 use crate::ty::codec::{RefDecodable, TyDecoder, TyEncoder};
 use crate::ty::context::TyCtxt;
+use crate::ty::query::QueryCtxt;
 use crate::ty::{self, Ty};
 use rustc_data_structures::fingerprint::{Fingerprint, FingerprintDecoder, FingerprintEncoder};
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet};
@@ -312,7 +313,7 @@ impl<'sess> OnDiskCache<'sess> {
                     ($($query:ident,)*) => {
                         $(
                             encode_query_results::<ty::query::queries::$query<'_>>(
-                                tcx,
+                                QueryCtxt(tcx),
                                 enc,
                                 qri
                             )?;
@@ -1230,12 +1231,12 @@ impl<'a> Decodable<opaque::Decoder<'a>> for IntEncodedWithFixedSize {
 }
 
 fn encode_query_results<'a, 'tcx, Q>(
-    tcx: TyCtxt<'tcx>,
+    tcx: QueryCtxt<'tcx>,
     encoder: &mut CacheEncoder<'a, 'tcx, FileEncoder>,
     query_result_index: &mut EncodedQueryResultIndex,
 ) -> FileEncodeResult
 where
-    Q: super::QueryDescription<TyCtxt<'tcx>> + super::QueryAccessors<TyCtxt<'tcx>>,
+    Q: super::QueryDescription<QueryCtxt<'tcx>> + super::QueryAccessors<QueryCtxt<'tcx>>,
     Q::Value: Encodable<CacheEncoder<'a, 'tcx, FileEncoder>>,
 {
     let _timer = tcx
diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs
index 8eb060a39a501..91a267e38d5ae 100644
--- a/compiler/rustc_middle/src/ty/query/plumbing.rs
+++ b/compiler/rustc_middle/src/ty/query/plumbing.rs
@@ -5,6 +5,7 @@
 use crate::ty::query::Query;
 use crate::ty::tls::{self, ImplicitCtxt};
 use crate::ty::{self, TyCtxt};
+use rustc_query_system::dep_graph::HasDepContext;
 use rustc_query_system::query::QueryContext;
 use rustc_query_system::query::{CycleError, QueryJobId, QueryJobInfo};
 
@@ -15,7 +16,29 @@ use rustc_errors::{struct_span_err, Diagnostic, DiagnosticBuilder, Handler, Leve
 use rustc_span::def_id::DefId;
 use rustc_span::Span;
 
-impl QueryContext for TyCtxt<'tcx> {
+#[derive(Copy, Clone)]
+pub struct QueryCtxt<'tcx>(pub TyCtxt<'tcx>);
+
+impl<'tcx> std::ops::Deref for QueryCtxt<'tcx> {
+    type Target = TyCtxt<'tcx>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl HasDepContext for QueryCtxt<'tcx> {
+    type DepKind = crate::dep_graph::DepKind;
+    type StableHashingContext = crate::ich::StableHashingContext<'tcx>;
+    type DepContext = TyCtxt<'tcx>;
+
+    #[inline]
+    fn dep_context(&self) -> &Self::DepContext {
+        &self.0
+    }
+}
+
+impl QueryContext for QueryCtxt<'tcx> {
     type Query = Query<'tcx>;
 
     fn incremental_verify_ich(&self) -> bool {
@@ -26,11 +49,11 @@ impl QueryContext for TyCtxt<'tcx> {
     }
 
     fn def_path_str(&self, def_id: DefId) -> String {
-        TyCtxt::def_path_str(*self, def_id)
+        self.0.def_path_str(def_id)
     }
 
     fn current_query_job(&self) -> Option<QueryJobId<Self::DepKind>> {
-        tls::with_related_context(*self, |icx| icx.query)
+        tls::with_related_context(**self, |icx| icx.query)
     }
 
     fn try_collect_active_jobs(
@@ -53,10 +76,10 @@ impl QueryContext for TyCtxt<'tcx> {
         // The `TyCtxt` stored in TLS has the same global interner lifetime
         // as `self`, so we use `with_related_context` to relate the 'tcx lifetimes
         // when accessing the `ImplicitCtxt`.
-        tls::with_related_context(*self, move |current_icx| {
+        tls::with_related_context(**self, move |current_icx| {
             // Update the `ImplicitCtxt` to point to our new query job.
             let new_icx = ImplicitCtxt {
-                tcx: *self,
+                tcx: **self,
                 query: Some(token),
                 diagnostics,
                 layout_depth: current_icx.layout_depth,
@@ -71,7 +94,7 @@ impl QueryContext for TyCtxt<'tcx> {
     }
 }
 
-impl<'tcx> TyCtxt<'tcx> {
+impl<'tcx> QueryCtxt<'tcx> {
     #[inline(never)]
     #[cold]
     pub(super) fn report_cycle(
@@ -81,7 +104,7 @@ impl<'tcx> TyCtxt<'tcx> {
         assert!(!stack.is_empty());
 
         let fix_span = |span: Span, query: &Query<'tcx>| {
-            self.sess.source_map().guess_head_span(query.default_span(self, span))
+            self.sess.source_map().guess_head_span(query.default_span(*self, span))
         };
 
         // Disable naming impls with types in this path, since that
@@ -119,7 +142,9 @@ impl<'tcx> TyCtxt<'tcx> {
             err
         })
     }
+}
 
+impl<'tcx> TyCtxt<'tcx> {
     pub fn try_print_query_stack(handler: &Handler, num_frames: Option<usize>) {
         eprintln!("query stack during panic:");
 
@@ -149,7 +174,7 @@ impl<'tcx> TyCtxt<'tcx> {
                             "#{} [{}] {}",
                             i,
                             query_info.info.query.name(),
-                            query_info.info.query.describe(icx.tcx)
+                            query_info.info.query.describe(QueryCtxt(icx.tcx))
                         ),
                     );
                     diag.span =
@@ -272,7 +297,7 @@ macro_rules! define_queries {
                 }
             }
 
-            pub fn describe(&self, tcx: TyCtxt<$tcx>) -> String {
+            pub(crate) fn describe(&self, tcx: QueryCtxt<$tcx>) -> String {
                 let (r, name) = match *self {
                     $(Query::$name(key) => {
                         (queries::$name::describe(tcx, key), stringify!($name))
@@ -362,7 +387,7 @@ macro_rules! define_queries {
             const NAME: &'static str = stringify!($name);
         }
 
-        impl<$tcx> QueryAccessors<TyCtxt<$tcx>> for queries::$name<$tcx> {
+        impl<$tcx> QueryAccessors<QueryCtxt<$tcx>> for queries::$name<$tcx> {
             const ANON: bool = is_anon!([$($modifiers)*]);
             const EVAL_ALWAYS: bool = is_eval_always!([$($modifiers)*]);
             const DEP_KIND: dep_graph::DepKind = dep_graph::DepKind::$name;
@@ -370,19 +395,19 @@ macro_rules! define_queries {
             type Cache = query_storage::$name<$tcx>;
 
             #[inline(always)]
-            fn query_state<'a>(tcx: TyCtxt<$tcx>) -> &'a QueryState<crate::dep_graph::DepKind, Query<$tcx>, Self::Key> {
+            fn query_state<'a>(tcx: QueryCtxt<$tcx>) -> &'a QueryState<crate::dep_graph::DepKind, Query<$tcx>, Self::Key> {
                 &tcx.queries.$name
             }
 
             #[inline(always)]
-            fn query_cache<'a>(tcx: TyCtxt<$tcx>) -> &'a QueryCacheStore<Self::Cache>
+            fn query_cache<'a>(tcx: QueryCtxt<$tcx>) -> &'a QueryCacheStore<Self::Cache>
                 where 'tcx:'a
             {
                 &tcx.query_caches.$name
             }
 
             #[inline]
-            fn compute(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value {
+            fn compute(tcx: QueryCtxt<'tcx>, key: Self::Key) -> Self::Value {
                 let provider = tcx.queries.providers.get(key.query_crate())
                     // HACK(eddyb) it's possible crates may be loaded after
                     // the query engine is created, and because crate loading
@@ -390,7 +415,7 @@ macro_rules! define_queries {
                     // would be missing appropriate entries in `providers`.
                     .unwrap_or(&tcx.queries.fallback_extern_providers)
                     .$name;
-                provider(tcx, key)
+                provider(*tcx, key)
             }
 
             fn hash_result(
@@ -401,7 +426,7 @@ macro_rules! define_queries {
             }
 
             fn handle_cycle_error(
-                tcx: TyCtxt<'tcx>,
+                tcx: QueryCtxt<'tcx>,
                 error: CycleError<Query<'tcx>>
             ) -> Self::Value {
                 handle_cycle_error!([$($modifiers)*][tcx, error])
@@ -425,7 +450,8 @@ macro_rules! define_queries {
                     Err(lookup) => lookup,
                 };
 
-                get_query::<queries::$name<'_>, _>(self.tcx, DUMMY_SP, key, lookup, QueryMode::Ensure);
+                let qcx = QueryCtxt(self.tcx);
+                get_query::<queries::$name<'_>, _>(qcx, DUMMY_SP, key, lookup, QueryMode::Ensure);
             })*
         }
 
@@ -516,7 +542,8 @@ macro_rules! define_queries {
                     Err(lookup) => lookup,
                 };
 
-                get_query::<queries::$name<'_>, _>(self.tcx, self.span, key, lookup, QueryMode::Get).unwrap()
+                let qcx = QueryCtxt(self.tcx);
+                get_query::<queries::$name<'_>, _>(qcx, self.span, key, lookup, QueryMode::Get).unwrap()
             })*
         }
 
@@ -558,12 +585,12 @@ macro_rules! define_queries_struct {
 
             pub(crate) fn try_collect_active_jobs(
                 &self
-            ) -> Option<FxHashMap<QueryJobId<crate::dep_graph::DepKind>, QueryJobInfo<crate::dep_graph::DepKind, <TyCtxt<$tcx> as QueryContext>::Query>>> {
+            ) -> Option<FxHashMap<QueryJobId<crate::dep_graph::DepKind>, QueryJobInfo<crate::dep_graph::DepKind, Query<$tcx>>>> {
                 let mut jobs = FxHashMap::default();
 
                 $(
                     self.$name.try_collect_active_jobs(
-                        <queries::$name<'tcx> as QueryAccessors<TyCtxt<'tcx>>>::DEP_KIND,
+                        <queries::$name<'tcx> as QueryAccessors<QueryCtxt<'tcx>>>::DEP_KIND,
                         Query::$name,
                         &mut jobs,
                     )?;
diff --git a/compiler/rustc_middle/src/ty/query/stats.rs b/compiler/rustc_middle/src/ty/query/stats.rs
index c885a10f80595..f6fe4f1563490 100644
--- a/compiler/rustc_middle/src/ty/query/stats.rs
+++ b/compiler/rustc_middle/src/ty/query/stats.rs
@@ -1,7 +1,7 @@
-use crate::ty::query::queries;
+use crate::ty::query::query_storage;
 use crate::ty::TyCtxt;
 use rustc_hir::def_id::{DefId, LOCAL_CRATE};
-use rustc_query_system::query::{QueryAccessors, QueryCache, QueryCacheStore};
+use rustc_query_system::query::{QueryCache, QueryCacheStore};
 
 use std::any::type_name;
 use std::mem;
@@ -125,7 +125,7 @@ macro_rules! print_stats {
 
             $(
                 queries.push(stats::<
-                    <queries::$name<'_> as QueryAccessors<TyCtxt<'_>>>::Cache,
+                    query_storage::$name<'_>,
                 >(
                     stringify!($name),
                     &tcx.query_caches.$name,
diff --git a/compiler/rustc_middle/src/ty/query/values.rs b/compiler/rustc_middle/src/ty/query/values.rs
index f28b0f499f082..fa15395db4e9b 100644
--- a/compiler/rustc_middle/src/ty/query/values.rs
+++ b/compiler/rustc_middle/src/ty/query/values.rs
@@ -1,18 +1,19 @@
-use crate::ty::{self, AdtSizedConstraint, Ty, TyCtxt, TyS};
+use crate::ty::query::QueryCtxt;
+use crate::ty::{self, AdtSizedConstraint, Ty, TyS};
 
 pub(super) trait Value<'tcx>: Sized {
-    fn from_cycle_error(tcx: TyCtxt<'tcx>) -> Self;
+    fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> Self;
 }
 
 impl<'tcx, T> Value<'tcx> for T {
-    default fn from_cycle_error(tcx: TyCtxt<'tcx>) -> T {
+    default fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> T {
         tcx.sess.abort_if_errors();
         bug!("Value::from_cycle_error called without errors");
     }
 }
 
 impl<'tcx> Value<'tcx> for &'_ TyS<'_> {
-    fn from_cycle_error(tcx: TyCtxt<'tcx>) -> Self {
+    fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> Self {
         // SAFETY: This is never called when `Self` is not `Ty<'tcx>`.
         // FIXME: Represent the above fact in the trait system somehow.
         unsafe { std::mem::transmute::<Ty<'tcx>, Ty<'_>>(tcx.ty_error()) }
@@ -20,19 +21,19 @@ impl<'tcx> Value<'tcx> for &'_ TyS<'_> {
 }
 
 impl<'tcx> Value<'tcx> for ty::SymbolName<'_> {
-    fn from_cycle_error(tcx: TyCtxt<'tcx>) -> Self {
+    fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> Self {
         // SAFETY: This is never called when `Self` is not `SymbolName<'tcx>`.
         // FIXME: Represent the above fact in the trait system somehow.
         unsafe {
             std::mem::transmute::<ty::SymbolName<'tcx>, ty::SymbolName<'_>>(ty::SymbolName::new(
-                tcx, "<error>",
+                *tcx, "<error>",
             ))
         }
     }
 }
 
 impl<'tcx> Value<'tcx> for AdtSizedConstraint<'_> {
-    fn from_cycle_error(tcx: TyCtxt<'tcx>) -> Self {
+    fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> Self {
         // SAFETY: This is never called when `Self` is not `AdtSizedConstraint<'tcx>`.
         // FIXME: Represent the above fact in the trait system somehow.
         unsafe {

From 469f24b751fa8e522bd99909e5f910879d6888e2 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sun, 17 Jan 2021 14:58:34 +0100
Subject: [PATCH 14/27] Opacify query invocation.

---
 .../rustc_middle/src/ty/query/plumbing.rs     | 20 +++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs
index 91a267e38d5ae..b084fd0132749 100644
--- a/compiler/rustc_middle/src/ty/query/plumbing.rs
+++ b/compiler/rustc_middle/src/ty/query/plumbing.rs
@@ -450,8 +450,7 @@ macro_rules! define_queries {
                     Err(lookup) => lookup,
                 };
 
-                let qcx = QueryCtxt(self.tcx);
-                get_query::<queries::$name<'_>, _>(qcx, DUMMY_SP, key, lookup, QueryMode::Ensure);
+                self.tcx.queries.$name(self.tcx, DUMMY_SP, key, lookup, QueryMode::Ensure);
             })*
         }
 
@@ -542,8 +541,7 @@ macro_rules! define_queries {
                     Err(lookup) => lookup,
                 };
 
-                let qcx = QueryCtxt(self.tcx);
-                get_query::<queries::$name<'_>, _>(qcx, self.span, key, lookup, QueryMode::Get).unwrap()
+                self.tcx.queries.$name(self.tcx, self.span, key, lookup, QueryMode::Get).unwrap()
             })*
         }
 
@@ -598,6 +596,20 @@ macro_rules! define_queries_struct {
 
                 Some(jobs)
             }
+
+            $($(#[$attr])*
+            #[inline(always)]
+            fn $name(
+                &self,
+                tcx: TyCtxt<$tcx>,
+                span: Span,
+                key: query_keys::$name<$tcx>,
+                lookup: QueryLookup,
+                mode: QueryMode,
+            ) -> Option<query_stored::$name<$tcx>> {
+                let qcx = QueryCtxt(tcx);
+                get_query::<queries::$name<$tcx>, _>(qcx, span, key, lookup, mode)
+            })*
         }
     };
 }

From 8071c0a401bbdd4132b6162b564410a2e6e01df9 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sun, 11 Oct 2020 10:34:13 +0200
Subject: [PATCH 15/27] Make QueryEngine opaque to TyCtxt.

---
 Cargo.lock                                    |  1 +
 compiler/rustc_interface/Cargo.toml           |  1 +
 compiler/rustc_interface/src/passes.rs        | 26 +++++---
 compiler/rustc_interface/src/queries.rs       |  6 +-
 compiler/rustc_macros/src/query.rs            | 12 ++--
 .../rustc_middle/src/dep_graph/dep_node.rs    |  5 +-
 compiler/rustc_middle/src/query/mod.rs        |  2 +-
 compiler/rustc_middle/src/ty/context.rs       | 13 ++--
 compiler/rustc_middle/src/ty/query/job.rs     |  5 +-
 compiler/rustc_middle/src/ty/query/mod.rs     |  1 +
 .../src/ty/query/on_disk_cache.rs             | 27 ++------
 .../rustc_middle/src/ty/query/plumbing.rs     | 63 ++++++++++++++++---
 12 files changed, 99 insertions(+), 63 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index a9e65ffc39107..3bf2159c91d2f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3890,6 +3890,7 @@ dependencies = [
  "rustc_expand",
  "rustc_hir",
  "rustc_incremental",
+ "rustc_index",
  "rustc_lint",
  "rustc_metadata",
  "rustc_middle",
diff --git a/compiler/rustc_interface/Cargo.toml b/compiler/rustc_interface/Cargo.toml
index 2481a27dee795..2441e0f69eef6 100644
--- a/compiler/rustc_interface/Cargo.toml
+++ b/compiler/rustc_interface/Cargo.toml
@@ -30,6 +30,7 @@ rustc_codegen_ssa = { path = "../rustc_codegen_ssa" }
 rustc_symbol_mangling = { path = "../rustc_symbol_mangling" }
 rustc_codegen_llvm = { path = "../rustc_codegen_llvm", optional = true }
 rustc_hir = { path = "../rustc_hir" }
+rustc_index = { path = "../rustc_index" }
 rustc_metadata = { path = "../rustc_metadata" }
 rustc_mir = { path = "../rustc_mir" }
 rustc_mir_build = { path = "../rustc_mir_build" }
diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs
index 56aa3939b22dc..2728a0e8b59ea 100644
--- a/compiler/rustc_interface/src/passes.rs
+++ b/compiler/rustc_interface/src/passes.rs
@@ -15,11 +15,13 @@ use rustc_expand::base::ExtCtxt;
 use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
 use rustc_hir::definitions::Definitions;
 use rustc_hir::Crate;
+use rustc_index::vec::IndexVec;
 use rustc_lint::LintStore;
 use rustc_middle::arena::Arena;
 use rustc_middle::dep_graph::DepGraph;
 use rustc_middle::middle;
 use rustc_middle::middle::cstore::{CrateStore, MetadataLoader, MetadataLoaderDyn};
+use rustc_middle::ty::query;
 use rustc_middle::ty::query::Providers;
 use rustc_middle::ty::{self, GlobalCtxt, ResolverOutputs, TyCtxt};
 use rustc_mir as mir;
@@ -738,20 +740,18 @@ pub static DEFAULT_EXTERN_QUERY_PROVIDERS: SyncLazy<Providers> = SyncLazy::new(|
     extern_providers
 });
 
-pub struct QueryContext<'tcx>(&'tcx GlobalCtxt<'tcx>);
+pub struct QueryContext<'tcx> {
+    gcx: &'tcx GlobalCtxt<'tcx>,
+}
 
 impl<'tcx> QueryContext<'tcx> {
     pub fn enter<F, R>(&mut self, f: F) -> R
     where
         F: FnOnce(TyCtxt<'tcx>) -> R,
     {
-        let icx = ty::tls::ImplicitCtxt::new(self.0);
+        let icx = ty::tls::ImplicitCtxt::new(self.gcx);
         ty::tls::enter_context(&icx, |_| f(icx.tcx))
     }
-
-    pub fn print_stats(&mut self) {
-        self.enter(ty::query::print_stats)
-    }
 }
 
 pub fn create_global_ctxt<'tcx>(
@@ -762,6 +762,7 @@ pub fn create_global_ctxt<'tcx>(
     mut resolver_outputs: ResolverOutputs,
     outputs: OutputFilenames,
     crate_name: &str,
+    queries: &'tcx OnceCell<query::Queries<'tcx>>,
     global_ctxt: &'tcx OnceCell<GlobalCtxt<'tcx>>,
     arena: &'tcx WorkerLocal<Arena<'tcx>>,
 ) -> QueryContext<'tcx> {
@@ -785,26 +786,33 @@ pub fn create_global_ctxt<'tcx>(
         callback(sess, &mut local_providers, &mut extern_providers);
     }
 
+    let queries = {
+        let crates = resolver_outputs.cstore.crates_untracked();
+        let max_cnum = crates.iter().map(|c| c.as_usize()).max().unwrap_or(0);
+        let mut providers = IndexVec::from_elem_n(extern_providers, max_cnum + 1);
+        providers[LOCAL_CRATE] = local_providers;
+        queries.get_or_init(|| query::Queries::new(providers, extern_providers))
+    };
+
     let gcx = sess.time("setup_global_ctxt", || {
         global_ctxt.get_or_init(|| {
             TyCtxt::create_global_ctxt(
                 sess,
                 lint_store,
-                local_providers,
-                extern_providers,
                 arena,
                 resolver_outputs,
                 krate,
                 defs,
                 dep_graph,
                 query_result_on_disk_cache,
+                queries,
                 &crate_name,
                 &outputs,
             )
         })
     });
 
-    QueryContext(gcx)
+    QueryContext { gcx }
 }
 
 /// Runs the resolution, type-checking, region checking and other
diff --git a/compiler/rustc_interface/src/queries.rs b/compiler/rustc_interface/src/queries.rs
index ac6b6d0311545..86d78d7e9ca3a 100644
--- a/compiler/rustc_interface/src/queries.rs
+++ b/compiler/rustc_interface/src/queries.rs
@@ -13,6 +13,7 @@ use rustc_incremental::DepGraphFuture;
 use rustc_lint::LintStore;
 use rustc_middle::arena::Arena;
 use rustc_middle::dep_graph::DepGraph;
+use rustc_middle::ty::query;
 use rustc_middle::ty::{GlobalCtxt, ResolverOutputs, TyCtxt};
 use rustc_serialize::json;
 use rustc_session::config::{self, OutputFilenames, OutputType};
@@ -71,6 +72,7 @@ impl<T> Default for Query<T> {
 pub struct Queries<'tcx> {
     compiler: &'tcx Compiler,
     gcx: OnceCell<GlobalCtxt<'tcx>>,
+    queries: OnceCell<query::Queries<'tcx>>,
 
     arena: WorkerLocal<Arena<'tcx>>,
     hir_arena: WorkerLocal<rustc_ast_lowering::Arena<'tcx>>,
@@ -92,6 +94,7 @@ impl<'tcx> Queries<'tcx> {
         Queries {
             compiler,
             gcx: OnceCell::new(),
+            queries: OnceCell::new(),
             arena: WorkerLocal::new(|_| Arena::default()),
             hir_arena: WorkerLocal::new(|_| rustc_ast_lowering::Arena::default()),
             dep_graph_future: Default::default(),
@@ -265,6 +268,7 @@ impl<'tcx> Queries<'tcx> {
                 resolver_outputs.steal(),
                 outputs,
                 &crate_name,
+                &self.queries,
                 &self.gcx,
                 &self.arena,
             ))
@@ -429,7 +433,7 @@ impl Compiler {
             }
 
             if self.session().opts.debugging_opts.query_stats {
-                gcx.print_stats();
+                gcx.enter(query::print_stats);
             }
         }
 
diff --git a/compiler/rustc_macros/src/query.rs b/compiler/rustc_macros/src/query.rs
index 1ac71824faf8b..e387020d82839 100644
--- a/compiler/rustc_macros/src/query.rs
+++ b/compiler/rustc_macros/src/query.rs
@@ -354,10 +354,9 @@ fn add_query_description_impl(
             quote! {
                 #[inline]
                 fn try_load_from_disk(
-                    tcx: QueryCtxt<'tcx>,
-                    id: SerializedDepNodeIndex
+                    #tcx: QueryCtxt<'tcx>,
+                    #id: SerializedDepNodeIndex
                 ) -> Option<Self::Value> {
-                    let (#tcx, #id) = (*tcx, id);
                     #block
                 }
             }
@@ -394,11 +393,10 @@ fn add_query_description_impl(
             #[inline]
             #[allow(unused_variables, unused_braces)]
             fn cache_on_disk(
-                tcx: QueryCtxt<'tcx>,
-                key: &Self::Key,
-                value: Option<&Self::Value>
+                #tcx: QueryCtxt<'tcx>,
+                #key: &Self::Key,
+                #value: Option<&Self::Value>
             ) -> bool {
-                let (#tcx, #key, #value) = (*tcx, key, value);
                 #expr
             }
 
diff --git a/compiler/rustc_middle/src/dep_graph/dep_node.rs b/compiler/rustc_middle/src/dep_graph/dep_node.rs
index ea1376e41c98f..94a17f4645ac3 100644
--- a/compiler/rustc_middle/src/dep_graph/dep_node.rs
+++ b/compiler/rustc_middle/src/dep_graph/dep_node.rs
@@ -262,7 +262,7 @@ pub mod dep_kind {
 
                     if let Some(key) = recover(tcx, dep_node) {
                         force_query::<queries::$variant<'_>, _>(
-                            QueryCtxt(tcx),
+                            QueryCtxt { tcx, queries: tcx.queries },
                             key,
                             DUMMY_SP,
                             *dep_node
@@ -288,7 +288,8 @@ pub mod dep_kind {
                                      .unwrap_or(false));
 
                     let key = recover(tcx, dep_node).unwrap_or_else(|| panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash));
-                    if queries::$variant::cache_on_disk(QueryCtxt(tcx), &key, None) {
+                    let qcx = QueryCtxt { tcx, queries: tcx.queries };
+                    if queries::$variant::cache_on_disk(qcx, &key, None) {
                         let _ = tcx.$variant(key);
                     }
                 }
diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs
index 77de3b216ee65..9162003267117 100644
--- a/compiler/rustc_middle/src/query/mod.rs
+++ b/compiler/rustc_middle/src/query/mod.rs
@@ -699,7 +699,7 @@ rustc_queries! {
         load_cached(tcx, id) {
             let typeck_results: Option<ty::TypeckResults<'tcx>> = tcx
                 .on_disk_cache.as_ref()
-                .and_then(|c| c.try_load_query_result(tcx, id));
+                .and_then(|c| c.try_load_query_result(*tcx, id));
 
             typeck_results.map(|x| &*tcx.arena.alloc(x))
         }
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index fa176193a93ee..ec680eeb30f06 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -14,7 +14,7 @@ use crate::middle::stability;
 use crate::mir::interpret::{self, Allocation, ConstValue, Scalar};
 use crate::mir::{Body, Field, Local, Place, PlaceElem, ProjectionKind, Promoted};
 use crate::traits;
-use crate::ty::query::{self, OnDiskCache, TyCtxtAt};
+use crate::ty::query::{self, OnDiskCache, Queries, TyCtxtAt};
 use crate::ty::subst::{GenericArg, GenericArgKind, InternalSubsts, Subst, SubstsRef, UserSubsts};
 use crate::ty::TyKind::*;
 use crate::ty::{
@@ -968,7 +968,7 @@ pub struct GlobalCtxt<'tcx> {
     /// This is `None` if we are not incremental compilation mode
     pub(crate) on_disk_cache: Option<OnDiskCache<'tcx>>,
 
-    pub queries: query::Queries<'tcx>,
+    pub queries: &'tcx query::Queries<'tcx>,
     pub query_caches: query::QueryCaches<'tcx>,
 
     maybe_unused_trait_imports: FxHashSet<LocalDefId>,
@@ -1109,14 +1109,13 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn create_global_ctxt(
         s: &'tcx Session,
         lint_store: Lrc<dyn Any + sync::Send + sync::Sync>,
-        local_providers: ty::query::Providers,
-        extern_providers: ty::query::Providers,
         arena: &'tcx WorkerLocal<Arena<'tcx>>,
         resolutions: ty::ResolverOutputs,
         krate: &'tcx hir::Crate<'tcx>,
         definitions: &'tcx Definitions,
         dep_graph: DepGraph,
         on_disk_cache: Option<query::OnDiskCache<'tcx>>,
+        queries: &'tcx Queries<'tcx>,
         crate_name: &str,
         output_filenames: &OutputFilenames,
     ) -> GlobalCtxt<'tcx> {
@@ -1128,10 +1127,6 @@ impl<'tcx> TyCtxt<'tcx> {
         let common_lifetimes = CommonLifetimes::new(&interners);
         let common_consts = CommonConsts::new(&interners, &common_types);
         let cstore = resolutions.cstore;
-        let crates = cstore.crates_untracked();
-        let max_cnum = crates.iter().map(|c| c.as_usize()).max().unwrap_or(0);
-        let mut providers = IndexVec::from_elem_n(extern_providers, max_cnum + 1);
-        providers[LOCAL_CRATE] = local_providers;
 
         let mut trait_map: FxHashMap<_, FxHashMap<_, _>> = FxHashMap::default();
         for (hir_id, v) in krate.trait_map.iter() {
@@ -1161,7 +1156,7 @@ impl<'tcx> TyCtxt<'tcx> {
             untracked_crate: krate,
             definitions,
             on_disk_cache,
-            queries: query::Queries::new(providers, extern_providers),
+            queries,
             query_caches: query::QueryCaches::default(),
             ty_rcache: Default::default(),
             pred_rcache: Default::default(),
diff --git a/compiler/rustc_middle/src/ty/query/job.rs b/compiler/rustc_middle/src/ty/query/job.rs
index 0ea8bcc9d9d56..668109693a8b4 100644
--- a/compiler/rustc_middle/src/ty/query/job.rs
+++ b/compiler/rustc_middle/src/ty/query/job.rs
@@ -1,7 +1,4 @@
-use crate::ty::query::QueryCtxt;
 use crate::ty::tls;
-
-use rustc_query_system::query::deadlock;
 use rustc_rayon_core as rayon_core;
 use std::thread;
 
@@ -21,7 +18,7 @@ pub unsafe fn handle_deadlock() {
     thread::spawn(move || {
         tls::enter_context(icx, |_| {
             rustc_span::SESSION_GLOBALS
-                .set(session_globals, || tls::with(|tcx| deadlock(QueryCtxt(tcx), &registry)))
+                .set(session_globals, || tls::with(|tcx| tcx.queries.deadlock(tcx, &registry)))
         });
     });
 }
diff --git a/compiler/rustc_middle/src/ty/query/mod.rs b/compiler/rustc_middle/src/ty/query/mod.rs
index 88d1efffea7cf..4c0b5846741d7 100644
--- a/compiler/rustc_middle/src/ty/query/mod.rs
+++ b/compiler/rustc_middle/src/ty/query/mod.rs
@@ -44,6 +44,7 @@ use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, DefIdSet, LocalDefId};
 use rustc_hir::lang_items::{LangItem, LanguageItems};
 use rustc_hir::{Crate, ItemLocalId, TraitCandidate};
 use rustc_index::{bit_set::FiniteBitSet, vec::IndexVec};
+use rustc_serialize::opaque;
 use rustc_session::config::{EntryFnType, OptLevel, OutputFilenames, SymbolManglingVersion};
 use rustc_session::utils::NativeLibKind;
 use rustc_session::CrateDisambiguator;
diff --git a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs
index 49d2602f8d4aa..cffb42b8ba737 100644
--- a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs
+++ b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs
@@ -133,7 +133,7 @@ struct Footer {
     foreign_def_path_hashes: UnhashMap<DefPathHash, RawDefId>,
 }
 
-type EncodedQueryResultIndex = Vec<(SerializedDepNodeIndex, AbsoluteBytePos)>;
+pub type EncodedQueryResultIndex = Vec<(SerializedDepNodeIndex, AbsoluteBytePos)>;
 type EncodedDiagnosticsIndex = Vec<(SerializedDepNodeIndex, AbsoluteBytePos)>;
 type EncodedDiagnostics = Vec<Diagnostic>;
 
@@ -141,7 +141,7 @@ type EncodedDiagnostics = Vec<Diagnostic>;
 struct SourceFileIndex(u32);
 
 #[derive(Copy, Clone, Debug, Hash, Eq, PartialEq, Encodable, Decodable)]
-struct AbsoluteBytePos(u32);
+pub struct AbsoluteBytePos(u32);
 
 impl AbsoluteBytePos {
     fn new(pos: usize) -> AbsoluteBytePos {
@@ -308,22 +308,7 @@ impl<'sess> OnDiskCache<'sess> {
             tcx.sess.time("encode_query_results", || -> FileEncodeResult {
                 let enc = &mut encoder;
                 let qri = &mut query_result_index;
-
-                macro_rules! encode_queries {
-                    ($($query:ident,)*) => {
-                        $(
-                            encode_query_results::<ty::query::queries::$query<'_>>(
-                                QueryCtxt(tcx),
-                                enc,
-                                qri
-                            )?;
-                        )*
-                    }
-                }
-
-                rustc_cached_queries!(encode_queries!);
-
-                Ok(())
+                tcx.queries.encode_query_results(tcx, enc, qri)
             })?;
 
             // Encode diagnostics.
@@ -973,7 +958,7 @@ impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for &'tcx [Span] {
 
 //- ENCODING -------------------------------------------------------------------
 
-trait OpaqueEncoder: Encoder {
+pub trait OpaqueEncoder: Encoder {
     fn position(&self) -> usize;
 }
 
@@ -985,7 +970,7 @@ impl OpaqueEncoder for FileEncoder {
 }
 
 /// An encoder that can write to the incremental compilation cache.
-struct CacheEncoder<'a, 'tcx, E: OpaqueEncoder> {
+pub struct CacheEncoder<'a, 'tcx, E: OpaqueEncoder> {
     tcx: TyCtxt<'tcx>,
     encoder: &'a mut E,
     type_shorthands: FxHashMap<Ty<'tcx>, usize>,
@@ -1230,7 +1215,7 @@ impl<'a> Decodable<opaque::Decoder<'a>> for IntEncodedWithFixedSize {
     }
 }
 
-fn encode_query_results<'a, 'tcx, Q>(
+pub fn encode_query_results<'a, 'tcx, Q>(
     tcx: QueryCtxt<'tcx>,
     encoder: &mut CacheEncoder<'a, 'tcx, FileEncoder>,
     query_result_index: &mut EncodedQueryResultIndex,
diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs
index b084fd0132749..1d4be7700059f 100644
--- a/compiler/rustc_middle/src/ty/query/plumbing.rs
+++ b/compiler/rustc_middle/src/ty/query/plumbing.rs
@@ -2,7 +2,7 @@
 //! generate the actual methods on tcx which find and execute the provider,
 //! manage the caches, and so forth.
 
-use crate::ty::query::Query;
+use crate::ty::query::{on_disk_cache, Query};
 use crate::ty::tls::{self, ImplicitCtxt};
 use crate::ty::{self, TyCtxt};
 use rustc_query_system::dep_graph::HasDepContext;
@@ -13,17 +13,21 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::Lock;
 use rustc_data_structures::thin_vec::ThinVec;
 use rustc_errors::{struct_span_err, Diagnostic, DiagnosticBuilder, Handler, Level};
+use rustc_serialize::opaque;
 use rustc_span::def_id::DefId;
 use rustc_span::Span;
 
 #[derive(Copy, Clone)]
-pub struct QueryCtxt<'tcx>(pub TyCtxt<'tcx>);
+pub struct QueryCtxt<'tcx> {
+    pub tcx: TyCtxt<'tcx>,
+    pub queries: &'tcx super::Queries<'tcx>,
+}
 
 impl<'tcx> std::ops::Deref for QueryCtxt<'tcx> {
     type Target = TyCtxt<'tcx>;
 
     fn deref(&self) -> &Self::Target {
-        &self.0
+        &self.tcx
     }
 }
 
@@ -34,7 +38,7 @@ impl HasDepContext for QueryCtxt<'tcx> {
 
     #[inline]
     fn dep_context(&self) -> &Self::DepContext {
-        &self.0
+        &self.tcx
     }
 }
 
@@ -49,7 +53,7 @@ impl QueryContext for QueryCtxt<'tcx> {
     }
 
     fn def_path_str(&self, def_id: DefId) -> String {
-        self.0.def_path_str(def_id)
+        self.tcx.def_path_str(def_id)
     }
 
     fn current_query_job(&self) -> Option<QueryJobId<Self::DepKind>> {
@@ -142,6 +146,28 @@ impl<'tcx> QueryCtxt<'tcx> {
             err
         })
     }
+
+    pub(super) fn encode_query_results(
+        self,
+        encoder: &mut on_disk_cache::CacheEncoder<'a, 'tcx, opaque::FileEncoder>,
+        query_result_index: &mut on_disk_cache::EncodedQueryResultIndex,
+    ) -> opaque::FileEncodeResult {
+        macro_rules! encode_queries {
+            ($($query:ident,)*) => {
+                $(
+                    on_disk_cache::encode_query_results::<ty::query::queries::$query<'_>>(
+                        self,
+                        encoder,
+                        query_result_index
+                    )?;
+                )*
+            }
+        }
+
+        rustc_cached_queries!(encode_queries!);
+
+        Ok(())
+    }
 }
 
 impl<'tcx> TyCtxt<'tcx> {
@@ -174,7 +200,10 @@ impl<'tcx> TyCtxt<'tcx> {
                             "#{} [{}] {}",
                             i,
                             query_info.info.query.name(),
-                            query_info.info.query.describe(QueryCtxt(icx.tcx))
+                            query_info
+                                .info
+                                .query
+                                .describe(QueryCtxt { tcx: icx.tcx, queries: icx.tcx.queries })
                         ),
                     );
                     diag.span =
@@ -570,7 +599,7 @@ macro_rules! define_queries_struct {
         }
 
         impl<$tcx> Queries<$tcx> {
-            pub(crate) fn new(
+            pub fn new(
                 providers: IndexVec<CrateNum, Providers>,
                 fallback_extern_providers: Providers,
             ) -> Self {
@@ -597,17 +626,33 @@ macro_rules! define_queries_struct {
                 Some(jobs)
             }
 
+            #[cfg(parallel_compiler)]
+            unsafe fn deadlock(&'tcx self, tcx: TyCtxt<'tcx>, registry: &rustc_rayon_core::Registry) {
+                let tcx = QueryCtxt { tcx, queries: self };
+                rustc_query_system::query::deadlock(tcx, registry)
+            }
+
+            pub(crate) fn encode_query_results(
+                &'tcx self,
+                tcx: TyCtxt<'tcx>,
+                encoder: &mut on_disk_cache::CacheEncoder<'a, 'tcx, opaque::FileEncoder>,
+                query_result_index: &mut on_disk_cache::EncodedQueryResultIndex,
+            ) -> opaque::FileEncodeResult {
+                let tcx = QueryCtxt { tcx, queries: self };
+                tcx.encode_query_results(encoder, query_result_index)
+            }
+
             $($(#[$attr])*
             #[inline(always)]
             fn $name(
-                &self,
+                &'tcx self,
                 tcx: TyCtxt<$tcx>,
                 span: Span,
                 key: query_keys::$name<$tcx>,
                 lookup: QueryLookup,
                 mode: QueryMode,
             ) -> Option<query_stored::$name<$tcx>> {
-                let qcx = QueryCtxt(tcx);
+                let qcx = QueryCtxt { tcx, queries: self };
                 get_query::<queries::$name<$tcx>, _>(qcx, span, key, lookup, mode)
             })*
         }

From edf96873a3c8f9cccba310fea0b9688c313f2b31 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sun, 17 Jan 2021 14:57:34 +0100
Subject: [PATCH 16/27] Make alloc_self_profile_query_strings a standalone
 function.

---
 compiler/rustc_interface/src/queries.rs       |  2 +-
 compiler/rustc_middle/src/ty/query/mod.rs     |  2 +-
 .../rustc_middle/src/ty/query/plumbing.rs     | 29 ------------
 .../src/ty/query/profiling_support.rs         | 46 ++++++++++++++++---
 4 files changed, 41 insertions(+), 38 deletions(-)

diff --git a/compiler/rustc_interface/src/queries.rs b/compiler/rustc_interface/src/queries.rs
index 86d78d7e9ca3a..f70701257e242 100644
--- a/compiler/rustc_interface/src/queries.rs
+++ b/compiler/rustc_interface/src/queries.rs
@@ -429,7 +429,7 @@ impl Compiler {
             {
                 let _prof_timer =
                     queries.session().prof.generic_activity("self_profile_alloc_query_strings");
-                gcx.enter(|tcx| tcx.alloc_self_profile_query_strings());
+                gcx.enter(query::alloc_self_profile_query_strings);
             }
 
             if self.session().opts.debugging_opts.query_stats {
diff --git a/compiler/rustc_middle/src/ty/query/mod.rs b/compiler/rustc_middle/src/ty/query/mod.rs
index 4c0b5846741d7..dd0723f200f76 100644
--- a/compiler/rustc_middle/src/ty/query/mod.rs
+++ b/compiler/rustc_middle/src/ty/query/mod.rs
@@ -88,7 +88,7 @@ mod on_disk_cache;
 pub use self::on_disk_cache::OnDiskCache;
 
 mod profiling_support;
-pub use self::profiling_support::{IntoSelfProfilingString, QueryKeyStringBuilder};
+pub use self::profiling_support::alloc_self_profile_query_strings;
 
 // Each of these queries corresponds to a function pointer field in the
 // `Providers` struct for requesting a value of that type, and a method
diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs
index 1d4be7700059f..a17fc71fd5c19 100644
--- a/compiler/rustc_middle/src/ty/query/plumbing.rs
+++ b/compiler/rustc_middle/src/ty/query/plumbing.rs
@@ -524,35 +524,6 @@ macro_rules! define_queries {
             {
                 self.at(DUMMY_SP).$name(key)
             })*
-
-            /// All self-profiling events generated by the query engine use
-            /// virtual `StringId`s for their `event_id`. This method makes all
-            /// those virtual `StringId`s point to actual strings.
-            ///
-            /// If we are recording only summary data, the ids will point to
-            /// just the query names. If we are recording query keys too, we
-            /// allocate the corresponding strings here.
-            pub fn alloc_self_profile_query_strings(self) {
-                use crate::ty::query::profiling_support::{
-                    alloc_self_profile_query_strings_for_query_cache,
-                    QueryKeyStringCache,
-                };
-
-                if !self.prof.enabled() {
-                    return;
-                }
-
-                let mut string_cache = QueryKeyStringCache::new();
-
-                $({
-                    alloc_self_profile_query_strings_for_query_cache(
-                        self,
-                        stringify!($name),
-                        &self.query_caches.$name,
-                        &mut string_cache,
-                    );
-                })*
-            }
         }
 
         impl TyCtxtAt<$tcx> {
diff --git a/compiler/rustc_middle/src/ty/query/profiling_support.rs b/compiler/rustc_middle/src/ty/query/profiling_support.rs
index 9976e7885090c..aab5fd9e8a194 100644
--- a/compiler/rustc_middle/src/ty/query/profiling_support.rs
+++ b/compiler/rustc_middle/src/ty/query/profiling_support.rs
@@ -9,24 +9,24 @@ use rustc_query_system::query::{QueryCache, QueryCacheStore};
 use std::fmt::Debug;
 use std::io::Write;
 
-pub struct QueryKeyStringCache {
+struct QueryKeyStringCache {
     def_id_cache: FxHashMap<DefId, StringId>,
 }
 
 impl QueryKeyStringCache {
-    pub fn new() -> QueryKeyStringCache {
+    fn new() -> QueryKeyStringCache {
         QueryKeyStringCache { def_id_cache: Default::default() }
     }
 }
 
-pub struct QueryKeyStringBuilder<'p, 'c, 'tcx> {
+struct QueryKeyStringBuilder<'p, 'c, 'tcx> {
     profiler: &'p SelfProfiler,
     tcx: TyCtxt<'tcx>,
     string_cache: &'c mut QueryKeyStringCache,
 }
 
 impl<'p, 'c, 'tcx> QueryKeyStringBuilder<'p, 'c, 'tcx> {
-    pub fn new(
+    fn new(
         profiler: &'p SelfProfiler,
         tcx: TyCtxt<'tcx>,
         string_cache: &'c mut QueryKeyStringCache,
@@ -98,7 +98,7 @@ impl<'p, 'c, 'tcx> QueryKeyStringBuilder<'p, 'c, 'tcx> {
     }
 }
 
-pub trait IntoSelfProfilingString {
+trait IntoSelfProfilingString {
     fn to_self_profile_string(&self, builder: &mut QueryKeyStringBuilder<'_, '_, '_>) -> StringId;
 }
 
@@ -123,7 +123,7 @@ impl<T: SpecIntoSelfProfilingString> IntoSelfProfilingString for T {
 }
 
 #[rustc_specialization_trait]
-pub trait SpecIntoSelfProfilingString: Debug {
+trait SpecIntoSelfProfilingString: Debug {
     fn spec_to_self_profile_string(
         &self,
         builder: &mut QueryKeyStringBuilder<'_, '_, '_>,
@@ -227,7 +227,7 @@ where
 /// Allocate the self-profiling query strings for a single query cache. This
 /// method is called from `alloc_self_profile_query_strings` which knows all
 /// the queries via macro magic.
-pub(super) fn alloc_self_profile_query_strings_for_query_cache<'tcx, C>(
+fn alloc_self_profile_query_strings_for_query_cache<'tcx, C>(
     tcx: TyCtxt<'tcx>,
     query_name: &'static str,
     query_cache: &QueryCacheStore<C>,
@@ -287,3 +287,35 @@ pub(super) fn alloc_self_profile_query_strings_for_query_cache<'tcx, C>(
         }
     });
 }
+
+/// All self-profiling events generated by the query engine use
+/// virtual `StringId`s for their `event_id`. This method makes all
+/// those virtual `StringId`s point to actual strings.
+///
+/// If we are recording only summary data, the ids will point to
+/// just the query names. If we are recording query keys too, we
+/// allocate the corresponding strings here.
+pub fn alloc_self_profile_query_strings(tcx: TyCtxt<'tcx>) {
+    if !tcx.prof.enabled() {
+        return;
+    }
+
+    let mut string_cache = QueryKeyStringCache::new();
+
+    macro_rules! alloc_once {
+        (<$tcx:tt>
+            $($(#[$attr:meta])* [$($modifiers:tt)*] fn $name:ident($K:ty) -> $V:ty,)*
+        ) => {
+            $({
+                alloc_self_profile_query_strings_for_query_cache(
+                    tcx,
+                    stringify!($name),
+                    &tcx.query_caches.$name,
+                    &mut string_cache,
+                );
+            })*
+        }
+    }
+
+    rustc_query_append! { [alloc_once!][<'tcx>] }
+}

From adbba4bdd98c11b9b206cc0177da2be818d92e96 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Wed, 8 Apr 2020 20:47:36 +0200
Subject: [PATCH 17/27] Move try_print_query_stack to rustc_interface.

---
 compiler/rustc_driver/src/lib.rs              |  3 +-
 compiler/rustc_interface/src/interface.rs     | 23 ++++-
 .../rustc_middle/src/ty/query/plumbing.rs     | 86 ++++++++-----------
 src/tools/clippy/src/driver.rs                |  4 +-
 4 files changed, 62 insertions(+), 54 deletions(-)

diff --git a/compiler/rustc_driver/src/lib.rs b/compiler/rustc_driver/src/lib.rs
index 8295e88f75ac7..a80b6090cafac 100644
--- a/compiler/rustc_driver/src/lib.rs
+++ b/compiler/rustc_driver/src/lib.rs
@@ -27,7 +27,6 @@ use rustc_interface::{interface, Queries};
 use rustc_lint::LintStore;
 use rustc_metadata::locator;
 use rustc_middle::middle::cstore::MetadataLoader;
-use rustc_middle::ty::TyCtxt;
 use rustc_save_analysis as save;
 use rustc_save_analysis::DumpHandler;
 use rustc_serialize::json::{self, ToJson};
@@ -1240,7 +1239,7 @@ pub fn report_ice(info: &panic::PanicInfo<'_>, bug_report_url: &str) {
 
     let num_frames = if backtrace { None } else { Some(2) };
 
-    TyCtxt::try_print_query_stack(&handler, num_frames);
+    interface::try_print_query_stack(&handler, num_frames);
 
     #[cfg(windows)]
     unsafe {
diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs
index 28eb1fed6a0ab..502e7155c2e0d 100644
--- a/compiler/rustc_interface/src/interface.rs
+++ b/compiler/rustc_interface/src/interface.rs
@@ -8,7 +8,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::sync::Lrc;
 use rustc_data_structures::OnDrop;
 use rustc_errors::registry::Registry;
-use rustc_errors::ErrorReported;
+use rustc_errors::{ErrorReported, Handler};
 use rustc_lint::LintStore;
 use rustc_middle::ty;
 use rustc_parse::new_parser_from_source_str;
@@ -213,3 +213,24 @@ pub fn run_compiler<R: Send>(mut config: Config, f: impl FnOnce(&Compiler) -> R
         || create_compiler_and_run(config, f),
     )
 }
+
+pub fn try_print_query_stack(handler: &Handler, num_frames: Option<usize>) {
+    eprintln!("query stack during panic:");
+
+    // Be careful relying on global state here: this code is called from
+    // a panic hook, which means that the global `Handler` may be in a weird
+    // state if it was responsible for triggering the panic.
+    let i = ty::tls::with_context_opt(|icx| {
+        if let Some(icx) = icx {
+            icx.tcx.queries.try_print_query_stack(icx.tcx, icx.query, handler, num_frames)
+        } else {
+            0
+        }
+    });
+
+    if num_frames == None || num_frames >= Some(i) {
+        eprintln!("end of query stack");
+    } else {
+        eprintln!("we're just showing a limited slice of the query stack");
+    }
+}
diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs
index a17fc71fd5c19..6aa9d88798fa7 100644
--- a/compiler/rustc_middle/src/ty/query/plumbing.rs
+++ b/compiler/rustc_middle/src/ty/query/plumbing.rs
@@ -2,7 +2,8 @@
 //! generate the actual methods on tcx which find and execute the provider,
 //! manage the caches, and so forth.
 
-use crate::ty::query::{on_disk_cache, Query};
+use crate::dep_graph;
+use crate::ty::query::{on_disk_cache, Queries, Query};
 use crate::ty::tls::{self, ImplicitCtxt};
 use crate::ty::{self, TyCtxt};
 use rustc_query_system::dep_graph::HasDepContext;
@@ -170,57 +171,46 @@ impl<'tcx> QueryCtxt<'tcx> {
     }
 }
 
-impl<'tcx> TyCtxt<'tcx> {
-    pub fn try_print_query_stack(handler: &Handler, num_frames: Option<usize>) {
-        eprintln!("query stack during panic:");
-
-        // Be careful relying on global state here: this code is called from
-        // a panic hook, which means that the global `Handler` may be in a weird
-        // state if it was responsible for triggering the panic.
+impl<'tcx> Queries<'tcx> {
+    pub fn try_print_query_stack(
+        &'tcx self,
+        tcx: TyCtxt<'tcx>,
+        query: Option<QueryJobId<dep_graph::DepKind>>,
+        handler: &Handler,
+        num_frames: Option<usize>,
+    ) -> usize {
+        let query_map = self.try_collect_active_jobs();
+
+        let mut current_query = query;
         let mut i = 0;
-        ty::tls::with_context_opt(|icx| {
-            if let Some(icx) = icx {
-                let query_map = icx.tcx.queries.try_collect_active_jobs();
-
-                let mut current_query = icx.query;
-
-                while let Some(query) = current_query {
-                    if Some(i) == num_frames {
-                        break;
-                    }
-                    let query_info =
-                        if let Some(info) = query_map.as_ref().and_then(|map| map.get(&query)) {
-                            info
-                        } else {
-                            break;
-                        };
-                    let mut diag = Diagnostic::new(
-                        Level::FailureNote,
-                        &format!(
-                            "#{} [{}] {}",
-                            i,
-                            query_info.info.query.name(),
-                            query_info
-                                .info
-                                .query
-                                .describe(QueryCtxt { tcx: icx.tcx, queries: icx.tcx.queries })
-                        ),
-                    );
-                    diag.span =
-                        icx.tcx.sess.source_map().guess_head_span(query_info.info.span).into();
-                    handler.force_print_diagnostic(diag);
-
-                    current_query = query_info.job.parent;
-                    i += 1;
-                }
+
+        while let Some(query) = current_query {
+            if Some(i) == num_frames {
+                break;
             }
-        });
+            let query_info = if let Some(info) = query_map.as_ref().and_then(|map| map.get(&query))
+            {
+                info
+            } else {
+                break;
+            };
+            let mut diag = Diagnostic::new(
+                Level::FailureNote,
+                &format!(
+                    "#{} [{}] {}",
+                    i,
+                    query_info.info.query.name(),
+                    query_info.info.query.describe(QueryCtxt { tcx, queries: self })
+                ),
+            );
+            diag.span = tcx.sess.source_map().guess_head_span(query_info.info.span).into();
+            handler.force_print_diagnostic(diag);
 
-        if num_frames == None || num_frames >= Some(i) {
-            eprintln!("end of query stack");
-        } else {
-            eprintln!("we're just showing a limited slice of the query stack");
+            current_query = query_info.job.parent;
+            i += 1;
         }
+
+        i
     }
 }
 
diff --git a/src/tools/clippy/src/driver.rs b/src/tools/clippy/src/driver.rs
index f5f6c09ed8e94..d5143e1438ee0 100644
--- a/src/tools/clippy/src/driver.rs
+++ b/src/tools/clippy/src/driver.rs
@@ -11,10 +11,8 @@
 extern crate rustc_driver;
 extern crate rustc_errors;
 extern crate rustc_interface;
-extern crate rustc_middle;
 
 use rustc_interface::interface;
-use rustc_middle::ty::TyCtxt;
 use rustc_tools_util::VersionInfo;
 
 use std::borrow::Cow;
@@ -168,7 +166,7 @@ fn report_clippy_ice(info: &panic::PanicInfo<'_>, bug_report_url: &str) {
 
     let num_frames = if backtrace { None } else { Some(2) };
 
-    TyCtxt::try_print_query_stack(&handler, num_frames);
+    interface::try_print_query_stack(&handler, num_frames);
 }
 
 fn toolchain_path(home: Option<String>, toolchain: Option<String>) -> Option<PathBuf> {

From bc4ba0629165956ab5af6c619e30d7d4d02e7c8a Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Tue, 5 Jan 2021 18:37:42 +0100
Subject: [PATCH 18/27] Move try_load_from_on_disk_cache to the QueryContext.

---
 compiler/rustc_middle/src/dep_graph/dep_node.rs     | 9 ++++-----
 compiler/rustc_middle/src/dep_graph/mod.rs          | 4 ----
 compiler/rustc_middle/src/ty/query/on_disk_cache.rs | 2 +-
 compiler/rustc_middle/src/ty/query/plumbing.rs      | 9 +++++++++
 compiler/rustc_query_system/src/dep_graph/graph.rs  | 6 ++++--
 compiler/rustc_query_system/src/dep_graph/mod.rs    | 3 ---
 compiler/rustc_query_system/src/query/mod.rs        | 5 ++++-
 7 files changed, 22 insertions(+), 16 deletions(-)

diff --git a/compiler/rustc_middle/src/dep_graph/dep_node.rs b/compiler/rustc_middle/src/dep_graph/dep_node.rs
index 94a17f4645ac3..14a10e84e9cd4 100644
--- a/compiler/rustc_middle/src/dep_graph/dep_node.rs
+++ b/compiler/rustc_middle/src/dep_graph/dep_node.rs
@@ -138,7 +138,7 @@ pub struct DepKindStruct {
     pub(super) force_from_dep_node: fn(tcx: TyCtxt<'_>, dep_node: &DepNode) -> bool,
 
     /// Invoke a query to put the on-disk cached value in memory.
-    pub(super) try_load_from_on_disk_cache: fn(TyCtxt<'_>, &DepNode),
+    pub(crate) try_load_from_on_disk_cache: fn(QueryCtxt<'_>, &DepNode),
 }
 
 impl std::ops::Deref for DepKind {
@@ -273,7 +273,7 @@ pub mod dep_kind {
                     false
                 }
 
-                fn try_load_from_on_disk_cache(tcx: TyCtxt<'_>, dep_node: &DepNode) {
+                fn try_load_from_on_disk_cache(tcx: QueryCtxt<'_>, dep_node: &DepNode) {
                     if is_anon {
                         return
                     }
@@ -287,9 +287,8 @@ pub mod dep_kind {
                                      .map(|c| c.is_green())
                                      .unwrap_or(false));
 
-                    let key = recover(tcx, dep_node).unwrap_or_else(|| panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash));
-                    let qcx = QueryCtxt { tcx, queries: tcx.queries };
-                    if queries::$variant::cache_on_disk(qcx, &key, None) {
+                    let key = recover(*tcx, dep_node).unwrap_or_else(|| panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash));
+                    if queries::$variant::cache_on_disk(tcx, &key, None) {
                         let _ = tcx.$variant(key);
                     }
                 }
diff --git a/compiler/rustc_middle/src/dep_graph/mod.rs b/compiler/rustc_middle/src/dep_graph/mod.rs
index 4746004cfca23..193bf3c91319a 100644
--- a/compiler/rustc_middle/src/dep_graph/mod.rs
+++ b/compiler/rustc_middle/src/dep_graph/mod.rs
@@ -180,10 +180,6 @@ impl<'tcx> DepContext for TyCtxt<'tcx> {
     }
 
     // Interactions with on_disk_cache
-    fn try_load_from_on_disk_cache(&self, dep_node: &DepNode) {
-        (dep_node.kind.try_load_from_on_disk_cache)(*self, dep_node)
-    }
-
     fn load_diagnostics(&self, prev_dep_node_index: SerializedDepNodeIndex) -> Vec<Diagnostic> {
         self.on_disk_cache
             .as_ref()
diff --git a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs
index cffb42b8ba737..9834bb407e32d 100644
--- a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs
+++ b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs
@@ -285,7 +285,7 @@ impl<'sess> OnDiskCache<'sess> {
             // Do this *before* we clone 'latest_foreign_def_path_hashes', since
             // loading existing queries may cause us to create new DepNodes, which
             // may in turn end up invoking `store_foreign_def_id_hash`
-            tcx.dep_graph.exec_cache_promotions(tcx);
+            tcx.queries.exec_cache_promotions(tcx);
 
             let latest_foreign_def_path_hashes = self.latest_foreign_def_path_hashes.lock().clone();
             let hygiene_encode_context = HygieneEncodeContext::default();
diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs
index 6aa9d88798fa7..83dbe4875cf01 100644
--- a/compiler/rustc_middle/src/ty/query/plumbing.rs
+++ b/compiler/rustc_middle/src/ty/query/plumbing.rs
@@ -68,6 +68,10 @@ impl QueryContext for QueryCtxt<'tcx> {
         self.queries.try_collect_active_jobs()
     }
 
+    fn try_load_from_on_disk_cache(&self, dep_node: &dep_graph::DepNode) {
+        (dep_node.kind.try_load_from_on_disk_cache)(*self, dep_node)
+    }
+
     /// Executes a job by changing the `ImplicitCtxt` to point to the
     /// new query job while it executes. It returns the diagnostics
     /// captured during execution and the actual result.
@@ -603,6 +607,11 @@ macro_rules! define_queries_struct {
                 tcx.encode_query_results(encoder, query_result_index)
             }
 
+            fn exec_cache_promotions(&'tcx self, tcx: TyCtxt<'tcx>) {
+                let tcx = QueryCtxt { tcx, queries: self };
+                tcx.dep_graph.exec_cache_promotions(tcx)
+            }
+
             $($(#[$attr])*
             #[inline(always)]
             fn $name(
diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs
index 9c441bf036434..f1281c81c8b2f 100644
--- a/compiler/rustc_query_system/src/dep_graph/graph.rs
+++ b/compiler/rustc_query_system/src/dep_graph/graph.rs
@@ -24,6 +24,7 @@ use super::prev::PreviousDepGraph;
 use super::query::DepGraphQuery;
 use super::serialized::SerializedDepNodeIndex;
 use super::{DepContext, DepKind, DepNode, HasDepContext, WorkProductId};
+use crate::query::QueryContext;
 
 #[derive(Clone)]
 pub struct DepGraph<K: DepKind> {
@@ -875,7 +876,8 @@ impl<K: DepKind> DepGraph<K> {
     //
     // This method will only load queries that will end up in the disk cache.
     // Other queries will not be executed.
-    pub fn exec_cache_promotions<Ctxt: DepContext<DepKind = K>>(&self, tcx: Ctxt) {
+    pub fn exec_cache_promotions<Ctxt: QueryContext<DepKind = K>>(&self, qcx: Ctxt) {
+        let tcx = qcx.dep_context();
         let _prof_timer = tcx.profiler().generic_activity("incr_comp_query_cache_promotion");
 
         let data = self.data.as_ref().unwrap();
@@ -883,7 +885,7 @@ impl<K: DepKind> DepGraph<K> {
             match data.colors.get(prev_index) {
                 Some(DepNodeColor::Green(_)) => {
                     let dep_node = data.previous.index_to_node(prev_index);
-                    tcx.try_load_from_on_disk_cache(&dep_node);
+                    qcx.try_load_from_on_disk_cache(&dep_node);
                 }
                 None | Some(DepNodeColor::Red) => {
                     // We can skip red nodes because a node can only be marked
diff --git a/compiler/rustc_query_system/src/dep_graph/mod.rs b/compiler/rustc_query_system/src/dep_graph/mod.rs
index db192d1cfe77a..a1a6ca9dacc16 100644
--- a/compiler/rustc_query_system/src/dep_graph/mod.rs
+++ b/compiler/rustc_query_system/src/dep_graph/mod.rs
@@ -43,9 +43,6 @@ pub trait DepContext: Copy {
     /// Return the diagnostic handler.
     fn diagnostic(&self) -> &rustc_errors::Handler;
 
-    /// Load data from the on-disk cache.
-    fn try_load_from_on_disk_cache(&self, dep_node: &DepNode<Self::DepKind>);
-
     /// Load diagnostics associated to the node in the previous session.
     fn load_diagnostics(&self, prev_dep_node_index: SerializedDepNodeIndex) -> Vec<Diagnostic>;
 
diff --git a/compiler/rustc_query_system/src/query/mod.rs b/compiler/rustc_query_system/src/query/mod.rs
index 2d678035d4d51..8d5c9d7bea7e4 100644
--- a/compiler/rustc_query_system/src/query/mod.rs
+++ b/compiler/rustc_query_system/src/query/mod.rs
@@ -14,7 +14,7 @@ pub use self::caches::{
 mod config;
 pub use self::config::{QueryAccessors, QueryConfig, QueryDescription};
 
-use crate::dep_graph::HasDepContext;
+use crate::dep_graph::{DepNode, HasDepContext};
 use crate::query::job::QueryMap;
 
 use rustc_data_structures::stable_hasher::HashStable;
@@ -37,6 +37,9 @@ pub trait QueryContext: HasDepContext {
 
     fn try_collect_active_jobs(&self) -> Option<QueryMap<Self::DepKind, Self::Query>>;
 
+    /// Load data from the on-disk cache.
+    fn try_load_from_on_disk_cache(&self, dep_node: &DepNode<Self::DepKind>);
+
     /// Executes a job by changing the `ImplicitCtxt` to point to the
     /// new query job while it executes. It returns the diagnostics
     /// captured during execution and the actual result.

From f6d5aae55d61462197f020725a40dfdb62521045 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Wed, 18 Nov 2020 19:02:31 +0100
Subject: [PATCH 19/27] Move handle_deadlock where it is used.

---
 Cargo.lock                                    |  1 +
 compiler/rustc_interface/Cargo.toml           |  1 +
 compiler/rustc_interface/src/util.rs          | 32 ++++++++++++++++---
 compiler/rustc_middle/src/ty/query/job.rs     | 24 --------------
 compiler/rustc_middle/src/ty/query/mod.rs     |  4 ---
 .../rustc_middle/src/ty/query/plumbing.rs     |  2 +-
 6 files changed, 31 insertions(+), 33 deletions(-)
 delete mode 100644 compiler/rustc_middle/src/ty/query/job.rs

diff --git a/Cargo.lock b/Cargo.lock
index 3bf2159c91d2f..b441c5dd95745 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3878,6 +3878,7 @@ version = "0.0.0"
 dependencies = [
  "libc",
  "rustc-rayon",
+ "rustc-rayon-core",
  "rustc_ast",
  "rustc_ast_lowering",
  "rustc_ast_passes",
diff --git a/compiler/rustc_interface/Cargo.toml b/compiler/rustc_interface/Cargo.toml
index 2441e0f69eef6..2be78e8d3b6b8 100644
--- a/compiler/rustc_interface/Cargo.toml
+++ b/compiler/rustc_interface/Cargo.toml
@@ -10,6 +10,7 @@ doctest = false
 [dependencies]
 libc = "0.2"
 tracing = "0.1"
+rustc-rayon-core = "0.3.0"
 rayon = { version = "0.3.0", package = "rustc-rayon" }
 smallvec = { version = "1.0", features = ["union", "may_dangle"] }
 rustc_ast = { path = "../rustc_ast" }
diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs
index b7dc539c6d606..798996263c700 100644
--- a/compiler/rustc_interface/src/util.rs
+++ b/compiler/rustc_interface/src/util.rs
@@ -10,6 +10,8 @@ use rustc_data_structures::stable_hasher::StableHasher;
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::registry::Registry;
 use rustc_metadata::dynamic_lib::DynamicLibrary;
+#[cfg(parallel_compiler)]
+use rustc_middle::ty::tls;
 use rustc_resolve::{self, Resolver};
 use rustc_session as session;
 use rustc_session::config::{self, CrateType};
@@ -29,11 +31,12 @@ use std::io;
 use std::lazy::SyncOnceCell;
 use std::mem;
 use std::ops::DerefMut;
+#[cfg(not(parallel_compiler))]
+use std::panic;
 use std::path::{Path, PathBuf};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex, Once};
-#[cfg(not(parallel_compiler))]
-use std::{panic, thread};
+use std::thread;
 use tracing::info;
 
 /// Adds `target_feature = "..."` cfgs for a variety of platform
@@ -156,6 +159,28 @@ pub fn setup_callbacks_and_run_in_thread_pool_with_globals<F: FnOnce() -> R + Se
     scoped_thread(cfg, main_handler)
 }
 
+/// Creates a new thread and forwards information in thread locals to it.
+/// The new thread runs the deadlock handler.
+/// Must only be called when a deadlock is about to happen.
+#[cfg(parallel_compiler)]
+unsafe fn handle_deadlock() {
+    let registry = rustc_rayon_core::Registry::current();
+
+    let context = tls::get_tlv();
+    assert!(context != 0);
+    rustc_data_structures::sync::assert_sync::<tls::ImplicitCtxt<'_, '_>>();
+    let icx: &tls::ImplicitCtxt<'_, '_> = &*(context as *const tls::ImplicitCtxt<'_, '_>);
+
+    let session_globals = rustc_span::SESSION_GLOBALS.with(|sg| sg as *const _);
+    let session_globals = &*session_globals;
+    thread::spawn(move || {
+        tls::enter_context(icx, |_| {
+            rustc_span::SESSION_GLOBALS
+                .set(session_globals, || tls::with(|tcx| tcx.queries.deadlock(tcx, &registry)))
+        });
+    });
+}
+
 #[cfg(parallel_compiler)]
 pub fn setup_callbacks_and_run_in_thread_pool_with_globals<F: FnOnce() -> R + Send, R: Send>(
     edition: Edition,
@@ -163,7 +188,6 @@ pub fn setup_callbacks_and_run_in_thread_pool_with_globals<F: FnOnce() -> R + Se
     stderr: &Option<Arc<Mutex<Vec<u8>>>>,
     f: F,
 ) -> R {
-    use rustc_middle::ty;
     crate::callbacks::setup_callbacks();
 
     let mut config = rayon::ThreadPoolBuilder::new()
@@ -171,7 +195,7 @@ pub fn setup_callbacks_and_run_in_thread_pool_with_globals<F: FnOnce() -> R + Se
         .acquire_thread_handler(jobserver::acquire_thread)
         .release_thread_handler(jobserver::release_thread)
         .num_threads(threads)
-        .deadlock_handler(|| unsafe { ty::query::handle_deadlock() });
+        .deadlock_handler(|| unsafe { handle_deadlock() });
 
     if let Some(size) = get_stack_size() {
         config = config.stack_size(size);
diff --git a/compiler/rustc_middle/src/ty/query/job.rs b/compiler/rustc_middle/src/ty/query/job.rs
deleted file mode 100644
index 668109693a8b4..0000000000000
--- a/compiler/rustc_middle/src/ty/query/job.rs
+++ /dev/null
@@ -1,24 +0,0 @@
-use crate::ty::tls;
-use rustc_rayon_core as rayon_core;
-use std::thread;
-
-/// Creates a new thread and forwards information in thread locals to it.
-/// The new thread runs the deadlock handler.
-/// Must only be called when a deadlock is about to happen.
-pub unsafe fn handle_deadlock() {
-    let registry = rayon_core::Registry::current();
-
-    let context = tls::get_tlv();
-    assert!(context != 0);
-    rustc_data_structures::sync::assert_sync::<tls::ImplicitCtxt<'_, '_>>();
-    let icx: &tls::ImplicitCtxt<'_, '_> = &*(context as *const tls::ImplicitCtxt<'_, '_>);
-
-    let session_globals = rustc_span::SESSION_GLOBALS.with(|sg| sg as *const _);
-    let session_globals = &*session_globals;
-    thread::spawn(move || {
-        tls::enter_context(icx, |_| {
-            rustc_span::SESSION_GLOBALS
-                .set(session_globals, || tls::with(|tcx| tcx.queries.deadlock(tcx, &registry)))
-        });
-    });
-}
diff --git a/compiler/rustc_middle/src/ty/query/mod.rs b/compiler/rustc_middle/src/ty/query/mod.rs
index dd0723f200f76..5a2c51f88cdfa 100644
--- a/compiler/rustc_middle/src/ty/query/mod.rs
+++ b/compiler/rustc_middle/src/ty/query/mod.rs
@@ -68,10 +68,6 @@ use rustc_query_system::query::*;
 mod stats;
 pub use self::stats::print_stats;
 
-#[cfg(parallel_compiler)]
-mod job;
-#[cfg(parallel_compiler)]
-pub use self::job::handle_deadlock;
 pub use rustc_query_system::query::{QueryInfo, QueryJob, QueryJobId};
 
 mod keys;
diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs
index 83dbe4875cf01..8d006ad656423 100644
--- a/compiler/rustc_middle/src/ty/query/plumbing.rs
+++ b/compiler/rustc_middle/src/ty/query/plumbing.rs
@@ -592,7 +592,7 @@ macro_rules! define_queries_struct {
             }
 
             #[cfg(parallel_compiler)]
-            unsafe fn deadlock(&'tcx self, tcx: TyCtxt<'tcx>, registry: &rustc_rayon_core::Registry) {
+            pub unsafe fn deadlock(&'tcx self, tcx: TyCtxt<'tcx>, registry: &rustc_rayon_core::Registry) {
                 let tcx = QueryCtxt { tcx, queries: self };
                 rustc_query_system::query::deadlock(tcx, registry)
             }

From 89d7a9501126a908a66086c4a8d5412e07f2467e Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Mon, 18 Jan 2021 22:21:29 +0100
Subject: [PATCH 20/27] Move definition of callbacks to parent module.

---
 compiler/rustc_middle/src/ty/query/mod.rs     | 88 +++++++++++++++++++
 .../rustc_middle/src/ty/query/plumbing.rs     | 83 -----------------
 2 files changed, 88 insertions(+), 83 deletions(-)

diff --git a/compiler/rustc_middle/src/ty/query/mod.rs b/compiler/rustc_middle/src/ty/query/mod.rs
index 5a2c51f88cdfa..c090e2de1f48a 100644
--- a/compiler/rustc_middle/src/ty/query/mod.rs
+++ b/compiler/rustc_middle/src/ty/query/mod.rs
@@ -86,6 +86,93 @@ pub use self::on_disk_cache::OnDiskCache;
 mod profiling_support;
 pub use self::profiling_support::alloc_self_profile_query_strings;
 
+#[derive(Copy, Clone)]
+pub struct TyCtxtAt<'tcx> {
+    pub tcx: TyCtxt<'tcx>,
+    pub span: Span,
+}
+
+impl Deref for TyCtxtAt<'tcx> {
+    type Target = TyCtxt<'tcx>;
+    #[inline(always)]
+    fn deref(&self) -> &Self::Target {
+        &self.tcx
+    }
+}
+
+#[derive(Copy, Clone)]
+pub struct TyCtxtEnsure<'tcx> {
+    pub tcx: TyCtxt<'tcx>,
+}
+
+impl TyCtxt<'tcx> {
+    /// Returns a transparent wrapper for `TyCtxt`, which ensures queries
+    /// are executed instead of just returning their results.
+    #[inline(always)]
+    pub fn ensure(self) -> TyCtxtEnsure<'tcx> {
+        TyCtxtEnsure { tcx: self }
+    }
+
+    /// Returns a transparent wrapper for `TyCtxt` which uses
+    /// `span` as the location of queries performed through it.
+    #[inline(always)]
+    pub fn at(self, span: Span) -> TyCtxtAt<'tcx> {
+        TyCtxtAt { tcx: self, span }
+    }
+}
+
+macro_rules! define_callbacks {
+    (<$tcx:tt>
+     $($(#[$attr:meta])*
+        [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => {
+
+        impl TyCtxtEnsure<$tcx> {
+            $($(#[$attr])*
+            #[inline(always)]
+            pub fn $name(self, key: query_helper_param_ty!($($K)*)) {
+                let key = key.into_query_param();
+                let cached = try_get_cached(self.tcx, &self.tcx.query_caches.$name, &key, |_| {});
+
+                let lookup = match cached {
+                    Ok(()) => return,
+                    Err(lookup) => lookup,
+                };
+
+                self.tcx.queries.$name(self.tcx, DUMMY_SP, key, lookup, QueryMode::Ensure);
+            })*
+        }
+
+        impl TyCtxt<$tcx> {
+            $($(#[$attr])*
+            #[inline(always)]
+            #[must_use]
+            pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> query_stored::$name<$tcx>
+            {
+                self.at(DUMMY_SP).$name(key)
+            })*
+        }
+
+        impl TyCtxtAt<$tcx> {
+            $($(#[$attr])*
+            #[inline(always)]
+            pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> query_stored::$name<$tcx>
+            {
+                let key = key.into_query_param();
+                let cached = try_get_cached(self.tcx, &self.tcx.query_caches.$name, &key, |value| {
+                    value.clone()
+                });
+
+                let lookup = match cached {
+                    Ok(value) => return value,
+                    Err(lookup) => lookup,
+                };
+
+                self.tcx.queries.$name(self.tcx, self.span, key, lookup, QueryMode::Get).unwrap()
+            })*
+        }
+    }
+}
+
 // Each of these queries corresponds to a function pointer field in the
 // `Providers` struct for requesting a value of that type, and a method
 // on `tcx: TyCtxt` (and `tcx.at(span)`) for doing that request in a way
@@ -99,6 +186,7 @@ pub use self::profiling_support::alloc_self_profile_query_strings;
 // as they will raise an fatal error on query cycles instead.
 
 rustc_query_append! { [define_queries!][<'tcx>] }
+rustc_query_append! { [define_callbacks!][<'tcx>] }
 
 mod sealed {
     use super::{DefId, LocalDefId};
diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs
index 8d006ad656423..0f38de35e51fa 100644
--- a/compiler/rustc_middle/src/ty/query/plumbing.rs
+++ b/compiler/rustc_middle/src/ty/query/plumbing.rs
@@ -456,89 +456,6 @@ macro_rules! define_queries {
             }
         })*
 
-        #[derive(Copy, Clone)]
-        pub struct TyCtxtEnsure<'tcx> {
-            pub tcx: TyCtxt<'tcx>,
-        }
-
-        impl TyCtxtEnsure<$tcx> {
-            $($(#[$attr])*
-            #[inline(always)]
-            pub fn $name(self, key: query_helper_param_ty!($($K)*)) {
-                let key = key.into_query_param();
-                let cached = try_get_cached(self.tcx, &self.tcx.query_caches.$name, &key, |_| {});
-
-                let lookup = match cached {
-                    Ok(()) => return,
-                    Err(lookup) => lookup,
-                };
-
-                self.tcx.queries.$name(self.tcx, DUMMY_SP, key, lookup, QueryMode::Ensure);
-            })*
-        }
-
-        #[derive(Copy, Clone)]
-        pub struct TyCtxtAt<'tcx> {
-            pub tcx: TyCtxt<'tcx>,
-            pub span: Span,
-        }
-
-        impl Deref for TyCtxtAt<'tcx> {
-            type Target = TyCtxt<'tcx>;
-            #[inline(always)]
-            fn deref(&self) -> &Self::Target {
-                &self.tcx
-            }
-        }
-
-        impl TyCtxt<$tcx> {
-            /// Returns a transparent wrapper for `TyCtxt`, which ensures queries
-            /// are executed instead of just returning their results.
-            #[inline(always)]
-            pub fn ensure(self) -> TyCtxtEnsure<$tcx> {
-                TyCtxtEnsure {
-                    tcx: self,
-                }
-            }
-
-            /// Returns a transparent wrapper for `TyCtxt` which uses
-            /// `span` as the location of queries performed through it.
-            #[inline(always)]
-            pub fn at(self, span: Span) -> TyCtxtAt<$tcx> {
-                TyCtxtAt {
-                    tcx: self,
-                    span
-                }
-            }
-
-            $($(#[$attr])*
-            #[inline(always)]
-            #[must_use]
-            pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> query_stored::$name<$tcx>
-            {
-                self.at(DUMMY_SP).$name(key)
-            })*
-        }
-
-        impl TyCtxtAt<$tcx> {
-            $($(#[$attr])*
-            #[inline(always)]
-            pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> query_stored::$name<$tcx>
-            {
-                let key = key.into_query_param();
-                let cached = try_get_cached(self.tcx, &self.tcx.query_caches.$name, &key, |value| {
-                    value.clone()
-                });
-
-                let lookup = match cached {
-                    Ok(value) => return value,
-                    Err(lookup) => lookup,
-                };
-
-                self.tcx.queries.$name(self.tcx, self.span, key, lookup, QueryMode::Get).unwrap()
-            })*
-        }
-
         define_provider_struct! {
             tcx: $tcx,
             input: ($(([$($modifiers)*] [$name] [$($K)*] [$V]))*)

From fd417672a98700bf8a7516957b5397e29a3e3054 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Wed, 18 Nov 2020 18:45:16 +0100
Subject: [PATCH 21/27] Select caching strategy per query.

The per-Key choice was not used.
---
 compiler/rustc_middle/src/ty/query/keys.rs    | 69 -------------------
 .../rustc_middle/src/ty/query/plumbing.rs     |  2 +-
 2 files changed, 1 insertion(+), 70 deletions(-)

diff --git a/compiler/rustc_middle/src/ty/query/keys.rs b/compiler/rustc_middle/src/ty/query/keys.rs
index 6b4714b1bb8c8..2f76237e8fb79 100644
--- a/compiler/rustc_middle/src/ty/query/keys.rs
+++ b/compiler/rustc_middle/src/ty/query/keys.rs
@@ -6,15 +6,12 @@ use crate::ty::fast_reject::SimplifiedType;
 use crate::ty::subst::{GenericArg, SubstsRef};
 use crate::ty::{self, Ty, TyCtxt};
 use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, LOCAL_CRATE};
-use rustc_query_system::query::DefaultCacheSelector;
 use rustc_span::symbol::{Ident, Symbol};
 use rustc_span::{Span, DUMMY_SP};
 
 /// The `Key` trait controls what types can legally be used as the key
 /// for a query.
 pub trait Key {
-    type CacheSelector;
-
     /// Given an instance of this key, what crate is it referring to?
     /// This is used to find the provider.
     fn query_crate(&self) -> CrateNum;
@@ -25,8 +22,6 @@ pub trait Key {
 }
 
 impl<'tcx> Key for ty::InstanceDef<'tcx> {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         LOCAL_CRATE
     }
@@ -37,8 +32,6 @@ impl<'tcx> Key for ty::InstanceDef<'tcx> {
 }
 
 impl<'tcx> Key for ty::Instance<'tcx> {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         LOCAL_CRATE
     }
@@ -49,8 +42,6 @@ impl<'tcx> Key for ty::Instance<'tcx> {
 }
 
 impl<'tcx> Key for mir::interpret::GlobalId<'tcx> {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         self.instance.query_crate()
     }
@@ -61,8 +52,6 @@ impl<'tcx> Key for mir::interpret::GlobalId<'tcx> {
 }
 
 impl<'tcx> Key for mir::interpret::LitToConstInput<'tcx> {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         LOCAL_CRATE
     }
@@ -73,8 +62,6 @@ impl<'tcx> Key for mir::interpret::LitToConstInput<'tcx> {
 }
 
 impl Key for CrateNum {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         *self
     }
@@ -84,8 +71,6 @@ impl Key for CrateNum {
 }
 
 impl Key for LocalDefId {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         self.to_def_id().query_crate()
     }
@@ -95,8 +80,6 @@ impl Key for LocalDefId {
 }
 
 impl Key for DefId {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         self.krate
     }
@@ -106,8 +89,6 @@ impl Key for DefId {
 }
 
 impl Key for ty::WithOptConstParam<LocalDefId> {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         self.did.query_crate()
     }
@@ -117,8 +98,6 @@ impl Key for ty::WithOptConstParam<LocalDefId> {
 }
 
 impl Key for (DefId, DefId) {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         self.0.krate
     }
@@ -128,8 +107,6 @@ impl Key for (DefId, DefId) {
 }
 
 impl Key for (ty::Instance<'tcx>, LocalDefId) {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         self.0.query_crate()
     }
@@ -139,8 +116,6 @@ impl Key for (ty::Instance<'tcx>, LocalDefId) {
 }
 
 impl Key for (DefId, LocalDefId) {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         self.0.krate
     }
@@ -150,8 +125,6 @@ impl Key for (DefId, LocalDefId) {
 }
 
 impl Key for (LocalDefId, DefId) {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         LOCAL_CRATE
     }
@@ -161,8 +134,6 @@ impl Key for (LocalDefId, DefId) {
 }
 
 impl Key for (DefId, Option<Ident>) {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         self.0.krate
     }
@@ -172,8 +143,6 @@ impl Key for (DefId, Option<Ident>) {
 }
 
 impl Key for (DefId, LocalDefId, Ident) {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         self.0.krate
     }
@@ -183,8 +152,6 @@ impl Key for (DefId, LocalDefId, Ident) {
 }
 
 impl Key for (CrateNum, DefId) {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         self.0
     }
@@ -194,8 +161,6 @@ impl Key for (CrateNum, DefId) {
 }
 
 impl Key for (DefId, SimplifiedType) {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         self.0.krate
     }
@@ -205,8 +170,6 @@ impl Key for (DefId, SimplifiedType) {
 }
 
 impl<'tcx> Key for SubstsRef<'tcx> {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         LOCAL_CRATE
     }
@@ -216,8 +179,6 @@ impl<'tcx> Key for SubstsRef<'tcx> {
 }
 
 impl<'tcx> Key for (DefId, SubstsRef<'tcx>) {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         self.0.krate
     }
@@ -232,8 +193,6 @@ impl<'tcx> Key
         (ty::WithOptConstParam<DefId>, SubstsRef<'tcx>),
     )
 {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         (self.0).0.did.krate
     }
@@ -243,8 +202,6 @@ impl<'tcx> Key
 }
 
 impl<'tcx> Key for (LocalDefId, DefId, SubstsRef<'tcx>) {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         LOCAL_CRATE
     }
@@ -254,8 +211,6 @@ impl<'tcx> Key for (LocalDefId, DefId, SubstsRef<'tcx>) {
 }
 
 impl<'tcx> Key for (ty::ParamEnv<'tcx>, ty::PolyTraitRef<'tcx>) {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         self.1.def_id().krate
     }
@@ -265,8 +220,6 @@ impl<'tcx> Key for (ty::ParamEnv<'tcx>, ty::PolyTraitRef<'tcx>) {
 }
 
 impl<'tcx> Key for (&'tcx ty::Const<'tcx>, mir::Field) {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         LOCAL_CRATE
     }
@@ -276,8 +229,6 @@ impl<'tcx> Key for (&'tcx ty::Const<'tcx>, mir::Field) {
 }
 
 impl<'tcx> Key for ty::PolyTraitRef<'tcx> {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         self.def_id().krate
     }
@@ -287,8 +238,6 @@ impl<'tcx> Key for ty::PolyTraitRef<'tcx> {
 }
 
 impl<'tcx> Key for GenericArg<'tcx> {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         LOCAL_CRATE
     }
@@ -298,8 +247,6 @@ impl<'tcx> Key for GenericArg<'tcx> {
 }
 
 impl<'tcx> Key for &'tcx ty::Const<'tcx> {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         LOCAL_CRATE
     }
@@ -309,8 +256,6 @@ impl<'tcx> Key for &'tcx ty::Const<'tcx> {
 }
 
 impl<'tcx> Key for Ty<'tcx> {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         LOCAL_CRATE
     }
@@ -320,8 +265,6 @@ impl<'tcx> Key for Ty<'tcx> {
 }
 
 impl<'tcx> Key for &'tcx ty::List<ty::Predicate<'tcx>> {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         LOCAL_CRATE
     }
@@ -331,8 +274,6 @@ impl<'tcx> Key for &'tcx ty::List<ty::Predicate<'tcx>> {
 }
 
 impl<'tcx> Key for ty::ParamEnv<'tcx> {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         LOCAL_CRATE
     }
@@ -342,8 +283,6 @@ impl<'tcx> Key for ty::ParamEnv<'tcx> {
 }
 
 impl<'tcx, T: Key> Key for ty::ParamEnvAnd<'tcx, T> {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         self.value.query_crate()
     }
@@ -353,8 +292,6 @@ impl<'tcx, T: Key> Key for ty::ParamEnvAnd<'tcx, T> {
 }
 
 impl Key for Symbol {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         LOCAL_CRATE
     }
@@ -366,8 +303,6 @@ impl Key for Symbol {
 /// Canonical query goals correspond to abstract trait operations that
 /// are not tied to any crate in particular.
 impl<'tcx, T> Key for Canonical<'tcx, T> {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         LOCAL_CRATE
     }
@@ -378,8 +313,6 @@ impl<'tcx, T> Key for Canonical<'tcx, T> {
 }
 
 impl Key for (Symbol, u32, u32) {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         LOCAL_CRATE
     }
@@ -390,8 +323,6 @@ impl Key for (Symbol, u32, u32) {
 }
 
 impl<'tcx> Key for (DefId, Ty<'tcx>, SubstsRef<'tcx>, ty::ParamEnv<'tcx>) {
-    type CacheSelector = DefaultCacheSelector;
-
     fn query_crate(&self) -> CrateNum {
         LOCAL_CRATE
     }
diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs
index 0f38de35e51fa..c1068528a1823 100644
--- a/compiler/rustc_middle/src/ty/query/plumbing.rs
+++ b/compiler/rustc_middle/src/ty/query/plumbing.rs
@@ -263,7 +263,7 @@ macro_rules! is_eval_always {
 
 macro_rules! query_storage {
     ([][$K:ty, $V:ty]) => {
-        <<$K as Key>::CacheSelector as CacheSelector<$K, $V>>::Cache
+        <DefaultCacheSelector as CacheSelector<$K, $V>>::Cache
     };
     ([storage($ty:ty) $($rest:tt)*][$K:ty, $V:ty]) => {
         <$ty as CacheSelector<$K, $V>>::Cache

From 79be9be6b835bb8b19dcf1834269a94db48a8417 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Mon, 18 Jan 2021 23:53:42 +0100
Subject: [PATCH 22/27] Use a QueryContext for try_mark_green.

---
 .../rustc_codegen_cranelift/src/driver/aot.rs |  6 +-
 compiler/rustc_codegen_ssa/src/base.rs        |  2 +-
 .../rustc_middle/src/dep_graph/dep_node.rs    |  2 +-
 compiler/rustc_middle/src/dep_graph/mod.rs    | 95 -------------------
 compiler/rustc_middle/src/ty/query/mod.rs     |  5 +
 .../rustc_middle/src/ty/query/plumbing.rs     | 91 +++++++++++++++++-
 .../rustc_query_system/src/dep_graph/graph.rs |  8 +-
 .../rustc_query_system/src/dep_graph/mod.rs   | 24 -----
 compiler/rustc_query_system/src/query/mod.rs  | 24 ++++-
 .../rustc_query_system/src/query/plumbing.rs  | 19 ++--
 10 files changed, 133 insertions(+), 143 deletions(-)

diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs
index df89883f0bbb7..ffedf1fdf060d 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs
@@ -465,9 +465,5 @@ fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguR
         cgu.name()
     );
 
-    if tcx.dep_graph.try_mark_green(tcx, &dep_node).is_some() {
-        CguReuse::PreLto
-    } else {
-        CguReuse::No
-    }
+    if tcx.try_mark_green(&dep_node) { CguReuse::PreLto } else { CguReuse::No }
 }
diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs
index 658ad3c375d29..08e31c3b37f34 100644
--- a/compiler/rustc_codegen_ssa/src/base.rs
+++ b/compiler/rustc_codegen_ssa/src/base.rs
@@ -867,7 +867,7 @@ fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguR
         cgu.name()
     );
 
-    if tcx.dep_graph.try_mark_green(tcx, &dep_node).is_some() {
+    if tcx.try_mark_green(&dep_node) {
         // We can re-use either the pre- or the post-thinlto state. If no LTO is
         // being performed then we can use post-LTO artifacts, otherwise we must
         // reuse pre-LTO artifacts
diff --git a/compiler/rustc_middle/src/dep_graph/dep_node.rs b/compiler/rustc_middle/src/dep_graph/dep_node.rs
index 14a10e84e9cd4..823bcebe23b14 100644
--- a/compiler/rustc_middle/src/dep_graph/dep_node.rs
+++ b/compiler/rustc_middle/src/dep_graph/dep_node.rs
@@ -135,7 +135,7 @@ pub struct DepKindStruct {
     /// then `force_from_dep_node()` should not fail for it. Otherwise, you can just
     /// add it to the "We don't have enough information to reconstruct..." group in
     /// the match below.
-    pub(super) force_from_dep_node: fn(tcx: TyCtxt<'_>, dep_node: &DepNode) -> bool,
+    pub(crate) force_from_dep_node: fn(tcx: TyCtxt<'_>, dep_node: &DepNode) -> bool,
 
     /// Invoke a query to put the on-disk cached value in memory.
     pub(crate) try_load_from_on_disk_cache: fn(QueryCtxt<'_>, &DepNode),
diff --git a/compiler/rustc_middle/src/dep_graph/mod.rs b/compiler/rustc_middle/src/dep_graph/mod.rs
index 193bf3c91319a..da9b0b7e48af4 100644
--- a/compiler/rustc_middle/src/dep_graph/mod.rs
+++ b/compiler/rustc_middle/src/dep_graph/mod.rs
@@ -2,9 +2,6 @@ use crate::ich::StableHashingContext;
 use crate::ty::{self, TyCtxt};
 use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_data_structures::sync::Lock;
-use rustc_data_structures::thin_vec::ThinVec;
-use rustc_errors::Diagnostic;
-use rustc_hir::def_id::LocalDefId;
 
 mod dep_node;
 
@@ -116,99 +113,7 @@ impl<'tcx> DepContext for TyCtxt<'tcx> {
         &self.dep_graph
     }
 
-    fn try_force_from_dep_node(&self, dep_node: &DepNode) -> bool {
-        // FIXME: This match is just a workaround for incremental bugs and should
-        // be removed. https://github.com/rust-lang/rust/issues/62649 is one such
-        // bug that must be fixed before removing this.
-        match dep_node.kind {
-            DepKind::hir_owner | DepKind::hir_owner_nodes => {
-                if let Some(def_id) = dep_node.extract_def_id(*self) {
-                    if !def_id_corresponds_to_hir_dep_node(*self, def_id.expect_local()) {
-                        // This `DefPath` does not have a
-                        // corresponding `DepNode` (e.g. a
-                        // struct field), and the ` DefPath`
-                        // collided with the `DefPath` of a
-                        // proper item that existed in the
-                        // previous compilation session.
-                        //
-                        // Since the given `DefPath` does not
-                        // denote the item that previously
-                        // existed, we just fail to mark green.
-                        return false;
-                    }
-                } else {
-                    // If the node does not exist anymore, we
-                    // just fail to mark green.
-                    return false;
-                }
-            }
-            _ => {
-                // For other kinds of nodes it's OK to be
-                // forced.
-            }
-        }
-
-        debug!("try_force_from_dep_node({:?}) --- trying to force", dep_node);
-
-        // We must avoid ever having to call `force_from_dep_node()` for a
-        // `DepNode::codegen_unit`:
-        // Since we cannot reconstruct the query key of a `DepNode::codegen_unit`, we
-        // would always end up having to evaluate the first caller of the
-        // `codegen_unit` query that *is* reconstructible. This might very well be
-        // the `compile_codegen_unit` query, thus re-codegenning the whole CGU just
-        // to re-trigger calling the `codegen_unit` query with the right key. At
-        // that point we would already have re-done all the work we are trying to
-        // avoid doing in the first place.
-        // The solution is simple: Just explicitly call the `codegen_unit` query for
-        // each CGU, right after partitioning. This way `try_mark_green` will always
-        // hit the cache instead of having to go through `force_from_dep_node`.
-        // This assertion makes sure, we actually keep applying the solution above.
-        debug_assert!(
-            dep_node.kind != DepKind::codegen_unit,
-            "calling force_from_dep_node() on DepKind::codegen_unit"
-        );
-
-        (dep_node.kind.force_from_dep_node)(*self, dep_node)
-    }
-
-    fn has_errors_or_delayed_span_bugs(&self) -> bool {
-        self.sess.has_errors_or_delayed_span_bugs()
-    }
-
-    fn diagnostic(&self) -> &rustc_errors::Handler {
-        self.sess.diagnostic()
-    }
-
-    // Interactions with on_disk_cache
-    fn load_diagnostics(&self, prev_dep_node_index: SerializedDepNodeIndex) -> Vec<Diagnostic> {
-        self.on_disk_cache
-            .as_ref()
-            .map(|c| c.load_diagnostics(*self, prev_dep_node_index))
-            .unwrap_or_default()
-    }
-
-    fn store_diagnostics(&self, dep_node_index: DepNodeIndex, diagnostics: ThinVec<Diagnostic>) {
-        if let Some(c) = self.on_disk_cache.as_ref() {
-            c.store_diagnostics(dep_node_index, diagnostics)
-        }
-    }
-
-    fn store_diagnostics_for_anon_node(
-        &self,
-        dep_node_index: DepNodeIndex,
-        diagnostics: ThinVec<Diagnostic>,
-    ) {
-        if let Some(c) = self.on_disk_cache.as_ref() {
-            c.store_diagnostics_for_anon_node(dep_node_index, diagnostics)
-        }
-    }
-
     fn profiler(&self) -> &SelfProfilerRef {
         &self.prof
     }
 }
-
-fn def_id_corresponds_to_hir_dep_node(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
-    def_id == hir_id.owner
-}
diff --git a/compiler/rustc_middle/src/ty/query/mod.rs b/compiler/rustc_middle/src/ty/query/mod.rs
index c090e2de1f48a..970deb659face 100644
--- a/compiler/rustc_middle/src/ty/query/mod.rs
+++ b/compiler/rustc_middle/src/ty/query/mod.rs
@@ -119,6 +119,11 @@ impl TyCtxt<'tcx> {
     pub fn at(self, span: Span) -> TyCtxtAt<'tcx> {
         TyCtxtAt { tcx: self, span }
     }
+
+    pub fn try_mark_green(self, dep_node: &dep_graph::DepNode) -> bool {
+        let qcx = QueryCtxt { tcx: self, queries: self.queries };
+        self.dep_graph.try_mark_green(qcx, dep_node).is_some()
+    }
 }
 
 macro_rules! define_callbacks {
diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs
index c1068528a1823..23eb7ce324898 100644
--- a/compiler/rustc_middle/src/ty/query/plumbing.rs
+++ b/compiler/rustc_middle/src/ty/query/plumbing.rs
@@ -2,7 +2,7 @@
 //! generate the actual methods on tcx which find and execute the provider,
 //! manage the caches, and so forth.
 
-use crate::dep_graph;
+use crate::dep_graph::{self, DepKind, DepNode, DepNodeExt, DepNodeIndex, SerializedDepNodeIndex};
 use crate::ty::query::{on_disk_cache, Queries, Query};
 use crate::ty::tls::{self, ImplicitCtxt};
 use crate::ty::{self, TyCtxt};
@@ -72,6 +72,95 @@ impl QueryContext for QueryCtxt<'tcx> {
         (dep_node.kind.try_load_from_on_disk_cache)(*self, dep_node)
     }
 
+    fn try_force_from_dep_node(&self, dep_node: &DepNode) -> bool {
+        // FIXME: This match is just a workaround for incremental bugs and should
+        // be removed. https://github.com/rust-lang/rust/issues/62649 is one such
+        // bug that must be fixed before removing this.
+        match dep_node.kind {
+            DepKind::hir_owner | DepKind::hir_owner_nodes => {
+                if let Some(def_id) = dep_node.extract_def_id(**self) {
+                    let def_id = def_id.expect_local();
+                    let hir_id = self.tcx.hir().local_def_id_to_hir_id(def_id);
+                    if def_id != hir_id.owner {
+                        // This `DefPath` does not have a
+                        // corresponding `DepNode` (e.g. a
+                        // struct field), and the ` DefPath`
+                        // collided with the `DefPath` of a
+                        // proper item that existed in the
+                        // previous compilation session.
+                        //
+                        // Since the given `DefPath` does not
+                        // denote the item that previously
+                        // existed, we just fail to mark green.
+                        return false;
+                    }
+                } else {
+                    // If the node does not exist anymore, we
+                    // just fail to mark green.
+                    return false;
+                }
+            }
+            _ => {
+                // For other kinds of nodes it's OK to be
+                // forced.
+            }
+        }
+
+        debug!("try_force_from_dep_node({:?}) --- trying to force", dep_node);
+
+        // We must avoid ever having to call `force_from_dep_node()` for a
+        // `DepNode::codegen_unit`:
+        // Since we cannot reconstruct the query key of a `DepNode::codegen_unit`, we
+        // would always end up having to evaluate the first caller of the
+        // `codegen_unit` query that *is* reconstructible. This might very well be
+        // the `compile_codegen_unit` query, thus re-codegenning the whole CGU just
+        // to re-trigger calling the `codegen_unit` query with the right key. At
+        // that point we would already have re-done all the work we are trying to
+        // avoid doing in the first place.
+        // The solution is simple: Just explicitly call the `codegen_unit` query for
+        // each CGU, right after partitioning. This way `try_mark_green` will always
+        // hit the cache instead of having to go through `force_from_dep_node`.
+        // This assertion makes sure, we actually keep applying the solution above.
+        debug_assert!(
+            dep_node.kind != DepKind::codegen_unit,
+            "calling force_from_dep_node() on DepKind::codegen_unit"
+        );
+
+        (dep_node.kind.force_from_dep_node)(**self, dep_node)
+    }
+
+    fn has_errors_or_delayed_span_bugs(&self) -> bool {
+        self.sess.has_errors_or_delayed_span_bugs()
+    }
+
+    fn diagnostic(&self) -> &rustc_errors::Handler {
+        self.sess.diagnostic()
+    }
+
+    // Interactions with on_disk_cache
+    fn load_diagnostics(&self, prev_dep_node_index: SerializedDepNodeIndex) -> Vec<Diagnostic> {
+        self.on_disk_cache
+            .as_ref()
+            .map(|c| c.load_diagnostics(**self, prev_dep_node_index))
+            .unwrap_or_default()
+    }
+
+    fn store_diagnostics(&self, dep_node_index: DepNodeIndex, diagnostics: ThinVec<Diagnostic>) {
+        if let Some(c) = self.on_disk_cache.as_ref() {
+            c.store_diagnostics(dep_node_index, diagnostics)
+        }
+    }
+
+    fn store_diagnostics_for_anon_node(
+        &self,
+        dep_node_index: DepNodeIndex,
+        diagnostics: ThinVec<Diagnostic>,
+    ) {
+        if let Some(c) = self.on_disk_cache.as_ref() {
+            c.store_diagnostics_for_anon_node(dep_node_index, diagnostics)
+        }
+    }
+
     /// Executes a job by changing the `ImplicitCtxt` to point to the
     /// new query job while it executes. It returns the diagnostics
     /// captured during execution and the actual result.
diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs
index f1281c81c8b2f..6d7c29b229ade 100644
--- a/compiler/rustc_query_system/src/dep_graph/graph.rs
+++ b/compiler/rustc_query_system/src/dep_graph/graph.rs
@@ -587,7 +587,7 @@ impl<K: DepKind> DepGraph<K> {
     /// A node will have an index, when it's already been marked green, or when we can mark it
     /// green. This function will mark the current task as a reader of the specified node, when
     /// a node index can be found for that node.
-    pub fn try_mark_green_and_read<Ctxt: DepContext<DepKind = K>>(
+    pub fn try_mark_green_and_read<Ctxt: QueryContext<DepKind = K>>(
         &self,
         tcx: Ctxt,
         dep_node: &DepNode<K>,
@@ -599,7 +599,7 @@ impl<K: DepKind> DepGraph<K> {
         })
     }
 
-    pub fn try_mark_green<Ctxt: DepContext<DepKind = K>>(
+    pub fn try_mark_green<Ctxt: QueryContext<DepKind = K>>(
         &self,
         tcx: Ctxt,
         dep_node: &DepNode<K>,
@@ -627,7 +627,7 @@ impl<K: DepKind> DepGraph<K> {
     }
 
     /// Try to mark a dep-node which existed in the previous compilation session as green.
-    fn try_mark_previous_green<Ctxt: DepContext<DepKind = K>>(
+    fn try_mark_previous_green<Ctxt: QueryContext<DepKind = K>>(
         &self,
         tcx: Ctxt,
         data: &DepGraphData<K>,
@@ -811,7 +811,7 @@ impl<K: DepKind> DepGraph<K> {
     /// This may be called concurrently on multiple threads for the same dep node.
     #[cold]
     #[inline(never)]
-    fn emit_diagnostics<Ctxt: DepContext<DepKind = K>>(
+    fn emit_diagnostics<Ctxt: QueryContext<DepKind = K>>(
         &self,
         tcx: Ctxt,
         data: &DepGraphData<K>,
diff --git a/compiler/rustc_query_system/src/dep_graph/mod.rs b/compiler/rustc_query_system/src/dep_graph/mod.rs
index a1a6ca9dacc16..a647381fb03fa 100644
--- a/compiler/rustc_query_system/src/dep_graph/mod.rs
+++ b/compiler/rustc_query_system/src/dep_graph/mod.rs
@@ -13,8 +13,6 @@ pub use serialized::{SerializedDepGraph, SerializedDepNodeIndex};
 
 use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_data_structures::sync::Lock;
-use rustc_data_structures::thin_vec::ThinVec;
-use rustc_errors::Diagnostic;
 
 use std::fmt;
 use std::hash::Hash;
@@ -32,30 +30,8 @@ pub trait DepContext: Copy {
     /// Access the DepGraph.
     fn dep_graph(&self) -> &DepGraph<Self::DepKind>;
 
-    /// Try to force a dep node to execute and see if it's green.
-    fn try_force_from_dep_node(&self, dep_node: &DepNode<Self::DepKind>) -> bool;
-
     fn register_reused_dep_node(&self, dep_node: &DepNode<Self::DepKind>);
 
-    /// Return whether the current session is tainted by errors.
-    fn has_errors_or_delayed_span_bugs(&self) -> bool;
-
-    /// Return the diagnostic handler.
-    fn diagnostic(&self) -> &rustc_errors::Handler;
-
-    /// Load diagnostics associated to the node in the previous session.
-    fn load_diagnostics(&self, prev_dep_node_index: SerializedDepNodeIndex) -> Vec<Diagnostic>;
-
-    /// Register diagnostics for the given node, for use in next session.
-    fn store_diagnostics(&self, dep_node_index: DepNodeIndex, diagnostics: ThinVec<Diagnostic>);
-
-    /// Register diagnostics for the given node, for use in next session.
-    fn store_diagnostics_for_anon_node(
-        &self,
-        dep_node_index: DepNodeIndex,
-        diagnostics: ThinVec<Diagnostic>,
-    );
-
     /// Access the profiler.
     fn profiler(&self) -> &SelfProfilerRef;
 }
diff --git a/compiler/rustc_query_system/src/query/mod.rs b/compiler/rustc_query_system/src/query/mod.rs
index 8d5c9d7bea7e4..c935e1b9c5cd6 100644
--- a/compiler/rustc_query_system/src/query/mod.rs
+++ b/compiler/rustc_query_system/src/query/mod.rs
@@ -14,7 +14,7 @@ pub use self::caches::{
 mod config;
 pub use self::config::{QueryAccessors, QueryConfig, QueryDescription};
 
-use crate::dep_graph::{DepNode, HasDepContext};
+use crate::dep_graph::{DepNode, DepNodeIndex, HasDepContext, SerializedDepNodeIndex};
 use crate::query::job::QueryMap;
 
 use rustc_data_structures::stable_hasher::HashStable;
@@ -40,6 +40,28 @@ pub trait QueryContext: HasDepContext {
     /// Load data from the on-disk cache.
     fn try_load_from_on_disk_cache(&self, dep_node: &DepNode<Self::DepKind>);
 
+    /// Try to force a dep node to execute and see if it's green.
+    fn try_force_from_dep_node(&self, dep_node: &DepNode<Self::DepKind>) -> bool;
+
+    /// Return whether the current session is tainted by errors.
+    fn has_errors_or_delayed_span_bugs(&self) -> bool;
+
+    /// Return the diagnostic handler.
+    fn diagnostic(&self) -> &rustc_errors::Handler;
+
+    /// Load diagnostics associated to the node in the previous session.
+    fn load_diagnostics(&self, prev_dep_node_index: SerializedDepNodeIndex) -> Vec<Diagnostic>;
+
+    /// Register diagnostics for the given node, for use in next session.
+    fn store_diagnostics(&self, dep_node_index: DepNodeIndex, diagnostics: ThinVec<Diagnostic>);
+
+    /// Register diagnostics for the given node, for use in next session.
+    fn store_diagnostics_for_anon_node(
+        &self,
+        dep_node_index: DepNodeIndex,
+        diagnostics: ThinVec<Diagnostic>,
+    );
+
     /// Executes a job by changing the `ImplicitCtxt` to point to the
     /// new query job while it executes. It returns the diagnostics
     /// captured during execution and the actual result.
diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs
index eaaba10344a21..bfa62a5b045da 100644
--- a/compiler/rustc_query_system/src/query/plumbing.rs
+++ b/compiler/rustc_query_system/src/query/plumbing.rs
@@ -459,7 +459,7 @@ where
         tcx.dep_context().dep_graph().read_index(dep_node_index);
 
         if unlikely!(!diagnostics.is_empty()) {
-            tcx.dep_context().store_diagnostics_for_anon_node(dep_node_index, diagnostics);
+            tcx.store_diagnostics_for_anon_node(dep_node_index, diagnostics);
         }
 
         return job.complete(result, dep_node_index);
@@ -472,10 +472,7 @@ where
         // promoted to the current session during
         // `try_mark_green()`, so we can ignore them here.
         let loaded = tcx.start_query(job.id, None, || {
-            let marked = tcx
-                .dep_context()
-                .dep_graph()
-                .try_mark_green_and_read(*tcx.dep_context(), &dep_node);
+            let marked = tcx.dep_context().dep_graph().try_mark_green_and_read(tcx, &dep_node);
             marked.map(|(prev_dep_node_index, dep_node_index)| {
                 (
                     load_from_disk_and_cache_in_memory(
@@ -640,7 +637,7 @@ where
     prof_timer.finish_with_query_invocation_id(dep_node_index.into());
 
     if unlikely!(!diagnostics.is_empty()) && dep_node.kind != DepKind::NULL {
-        tcx.dep_context().store_diagnostics(dep_node_index, diagnostics);
+        tcx.store_diagnostics(dep_node_index, diagnostics);
     }
 
     let result = job.complete(result, dep_node_index);
@@ -675,7 +672,7 @@ where
 ///
 /// Note: The optimization is only available during incr. comp.
 #[inline(never)]
-fn ensure_must_run<CTX, K, V>(tcx: CTX::DepContext, key: &K, query: &QueryVtable<CTX, K, V>) -> bool
+fn ensure_must_run<CTX, K, V>(tcx: CTX, key: &K, query: &QueryVtable<CTX, K, V>) -> bool
 where
     K: crate::dep_graph::DepNodeParams<CTX::DepContext>,
     CTX: QueryContext,
@@ -687,9 +684,9 @@ where
     // Ensuring an anonymous query makes no sense
     assert!(!query.anon);
 
-    let dep_node = query.to_dep_node(tcx, key);
+    let dep_node = query.to_dep_node(*tcx.dep_context(), key);
 
-    match tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node) {
+    match tcx.dep_context().dep_graph().try_mark_green_and_read(tcx, &dep_node) {
         None => {
             // A None return from `try_mark_green_and_read` means that this is either
             // a new dep node or that the dep node has already been marked red.
@@ -700,7 +697,7 @@ where
             true
         }
         Some((_, dep_node_index)) => {
-            tcx.profiler().query_cache_hit(dep_node_index.into());
+            tcx.dep_context().profiler().query_cache_hit(dep_node_index.into());
             false
         }
     }
@@ -767,7 +764,7 @@ where
 {
     let query = &Q::VTABLE;
     if let QueryMode::Ensure = mode {
-        if !ensure_must_run(*tcx.dep_context(), &key, query) {
+        if !ensure_must_run(tcx, &key, query) {
             return None;
         }
     }

From c425641441eb17e389e92e83e07e9bd958e2f29d Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Tue, 19 Jan 2021 18:24:08 +0100
Subject: [PATCH 23/27] Use QueryCtxt in DepKindStruct.

---
 compiler/rustc_middle/src/dep_graph/dep_node.rs | 13 ++++---------
 compiler/rustc_middle/src/ty/query/plumbing.rs  |  4 ++--
 2 files changed, 6 insertions(+), 11 deletions(-)

diff --git a/compiler/rustc_middle/src/dep_graph/dep_node.rs b/compiler/rustc_middle/src/dep_graph/dep_node.rs
index 823bcebe23b14..063a57e484f32 100644
--- a/compiler/rustc_middle/src/dep_graph/dep_node.rs
+++ b/compiler/rustc_middle/src/dep_graph/dep_node.rs
@@ -135,7 +135,7 @@ pub struct DepKindStruct {
     /// then `force_from_dep_node()` should not fail for it. Otherwise, you can just
     /// add it to the "We don't have enough information to reconstruct..." group in
     /// the match below.
-    pub(crate) force_from_dep_node: fn(tcx: TyCtxt<'_>, dep_node: &DepNode) -> bool,
+    pub(crate) force_from_dep_node: fn(tcx: QueryCtxt<'_>, dep_node: &DepNode) -> bool,
 
     /// Invoke a query to put the on-disk cached value in memory.
     pub(crate) try_load_from_on_disk_cache: fn(QueryCtxt<'_>, &DepNode),
@@ -251,7 +251,7 @@ pub mod dep_kind {
                     <query_keys::$variant<'_> as DepNodeParams<TyCtxt<'_>>>::recover(tcx, dep_node)
                 }
 
-                fn force_from_dep_node(tcx: TyCtxt<'_>, dep_node: &DepNode) -> bool {
+                fn force_from_dep_node(tcx: QueryCtxt<'_>, dep_node: &DepNode) -> bool {
                     if is_anon {
                         return false;
                     }
@@ -260,13 +260,8 @@ pub mod dep_kind {
                         return false;
                     }
 
-                    if let Some(key) = recover(tcx, dep_node) {
-                        force_query::<queries::$variant<'_>, _>(
-                            QueryCtxt { tcx, queries: tcx.queries },
-                            key,
-                            DUMMY_SP,
-                            *dep_node
-                        );
+                    if let Some(key) = recover(*tcx, dep_node) {
+                        force_query::<queries::$variant<'_>, _>(tcx, key, DUMMY_SP, *dep_node);
                         return true;
                     }
 
diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs
index 23eb7ce324898..66ed3adb65ac0 100644
--- a/compiler/rustc_middle/src/ty/query/plumbing.rs
+++ b/compiler/rustc_middle/src/ty/query/plumbing.rs
@@ -68,7 +68,7 @@ impl QueryContext for QueryCtxt<'tcx> {
         self.queries.try_collect_active_jobs()
     }
 
-    fn try_load_from_on_disk_cache(&self, dep_node: &dep_graph::DepNode) {
+    fn try_load_from_on_disk_cache(&self, dep_node: &DepNode) {
         (dep_node.kind.try_load_from_on_disk_cache)(*self, dep_node)
     }
 
@@ -126,7 +126,7 @@ impl QueryContext for QueryCtxt<'tcx> {
             "calling force_from_dep_node() on DepKind::codegen_unit"
         );
 
-        (dep_node.kind.force_from_dep_node)(**self, dep_node)
+        (dep_node.kind.force_from_dep_node)(*self, dep_node)
     }
 
     fn has_errors_or_delayed_span_bugs(&self) -> bool {

From fbb37a68d5d6ab4b881095e57f281ade18e6e116 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Tue, 19 Jan 2021 19:07:06 +0100
Subject: [PATCH 24/27] Split DepKindStruct in two.

---
 .../rustc_middle/src/dep_graph/dep_node.rs    | 108 +------------
 compiler/rustc_middle/src/dep_graph/mod.rs    |   1 +
 compiler/rustc_middle/src/lib.rs              |   1 +
 compiler/rustc_middle/src/ty/query/mod.rs     |   1 +
 .../rustc_middle/src/ty/query/plumbing.rs     | 143 +++++++++++++++++-
 5 files changed, 151 insertions(+), 103 deletions(-)

diff --git a/compiler/rustc_middle/src/dep_graph/dep_node.rs b/compiler/rustc_middle/src/dep_graph/dep_node.rs
index 063a57e484f32..ba9d0a40732e6 100644
--- a/compiler/rustc_middle/src/dep_graph/dep_node.rs
+++ b/compiler/rustc_middle/src/dep_graph/dep_node.rs
@@ -55,7 +55,6 @@
 //!
 //! [dependency graph]: https://rustc-dev-guide.rust-lang.org/query.html
 
-use crate::ty::query::QueryCtxt;
 use crate::ty::TyCtxt;
 
 use rustc_data_structures::fingerprint::Fingerprint;
@@ -63,7 +62,6 @@ use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, CRATE_DEF_INDEX};
 use rustc_hir::definitions::DefPathHash;
 use rustc_hir::HirId;
 use rustc_span::symbol::Symbol;
-use rustc_span::DUMMY_SP;
 use std::hash::Hash;
 
 pub use rustc_query_system::dep_graph::{DepContext, DepNodeParams};
@@ -92,53 +90,6 @@ pub struct DepKindStruct {
     // FIXME: Make this a simple boolean once DepNodeParams::can_reconstruct_query_key
     // can be made a specialized associated const.
     can_reconstruct_query_key: fn() -> bool,
-
-    /// The red/green evaluation system will try to mark a specific DepNode in the
-    /// dependency graph as green by recursively trying to mark the dependencies of
-    /// that `DepNode` as green. While doing so, it will sometimes encounter a `DepNode`
-    /// where we don't know if it is red or green and we therefore actually have
-    /// to recompute its value in order to find out. Since the only piece of
-    /// information that we have at that point is the `DepNode` we are trying to
-    /// re-evaluate, we need some way to re-run a query from just that. This is what
-    /// `force_from_dep_node()` implements.
-    ///
-    /// In the general case, a `DepNode` consists of a `DepKind` and an opaque
-    /// GUID/fingerprint that will uniquely identify the node. This GUID/fingerprint
-    /// is usually constructed by computing a stable hash of the query-key that the
-    /// `DepNode` corresponds to. Consequently, it is not in general possible to go
-    /// back from hash to query-key (since hash functions are not reversible). For
-    /// this reason `force_from_dep_node()` is expected to fail from time to time
-    /// because we just cannot find out, from the `DepNode` alone, what the
-    /// corresponding query-key is and therefore cannot re-run the query.
-    ///
-    /// The system deals with this case letting `try_mark_green` fail which forces
-    /// the root query to be re-evaluated.
-    ///
-    /// Now, if `force_from_dep_node()` would always fail, it would be pretty useless.
-    /// Fortunately, we can use some contextual information that will allow us to
-    /// reconstruct query-keys for certain kinds of `DepNode`s. In particular, we
-    /// enforce by construction that the GUID/fingerprint of certain `DepNode`s is a
-    /// valid `DefPathHash`. Since we also always build a huge table that maps every
-    /// `DefPathHash` in the current codebase to the corresponding `DefId`, we have
-    /// everything we need to re-run the query.
-    ///
-    /// Take the `mir_promoted` query as an example. Like many other queries, it
-    /// just has a single parameter: the `DefId` of the item it will compute the
-    /// validated MIR for. Now, when we call `force_from_dep_node()` on a `DepNode`
-    /// with kind `MirValidated`, we know that the GUID/fingerprint of the `DepNode`
-    /// is actually a `DefPathHash`, and can therefore just look up the corresponding
-    /// `DefId` in `tcx.def_path_hash_to_def_id`.
-    ///
-    /// When you implement a new query, it will likely have a corresponding new
-    /// `DepKind`, and you'll have to support it here in `force_from_dep_node()`. As
-    /// a rule of thumb, if your query takes a `DefId` or `LocalDefId` as sole parameter,
-    /// then `force_from_dep_node()` should not fail for it. Otherwise, you can just
-    /// add it to the "We don't have enough information to reconstruct..." group in
-    /// the match below.
-    pub(crate) force_from_dep_node: fn(tcx: QueryCtxt<'_>, dep_node: &DepNode) -> bool,
-
-    /// Invoke a query to put the on-disk cached value in memory.
-    pub(crate) try_load_from_on_disk_cache: fn(QueryCtxt<'_>, &DepNode),
 }
 
 impl std::ops::Deref for DepKind {
@@ -197,8 +148,7 @@ macro_rules! contains_eval_always_attr {
 #[allow(non_upper_case_globals)]
 pub mod dep_kind {
     use super::*;
-    use crate::ty::query::{queries, query_keys};
-    use rustc_query_system::query::{force_query, QueryDescription};
+    use crate::ty::query::query_keys;
 
     // We use this for most things when incr. comp. is turned off.
     pub const Null: DepKindStruct = DepKindStruct {
@@ -207,8 +157,6 @@ pub mod dep_kind {
         is_eval_always: false,
 
         can_reconstruct_query_key: || true,
-        force_from_dep_node: |_, dep_node| bug!("force_from_dep_node: encountered {:?}", dep_node),
-        try_load_from_on_disk_cache: |_, _| {},
     };
 
     pub const TraitSelect: DepKindStruct = DepKindStruct {
@@ -217,8 +165,6 @@ pub mod dep_kind {
         is_eval_always: false,
 
         can_reconstruct_query_key: || true,
-        force_from_dep_node: |_, _| false,
-        try_load_from_on_disk_cache: |_, _| {},
     };
 
     pub const CompileCodegenUnit: DepKindStruct = DepKindStruct {
@@ -227,8 +173,6 @@ pub mod dep_kind {
         is_eval_always: false,
 
         can_reconstruct_query_key: || false,
-        force_from_dep_node: |_, _| false,
-        try_load_from_on_disk_cache: |_, _| {},
     };
 
     macro_rules! define_query_dep_kinds {
@@ -247,54 +191,11 @@ pub mod dep_kind {
                         ::can_reconstruct_query_key()
                 }
 
-                fn recover<'tcx>(tcx: TyCtxt<'tcx>, dep_node: &DepNode) -> Option<query_keys::$variant<'tcx>> {
-                    <query_keys::$variant<'_> as DepNodeParams<TyCtxt<'_>>>::recover(tcx, dep_node)
-                }
-
-                fn force_from_dep_node(tcx: QueryCtxt<'_>, dep_node: &DepNode) -> bool {
-                    if is_anon {
-                        return false;
-                    }
-
-                    if !can_reconstruct_query_key() {
-                        return false;
-                    }
-
-                    if let Some(key) = recover(*tcx, dep_node) {
-                        force_query::<queries::$variant<'_>, _>(tcx, key, DUMMY_SP, *dep_node);
-                        return true;
-                    }
-
-                    false
-                }
-
-                fn try_load_from_on_disk_cache(tcx: QueryCtxt<'_>, dep_node: &DepNode) {
-                    if is_anon {
-                        return
-                    }
-
-                    if !can_reconstruct_query_key() {
-                        return
-                    }
-
-                    debug_assert!(tcx.dep_graph
-                                     .node_color(dep_node)
-                                     .map(|c| c.is_green())
-                                     .unwrap_or(false));
-
-                    let key = recover(*tcx, dep_node).unwrap_or_else(|| panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash));
-                    if queries::$variant::cache_on_disk(tcx, &key, None) {
-                        let _ = tcx.$variant(key);
-                    }
-                }
-
                 DepKindStruct {
                     has_params,
                     is_anon,
                     is_eval_always,
                     can_reconstruct_query_key,
-                    force_from_dep_node,
-                    try_load_from_on_disk_cache,
                 }
             };)*
         );
@@ -310,7 +211,12 @@ macro_rules! define_dep_nodes {
         $variant:ident $(( $tuple_arg_ty:ty $(,)? ))*
       ,)*
     ) => (
-        static DEP_KINDS: &[DepKindStruct] = &[ $(dep_kind::$variant),* ];
+        #[macro_export]
+        macro_rules! make_dep_kind_array {
+            ($mod:ident) => {[ $(($mod::$variant),)* ]};
+        }
+
+        static DEP_KINDS: &[DepKindStruct] = &make_dep_kind_array!(dep_kind);
 
         /// This enum serves as an index into the `DEP_KINDS` array.
         #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Encodable, Decodable)]
diff --git a/compiler/rustc_middle/src/dep_graph/mod.rs b/compiler/rustc_middle/src/dep_graph/mod.rs
index da9b0b7e48af4..d862db2674ebd 100644
--- a/compiler/rustc_middle/src/dep_graph/mod.rs
+++ b/compiler/rustc_middle/src/dep_graph/mod.rs
@@ -3,6 +3,7 @@ use crate::ty::{self, TyCtxt};
 use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_data_structures::sync::Lock;
 
+#[macro_use]
 mod dep_node;
 
 pub use rustc_query_system::dep_graph::{
diff --git a/compiler/rustc_middle/src/lib.rs b/compiler/rustc_middle/src/lib.rs
index 6ae83a7f66750..d9e8826505002 100644
--- a/compiler/rustc_middle/src/lib.rs
+++ b/compiler/rustc_middle/src/lib.rs
@@ -76,6 +76,7 @@ pub mod query;
 
 #[macro_use]
 pub mod arena;
+#[macro_use]
 pub mod dep_graph;
 pub mod hir;
 pub mod ich;
diff --git a/compiler/rustc_middle/src/ty/query/mod.rs b/compiler/rustc_middle/src/ty/query/mod.rs
index 970deb659face..a2f3c6eedcb8b 100644
--- a/compiler/rustc_middle/src/ty/query/mod.rs
+++ b/compiler/rustc_middle/src/ty/query/mod.rs
@@ -62,6 +62,7 @@ use std::sync::Arc;
 #[macro_use]
 mod plumbing;
 pub use plumbing::QueryCtxt;
+use plumbing::QueryStruct;
 pub(crate) use rustc_query_system::query::CycleError;
 use rustc_query_system::query::*;
 
diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs
index 66ed3adb65ac0..18ae6318a0069 100644
--- a/compiler/rustc_middle/src/ty/query/plumbing.rs
+++ b/compiler/rustc_middle/src/ty/query/plumbing.rs
@@ -69,7 +69,8 @@ impl QueryContext for QueryCtxt<'tcx> {
     }
 
     fn try_load_from_on_disk_cache(&self, dep_node: &DepNode) {
-        (dep_node.kind.try_load_from_on_disk_cache)(*self, dep_node)
+        let cb = &super::QUERY_CALLBACKS[dep_node.kind as usize];
+        (cb.try_load_from_on_disk_cache)(*self, dep_node)
     }
 
     fn try_force_from_dep_node(&self, dep_node: &DepNode) -> bool {
@@ -126,7 +127,8 @@ impl QueryContext for QueryCtxt<'tcx> {
             "calling force_from_dep_node() on DepKind::codegen_unit"
         );
 
-        (dep_node.kind.force_from_dep_node)(*self, dep_node)
+        let cb = &super::QUERY_CALLBACKS[dep_node.kind as usize];
+        (cb.force_from_dep_node)(*self, dep_node)
     }
 
     fn has_errors_or_delayed_span_bugs(&self) -> bool {
@@ -307,6 +309,60 @@ impl<'tcx> Queries<'tcx> {
     }
 }
 
+/// This struct stores metadata about each Query.
+///
+/// Information is retrieved by indexing the `QUERIES` array using the integer value
+/// of the `DepKind`. Overall, this allows to implement `QueryContext` using this manual
+/// jump table instead of large matches.
+pub struct QueryStruct {
+    /// The red/green evaluation system will try to mark a specific DepNode in the
+    /// dependency graph as green by recursively trying to mark the dependencies of
+    /// that `DepNode` as green. While doing so, it will sometimes encounter a `DepNode`
+    /// where we don't know if it is red or green and we therefore actually have
+    /// to recompute its value in order to find out. Since the only piece of
+    /// information that we have at that point is the `DepNode` we are trying to
+    /// re-evaluate, we need some way to re-run a query from just that. This is what
+    /// `force_from_dep_node()` implements.
+    ///
+    /// In the general case, a `DepNode` consists of a `DepKind` and an opaque
+    /// GUID/fingerprint that will uniquely identify the node. This GUID/fingerprint
+    /// is usually constructed by computing a stable hash of the query-key that the
+    /// `DepNode` corresponds to. Consequently, it is not in general possible to go
+    /// back from hash to query-key (since hash functions are not reversible). For
+    /// this reason `force_from_dep_node()` is expected to fail from time to time
+    /// because we just cannot find out, from the `DepNode` alone, what the
+    /// corresponding query-key is and therefore cannot re-run the query.
+    ///
+    /// The system deals with this case letting `try_mark_green` fail which forces
+    /// the root query to be re-evaluated.
+    ///
+    /// Now, if `force_from_dep_node()` would always fail, it would be pretty useless.
+    /// Fortunately, we can use some contextual information that will allow us to
+    /// reconstruct query-keys for certain kinds of `DepNode`s. In particular, we
+    /// enforce by construction that the GUID/fingerprint of certain `DepNode`s is a
+    /// valid `DefPathHash`. Since we also always build a huge table that maps every
+    /// `DefPathHash` in the current codebase to the corresponding `DefId`, we have
+    /// everything we need to re-run the query.
+    ///
+    /// Take the `mir_promoted` query as an example. Like many other queries, it
+    /// just has a single parameter: the `DefId` of the item it will compute the
+    /// validated MIR for. Now, when we call `force_from_dep_node()` on a `DepNode`
+    /// with kind `MirValidated`, we know that the GUID/fingerprint of the `DepNode`
+    /// is actually a `DefPathHash`, and can therefore just look up the corresponding
+    /// `DefId` in `tcx.def_path_hash_to_def_id`.
+    ///
+    /// When you implement a new query, it will likely have a corresponding new
+    /// `DepKind`, and you'll have to support it here in `force_from_dep_node()`. As
+    /// a rule of thumb, if your query takes a `DefId` or `LocalDefId` as sole parameter,
+    /// then `force_from_dep_node()` should not fail for it. Otherwise, you can just
+    /// add it to the "We don't have enough information to reconstruct..." group in
+    /// the match below.
+    pub(crate) force_from_dep_node: fn(tcx: QueryCtxt<'_>, dep_node: &DepNode) -> bool,
+
+    /// Invoke a query to put the on-disk cached value in memory.
+    pub(crate) try_load_from_on_disk_cache: fn(QueryCtxt<'_>, &DepNode),
+}
+
 macro_rules! handle_cycle_error {
     ([][$tcx: expr, $error:expr]) => {{
         $tcx.report_cycle($error).emit();
@@ -545,6 +601,89 @@ macro_rules! define_queries {
             }
         })*
 
+        #[allow(non_upper_case_globals)]
+        pub mod query_callbacks {
+            use super::*;
+            use crate::dep_graph::DepNode;
+            use crate::ty::query::{queries, query_keys};
+            use rustc_query_system::dep_graph::DepNodeParams;
+            use rustc_query_system::query::{force_query, QueryDescription};
+
+            // We use this for most things when incr. comp. is turned off.
+            pub const Null: QueryStruct = QueryStruct {
+                force_from_dep_node: |_, dep_node| bug!("force_from_dep_node: encountered {:?}", dep_node),
+                try_load_from_on_disk_cache: |_, _| {},
+            };
+
+            pub const TraitSelect: QueryStruct = QueryStruct {
+                force_from_dep_node: |_, _| false,
+                try_load_from_on_disk_cache: |_, _| {},
+            };
+
+            pub const CompileCodegenUnit: QueryStruct = QueryStruct {
+                force_from_dep_node: |_, _| false,
+                try_load_from_on_disk_cache: |_, _| {},
+            };
+
+            $(pub const $name: QueryStruct = {
+                const is_anon: bool = is_anon!([$($modifiers)*]);
+
+                #[inline(always)]
+                fn can_reconstruct_query_key() -> bool {
+                    <query_keys::$name<'_> as DepNodeParams<TyCtxt<'_>>>
+                        ::can_reconstruct_query_key()
+                }
+
+                fn recover<'tcx>(tcx: TyCtxt<'tcx>, dep_node: &DepNode) -> Option<query_keys::$name<'tcx>> {
+                    <query_keys::$name<'_> as DepNodeParams<TyCtxt<'_>>>::recover(tcx, dep_node)
+                }
+
+                fn force_from_dep_node(tcx: QueryCtxt<'_>, dep_node: &DepNode) -> bool {
+                    if is_anon {
+                        return false;
+                    }
+
+                    if !can_reconstruct_query_key() {
+                        return false;
+                    }
+
+                    if let Some(key) = recover(*tcx, dep_node) {
+                        force_query::<queries::$name<'_>, _>(tcx, key, DUMMY_SP, *dep_node);
+                        return true;
+                    }
+
+                    false
+                }
+
+                fn try_load_from_on_disk_cache(tcx: QueryCtxt<'_>, dep_node: &DepNode) {
+                    if is_anon {
+                        return
+                    }
+
+                    if !can_reconstruct_query_key() {
+                        return
+                    }
+
+                    debug_assert!(tcx.dep_graph
+                                     .node_color(dep_node)
+                                     .map(|c| c.is_green())
+                                     .unwrap_or(false));
+
+                    let key = recover(*tcx, dep_node).unwrap_or_else(|| panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash));
+                    if queries::$name::cache_on_disk(tcx, &key, None) {
+                        let _ = tcx.$name(key);
+                    }
+                }
+
+                QueryStruct {
+                    force_from_dep_node,
+                    try_load_from_on_disk_cache,
+                }
+            };)*
+        }
+
+        static QUERY_CALLBACKS: &[QueryStruct] = &make_dep_kind_array!(query_callbacks);
+
         define_provider_struct! {
             tcx: $tcx,
             input: ($(([$($modifiers)*] [$name] [$($K)*] [$V]))*)

From 74223919685ab7c1e0fa761bd6b1eae89d15cf74 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Tue, 19 Jan 2021 19:43:59 +0100
Subject: [PATCH 25/27] Wrap QueryDescription into a macro.

---
 compiler/rustc_macros/src/query.rs            |  8 +++---
 compiler/rustc_middle/src/query/mod.rs        | 25 -------------------
 .../rustc_middle/src/ty/query/plumbing.rs     | 16 +++++++++---
 3 files changed, 17 insertions(+), 32 deletions(-)

diff --git a/compiler/rustc_macros/src/query.rs b/compiler/rustc_macros/src/query.rs
index e387020d82839..5326f0ae26966 100644
--- a/compiler/rustc_macros/src/query.rs
+++ b/compiler/rustc_macros/src/query.rs
@@ -344,7 +344,6 @@ fn add_query_description_impl(
     impls: &mut proc_macro2::TokenStream,
 ) {
     let name = &query.name;
-    let arg = &query.arg;
     let key = &query.key.0;
 
     // Find out if we should cache the query on disk
@@ -414,7 +413,7 @@ fn add_query_description_impl(
 
     let desc = quote! {
         #[allow(unused_variables)]
-        fn describe(tcx: QueryCtxt<'tcx>, key: #arg) -> String {
+        fn describe(tcx: QueryCtxt<'tcx>, key: Self::Key) -> String {
             let (#tcx, #key) = (*tcx, key);
             ::rustc_middle::ty::print::with_no_trimmed_paths(|| format!(#desc).into())
         }
@@ -520,7 +519,8 @@ pub fn rustc_queries(input: TokenStream) -> TokenStream {
                 $($macro)*(#cached_queries);
             }
         }
-
-        #query_description_stream
+        macro_rules! rustc_query_description {
+            () => { #query_description_stream }
+        }
     })
 }
diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs
index 9162003267117..d9abcf76ed09d 100644
--- a/compiler/rustc_middle/src/query/mod.rs
+++ b/compiler/rustc_middle/src/query/mod.rs
@@ -1,28 +1,3 @@
-use crate::dep_graph::SerializedDepNodeIndex;
-use crate::mir::interpret::{GlobalId, LitToConstInput};
-use crate::traits;
-use crate::traits::query::{
-    CanonicalPredicateGoal, CanonicalProjectionGoal, CanonicalTyGoal,
-    CanonicalTypeOpAscribeUserTypeGoal, CanonicalTypeOpEqGoal, CanonicalTypeOpNormalizeGoal,
-    CanonicalTypeOpProvePredicateGoal, CanonicalTypeOpSubtypeGoal,
-};
-use crate::ty::query::queries;
-use crate::ty::query::QueryCtxt;
-use crate::ty::subst::{GenericArg, SubstsRef};
-use crate::ty::{self, ParamEnvAnd, Ty, TyCtxt};
-use rustc_hir::def_id::{CrateNum, DefId, LocalDefId};
-use rustc_query_system::query::QueryDescription;
-
-use rustc_span::symbol::Symbol;
-
-fn describe_as_module(def_id: LocalDefId, tcx: TyCtxt<'_>) -> String {
-    if def_id.is_top_level_module() {
-        "top-level module".to_string()
-    } else {
-        format!("module `{}`", tcx.def_path_str(def_id.to_def_id()))
-    }
-}
-
 // Each of these queries corresponds to a function pointer field in the
 // `Providers` struct for requesting a value of that type, and a method
 // on `tcx: TyCtxt` (and `tcx.at(span)`) for doing that request in a way
diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs
index 18ae6318a0069..61bc4f3c1c446 100644
--- a/compiler/rustc_middle/src/ty/query/plumbing.rs
+++ b/compiler/rustc_middle/src/ty/query/plumbing.rs
@@ -3,19 +3,19 @@
 //! manage the caches, and so forth.
 
 use crate::dep_graph::{self, DepKind, DepNode, DepNodeExt, DepNodeIndex, SerializedDepNodeIndex};
-use crate::ty::query::{on_disk_cache, Queries, Query};
+use crate::ty::query::{on_disk_cache, queries, Queries, Query};
 use crate::ty::tls::{self, ImplicitCtxt};
 use crate::ty::{self, TyCtxt};
 use rustc_query_system::dep_graph::HasDepContext;
-use rustc_query_system::query::QueryContext;
 use rustc_query_system::query::{CycleError, QueryJobId, QueryJobInfo};
+use rustc_query_system::query::{QueryContext, QueryDescription};
 
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::Lock;
 use rustc_data_structures::thin_vec::ThinVec;
 use rustc_errors::{struct_span_err, Diagnostic, DiagnosticBuilder, Handler, Level};
 use rustc_serialize::opaque;
-use rustc_span::def_id::DefId;
+use rustc_span::def_id::{DefId, LocalDefId};
 use rustc_span::Span;
 
 #[derive(Copy, Clone)]
@@ -797,3 +797,13 @@ macro_rules! define_provider_struct {
         }
     };
 }
+
+fn describe_as_module(def_id: LocalDefId, tcx: TyCtxt<'_>) -> String {
+    if def_id.is_top_level_module() {
+        "top-level module".to_string()
+    } else {
+        format!("module `{}`", tcx.def_path_str(def_id.to_def_id()))
+    }
+}
+
+rustc_query_description! {}

From d15633b9a75fb78383beb42cdb8eae1f42d8af4e Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Tue, 19 Jan 2021 19:44:27 +0100
Subject: [PATCH 26/27] Move query names and Providers to parent module.

---
 compiler/rustc_middle/src/ty/query/mod.rs     | 66 ++++++++++++++++-
 .../rustc_middle/src/ty/query/plumbing.rs     | 70 -------------------
 2 files changed, 65 insertions(+), 71 deletions(-)

diff --git a/compiler/rustc_middle/src/ty/query/mod.rs b/compiler/rustc_middle/src/ty/query/mod.rs
index a2f3c6eedcb8b..faef1f048cc2b 100644
--- a/compiler/rustc_middle/src/ty/query/mod.rs
+++ b/compiler/rustc_middle/src/ty/query/mod.rs
@@ -127,11 +127,52 @@ impl TyCtxt<'tcx> {
     }
 }
 
+macro_rules! query_helper_param_ty {
+    (DefId) => { impl IntoQueryParam<DefId> };
+    ($K:ty) => { $K };
+}
+
 macro_rules! define_callbacks {
     (<$tcx:tt>
      $($(#[$attr:meta])*
         [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => {
 
+        // HACK(eddyb) this is like the `impl QueryConfig for queries::$name`
+        // below, but using type aliases instead of associated types, to bypass
+        // the limitations around normalizing under HRTB - for example, this:
+        // `for<'tcx> fn(...) -> <queries::$name<'tcx> as QueryConfig<TyCtxt<'tcx>>>::Value`
+        // doesn't currently normalize to `for<'tcx> fn(...) -> query_values::$name<'tcx>`.
+        // This is primarily used by the `provide!` macro in `rustc_metadata`.
+        #[allow(nonstandard_style, unused_lifetimes)]
+        pub mod query_keys {
+            use super::*;
+
+            $(pub type $name<$tcx> = $($K)*;)*
+        }
+        #[allow(nonstandard_style, unused_lifetimes)]
+        pub mod query_values {
+            use super::*;
+
+            $(pub type $name<$tcx> = $V;)*
+        }
+        #[allow(nonstandard_style, unused_lifetimes)]
+        pub mod query_storage {
+            use super::*;
+
+            $(pub type $name<$tcx> = query_storage!([$($modifiers)*][$($K)*, $V]);)*
+        }
+        #[allow(nonstandard_style, unused_lifetimes)]
+        pub mod query_stored {
+            use super::*;
+
+            $(pub type $name<$tcx> = <query_storage::$name<$tcx> as QueryStorage>::Stored;)*
+        }
+
+        #[derive(Default)]
+        pub struct QueryCaches<$tcx> {
+            $($(#[$attr])* $name: QueryCacheStore<query_storage::$name<$tcx>>,)*
+        }
+
         impl TyCtxtEnsure<$tcx> {
             $($(#[$attr])*
             #[inline(always)]
@@ -176,7 +217,30 @@ macro_rules! define_callbacks {
                 self.tcx.queries.$name(self.tcx, self.span, key, lookup, QueryMode::Get).unwrap()
             })*
         }
-    }
+
+        pub struct Providers {
+            $(pub $name: for<'tcx> fn(
+                TyCtxt<'tcx>,
+                query_keys::$name<'tcx>,
+            ) -> query_values::$name<'tcx>,)*
+        }
+
+        impl Default for Providers {
+            fn default() -> Self {
+                Providers {
+                    $($name: |_, key| bug!(
+                        "`tcx.{}({:?})` unsupported by its crate",
+                         stringify!($name), key
+                    ),)*
+                }
+            }
+        }
+
+        impl Copy for Providers {}
+        impl Clone for Providers {
+            fn clone(&self) -> Self { *self }
+        }
+    };
 }
 
 // Each of these queries corresponds to a function pointer field in the
diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs
index 61bc4f3c1c446..32ed7e7363510 100644
--- a/compiler/rustc_middle/src/ty/query/plumbing.rs
+++ b/compiler/rustc_middle/src/ty/query/plumbing.rs
@@ -430,11 +430,6 @@ macro_rules! hash_result {
     };
 }
 
-macro_rules! query_helper_param_ty {
-    (DefId) => { impl IntoQueryParam<DefId> };
-    ($K:ty) => { $K };
-}
-
 macro_rules! define_queries {
     (<$tcx:tt>
      $($(#[$attr:meta])*
@@ -512,42 +507,6 @@ macro_rules! define_queries {
             })*
         }
 
-        // HACK(eddyb) this is like the `impl QueryConfig for queries::$name`
-        // below, but using type aliases instead of associated types, to bypass
-        // the limitations around normalizing under HRTB - for example, this:
-        // `for<'tcx> fn(...) -> <queries::$name<'tcx> as QueryConfig<TyCtxt<'tcx>>>::Value`
-        // doesn't currently normalize to `for<'tcx> fn(...) -> query_values::$name<'tcx>`.
-        // This is primarily used by the `provide!` macro in `rustc_metadata`.
-        #[allow(nonstandard_style, unused_lifetimes)]
-        pub mod query_keys {
-            use super::*;
-
-            $(pub type $name<$tcx> = $($K)*;)*
-        }
-        #[allow(nonstandard_style, unused_lifetimes)]
-        pub mod query_values {
-            use super::*;
-
-            $(pub type $name<$tcx> = $V;)*
-        }
-        #[allow(nonstandard_style, unused_lifetimes)]
-        pub mod query_storage {
-            use super::*;
-
-            $(pub type $name<$tcx> = query_storage!([$($modifiers)*][$($K)*, $V]);)*
-        }
-        #[allow(nonstandard_style, unused_lifetimes)]
-        pub mod query_stored {
-            use super::*;
-
-            $(pub type $name<$tcx> = <query_storage::$name<$tcx> as QueryStorage>::Stored;)*
-        }
-
-        #[derive(Default)]
-        pub struct QueryCaches<$tcx> {
-            $($(#[$attr])* $name: QueryCacheStore<query_storage::$name<$tcx>>,)*
-        }
-
         $(impl<$tcx> QueryConfig for queries::$name<$tcx> {
             type Key = $($K)*;
             type Value = $V;
@@ -683,11 +642,6 @@ macro_rules! define_queries {
         }
 
         static QUERY_CALLBACKS: &[QueryStruct] = &make_dep_kind_array!(query_callbacks);
-
-        define_provider_struct! {
-            tcx: $tcx,
-            input: ($(([$($modifiers)*] [$name] [$($K)*] [$V]))*)
-        }
     }
 }
 
@@ -774,30 +728,6 @@ macro_rules! define_queries_struct {
     };
 }
 
-macro_rules! define_provider_struct {
-    (tcx: $tcx:tt,
-     input: ($(([$($modifiers:tt)*] [$name:ident] [$K:ty] [$R:ty]))*)) => {
-        pub struct Providers {
-            $(pub $name: for<$tcx> fn(TyCtxt<$tcx>, $K) -> $R,)*
-        }
-
-        impl Default for Providers {
-            fn default() -> Self {
-                $(fn $name<$tcx>(_: TyCtxt<$tcx>, key: $K) -> $R {
-                    bug!("`tcx.{}({:?})` unsupported by its crate",
-                         stringify!($name), key);
-                })*
-                Providers { $($name),* }
-            }
-        }
-
-        impl Copy for Providers {}
-        impl Clone for Providers {
-            fn clone(&self) -> Self { *self }
-        }
-    };
-}
-
 fn describe_as_module(def_id: LocalDefId, tcx: TyCtxt<'_>) -> String {
     if def_id.is_top_level_module() {
         "top-level module".to_string()

From b0927afef2efcbe9685ec595d1e0eddfe371b500 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Tue, 19 Jan 2021 20:04:40 +0100
Subject: [PATCH 27/27] Make encode_query_results more generic.

---
 .../rustc_middle/src/ty/query/on_disk_cache.rs     | 14 ++++++++------
 compiler/rustc_middle/src/ty/query/plumbing.rs     |  2 +-
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs
index 9834bb407e32d..adeeb309205d2 100644
--- a/compiler/rustc_middle/src/ty/query/on_disk_cache.rs
+++ b/compiler/rustc_middle/src/ty/query/on_disk_cache.rs
@@ -3,7 +3,6 @@ use crate::mir::interpret::{AllocDecodingSession, AllocDecodingState};
 use crate::mir::{self, interpret};
 use crate::ty::codec::{RefDecodable, TyDecoder, TyEncoder};
 use crate::ty::context::TyCtxt;
-use crate::ty::query::QueryCtxt;
 use crate::ty::{self, Ty};
 use rustc_data_structures::fingerprint::{Fingerprint, FingerprintDecoder, FingerprintEncoder};
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet};
@@ -15,6 +14,8 @@ use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, LOCAL_CRATE};
 use rustc_hir::definitions::DefPathHash;
 use rustc_hir::definitions::Definitions;
 use rustc_index::vec::{Idx, IndexVec};
+use rustc_query_system::dep_graph::DepContext;
+use rustc_query_system::query::QueryContext;
 use rustc_serialize::{
     opaque::{self, FileEncodeResult, FileEncoder},
     Decodable, Decoder, Encodable, Encoder,
@@ -1215,18 +1216,19 @@ impl<'a> Decodable<opaque::Decoder<'a>> for IntEncodedWithFixedSize {
     }
 }
 
-pub fn encode_query_results<'a, 'tcx, Q>(
-    tcx: QueryCtxt<'tcx>,
+pub fn encode_query_results<'a, 'tcx, CTX, Q>(
+    tcx: CTX,
     encoder: &mut CacheEncoder<'a, 'tcx, FileEncoder>,
     query_result_index: &mut EncodedQueryResultIndex,
 ) -> FileEncodeResult
 where
-    Q: super::QueryDescription<QueryCtxt<'tcx>> + super::QueryAccessors<QueryCtxt<'tcx>>,
+    CTX: QueryContext + 'tcx,
+    Q: super::QueryDescription<CTX> + super::QueryAccessors<CTX>,
     Q::Value: Encodable<CacheEncoder<'a, 'tcx, FileEncoder>>,
 {
     let _timer = tcx
-        .sess
-        .prof
+        .dep_context()
+        .profiler()
         .extra_verbose_generic_activity("encode_query_results_for", std::any::type_name::<Q>());
 
     assert!(Q::query_state(tcx).all_inactive());
diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs
index 32ed7e7363510..53bac4181e67b 100644
--- a/compiler/rustc_middle/src/ty/query/plumbing.rs
+++ b/compiler/rustc_middle/src/ty/query/plumbing.rs
@@ -251,7 +251,7 @@ impl<'tcx> QueryCtxt<'tcx> {
         macro_rules! encode_queries {
             ($($query:ident,)*) => {
                 $(
-                    on_disk_cache::encode_query_results::<ty::query::queries::$query<'_>>(
+                    on_disk_cache::encode_query_results::<_, ty::query::queries::$query<'_>>(
                         self,
                         encoder,
                         query_result_index