diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs
index 6b4f08d9f9335..6b67d509da00d 100644
--- a/compiler/rustc_middle/src/ty/sty.rs
+++ b/compiler/rustc_middle/src/ty/sty.rs
@@ -1837,6 +1837,15 @@ impl<'tcx> TyS<'tcx> {
         )
     }
 
+    /// Get the mutability of the reference or `None` when not a reference
+    #[inline]
+    pub fn ref_mutability(&self) -> Option<hir::Mutability> {
+        match self.kind() {
+            Ref(_, _, mutability) => Some(*mutability),
+            _ => None,
+        }
+    }
+
     #[inline]
     pub fn is_unsafe_ptr(&self) -> bool {
         matches!(self.kind(), RawPtr(_))
diff --git a/compiler/rustc_trait_selection/src/traits/coherence.rs b/compiler/rustc_trait_selection/src/traits/coherence.rs
index 99b96f6096476..e8ae1f44a3671 100644
--- a/compiler/rustc_trait_selection/src/traits/coherence.rs
+++ b/compiler/rustc_trait_selection/src/traits/coherence.rs
@@ -11,7 +11,7 @@ use crate::traits::{self, Normalized, Obligation, ObligationCause, SelectionCont
 use rustc_hir::def_id::{DefId, LOCAL_CRATE};
 use rustc_middle::ty::fold::TypeFoldable;
 use rustc_middle::ty::subst::Subst;
-use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_middle::ty::{self, fast_reject, Ty, TyCtxt};
 use rustc_span::symbol::sym;
 use rustc_span::DUMMY_SP;
 use std::iter;
@@ -67,6 +67,34 @@ where
            impl2_def_id={:?})",
         impl1_def_id, impl2_def_id,
     );
+    // Before doing expensive operations like entering an inference context, do
+    // a quick check via fast_reject to tell if the impl headers could possibly
+    // unify.
+    let impl1_ref = tcx.impl_trait_ref(impl1_def_id);
+    let impl2_ref = tcx.impl_trait_ref(impl2_def_id);
+
+    // Check if any of the input types definitely do not unify.
+    if impl1_ref
+        .iter()
+        .flat_map(|tref| tref.substs.types())
+        .zip(impl2_ref.iter().flat_map(|tref| tref.substs.types()))
+        .any(|(ty1, ty2)| {
+            let t1 = fast_reject::simplify_type(tcx, ty1, false);
+            let t2 = fast_reject::simplify_type(tcx, ty2, false);
+            if let (Some(t1), Some(t2)) = (t1, t2) {
+                // Simplified successfully
+                // Types cannot unify if they differ in their reference mutability or simplify to different types
+                t1 != t2 || ty1.ref_mutability() != ty2.ref_mutability()
+            } else {
+                // Types might unify
+                false
+            }
+        })
+    {
+        // Some types involved are definitely different, so the impls couldn't possibly overlap.
+        debug!("overlapping_impls: fast_reject early-exit");
+        return no_overlap();
+    }
 
     let overlaps = tcx.infer_ctxt().enter(|infcx| {
         let selcx = &mut SelectionContext::intercrate(&infcx);