compiler: report error when trait object type param reference self

compiler/rustc_hir_analysis/messages.ftl

@@ -486,6 +486,9 @@ hir_analysis_self_in_impl_self =
     `Self` is not valid in the self type of an impl block
     .note = replace `Self` with a different type
 
+hir_analysis_self_in_type_alias = `Self` is not allowed in type aliases
+    .label = `Self` is only available in impls, traits, and concrete type definitions
+
 hir_analysis_self_ty_not_captured = `impl Trait` must mention the `Self` type of the trait in `use<...>`
     .label = `Self` type parameter is implicitly captured by this `impl Trait`
     .note = currently, all type parameters are required to be mentioned in the precise captures list

compiler/rustc_hir_analysis/src/errors.rs

@@ -1707,3 +1707,11 @@ pub(crate) enum SupertraitItemShadowee {
         traits: DiagSymbolList,
     },
 }
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_self_in_type_alias, code = E0411)]
+pub(crate) struct SelfInTypeAlias {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+}

compiler/rustc_hir_analysis/src/hir_ty_lowering/dyn_compatibility.rs

@@ -16,6 +16,7 @@ use smallvec::{SmallVec, smallvec};
 use tracing::{debug, instrument};
 
 use super::HirTyLowerer;
+use crate::errors::SelfInTypeAlias;
 use crate::hir_ty_lowering::{
     GenericArgCountMismatch, GenericArgCountResult, PredicateFilter, RegionInferReason,
 };
@@ -113,6 +114,19 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
         // ```
         let mut projection_bounds = FxIndexMap::default();
         for (proj, proj_span) in elaborated_projection_bounds {
+            let proj = proj.map_bound(|mut b| {
+                if let Some(term_ty) = &b.term.as_type() {
+                    let references_self = term_ty.walk().any(|arg| arg == dummy_self.into());
+                    if references_self {
+                        // With trait alias and type alias combined, type resolver
+                        // may not be able to catch all illegal `Self` usages (issue 139082)
+                        let guar = tcx.dcx().emit_err(SelfInTypeAlias { span });
+                        b.term = replace_dummy_self_with_error(tcx, b.term, guar);
+                    }
+                }
+                b
+            });
+
             let key = (
                 proj.skip_binder().projection_term.def_id,
                 tcx.anonymize_bound_vars(

tests/ui/dyn-compatibility/trait-alias-self-projection.rs

@@ -0,0 +1,12 @@
+#![feature(trait_alias)]
+trait B = Fn() -> Self;
+type D = &'static dyn B;
+//~^ ERROR E0411
+
+fn a() -> D {
+    unreachable!();
+}
+
+fn main() {
+    _ = a();
+}

tests/ui/dyn-compatibility/trait-alias-self-projection.stderr

@@ -0,0 +1,9 @@
+error[E0411]: `Self` is not allowed in type aliases
+  --> $DIR/trait-alias-self-projection.rs:3:19
+   |
+LL | type D = &'static dyn B;
+   |                   ^^^^^ `Self` is only available in impls, traits, and concrete type definitions
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0411`.