Rollup of 7 pull requests

compiler/rustc_codegen_cranelift/src/base.rs

@@ -677,21 +677,22 @@ fn codegen_stmt<'tcx>(
                     CastKind::PointerCoercion(PointerCoercion::UnsafeFnPointer),
                     ref operand,
                     to_ty,
-                )
-                | Rvalue::Cast(
-                    CastKind::PointerCoercion(PointerCoercion::MutToConstPointer),
-                    ref operand,
-                    to_ty,
-                )
-                | Rvalue::Cast(
-                    CastKind::PointerCoercion(PointerCoercion::ArrayToPointer),
-                    ref operand,
-                    to_ty,
                 ) => {
                     let to_layout = fx.layout_of(fx.monomorphize(to_ty));
                     let operand = codegen_operand(fx, operand);
                     lval.write_cvalue(fx, operand.cast_pointer_to(to_layout));
                 }
+                Rvalue::Cast(
+                    CastKind::PointerCoercion(
+                        PointerCoercion::MutToConstPointer | PointerCoercion::ArrayToPointer,
+                    ),
+                    ..,
+                ) => {
+                    bug!(
+                        "{:?} is for borrowck, and should never appear in codegen",
+                        to_place_and_rval.1
+                    );
+                }
                 Rvalue::Cast(
                     CastKind::IntToInt
                     | CastKind::FloatToFloat

compiler/rustc_codegen_ssa/src/back/symbol_export.rs

@@ -371,7 +371,7 @@ fn exported_symbols_provider_local(
                 }) => {
                     // A little sanity-check
                     debug_assert_eq!(
-                        args.non_erasable_generics(tcx, def_id).skip(1).next(),
+                        args.non_erasable_generics(tcx, def_id).next(),
                         Some(GenericArgKind::Type(ty))
                     );
                     symbols.push((
@@ -422,10 +422,7 @@ fn upstream_monomorphizations_provider(
                 }
                 ExportedSymbol::AsyncDropGlueCtorShim(ty) => {
                     if let Some(async_drop_in_place_fn_def_id) = async_drop_in_place_fn_def_id {
-                        (
-                            async_drop_in_place_fn_def_id,
-                            tcx.mk_args(&[tcx.lifetimes.re_erased.into(), ty.into()]),
-                        )
+                        (async_drop_in_place_fn_def_id, tcx.mk_args(&[ty.into()]))
                     } else {
                         // `drop_in_place` in place does not exist, don't try
                         // to use it.
@@ -473,11 +470,16 @@ fn upstream_drop_glue_for_provider<'tcx>(
     tcx: TyCtxt<'tcx>,
     args: GenericArgsRef<'tcx>,
 ) -> Option<CrateNum> {
-    if let Some(def_id) = tcx.lang_items().drop_in_place_fn() {
-        tcx.upstream_monomorphizations_for(def_id).and_then(|monos| monos.get(&args).cloned())
-    } else {
-        None
-    }
+    let def_id = tcx.lang_items().drop_in_place_fn()?;
+    tcx.upstream_monomorphizations_for(def_id)?.get(&args).cloned()
+}
+
+fn upstream_async_drop_glue_for_provider<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    args: GenericArgsRef<'tcx>,
+) -> Option<CrateNum> {
+    let def_id = tcx.lang_items().async_drop_in_place_fn()?;
+    tcx.upstream_monomorphizations_for(def_id)?.get(&args).cloned()
 }
 
 fn is_unreachable_local_definition_provider(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
@@ -491,6 +493,7 @@ pub fn provide(providers: &mut Providers) {
     providers.upstream_monomorphizations = upstream_monomorphizations_provider;
     providers.is_unreachable_local_definition = is_unreachable_local_definition_provider;
     providers.upstream_drop_glue_for = upstream_drop_glue_for_provider;
+    providers.upstream_async_drop_glue_for = upstream_async_drop_glue_for_provider;
     providers.wasm_import_module_map = wasm_import_module_map;
     providers.extern_queries.is_reachable_non_generic = is_reachable_non_generic_provider_extern;
     providers.extern_queries.upstream_monomorphizations_for =

compiler/rustc_codegen_ssa/src/mir/rvalue.rs

@@ -456,8 +456,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                             base::unsize_ptr(bx, lldata, operand.layout.ty, cast.ty, llextra);
                         OperandValue::Pair(lldata, llextra)
                     }
-                    mir::CastKind::PointerCoercion(PointerCoercion::MutToConstPointer)
-                    | mir::CastKind::PtrToPtr
+                    mir::CastKind::PointerCoercion(
+                        PointerCoercion::MutToConstPointer | PointerCoercion::ArrayToPointer,
+                    ) => {
+                        bug!("{kind:?} is for borrowck, and should never appear in codegen");
+                    }
+                    mir::CastKind::PtrToPtr
                         if bx.cx().is_backend_scalar_pair(operand.layout) =>
                     {
                         if let OperandValue::Pair(data_ptr, meta) = operand.val {
@@ -477,9 +481,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                             base::cast_to_dyn_star(bx, lldata, operand.layout, cast.ty, llextra);
                         OperandValue::Pair(lldata, llextra)
                     }
-                    mir::CastKind::PointerCoercion(
-                        PointerCoercion::MutToConstPointer | PointerCoercion::ArrayToPointer,
-                    )
                     | mir::CastKind::IntToInt
                     | mir::CastKind::FloatToInt
                     | mir::CastKind::FloatToFloat

compiler/rustc_const_eval/src/interpret/cast.rs

@@ -70,9 +70,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             CastKind::PointerCoercion(
                 PointerCoercion::MutToConstPointer | PointerCoercion::ArrayToPointer,
             ) => {
-                // These are NOPs, but can be wide pointers.
-                let v = self.read_immediate(src)?;
-                self.write_immediate(*v, dest)?;
+                bug!("{cast_kind:?} casts are for borrowck only, not runtime MIR");
             }
 
             CastKind::PointerCoercion(PointerCoercion::ReifyFnPointer) => {

compiler/rustc_middle/src/mir/syntax.rs

@@ -127,6 +127,9 @@ pub enum AnalysisPhase {
     /// * [`StatementKind::AscribeUserType`]
     /// * [`StatementKind::Coverage`] with [`CoverageKind::BlockMarker`] or [`CoverageKind::SpanMarker`]
     /// * [`Rvalue::Ref`] with `BorrowKind::Fake`
+    /// * [`CastKind::PointerCoercion`] with any of the following:
+    ///   * [`PointerCoercion::ArrayToPointer`]
+    ///   * [`PointerCoercion::MutToConstPointer`]
     ///
     /// Furthermore, `Deref` projections must be the first projection within any place (if they
     /// appear at all)
@@ -1281,8 +1284,7 @@ pub enum Rvalue<'tcx> {
     ///
     /// This allows for casts from/to a variety of types.
     ///
-    /// **FIXME**: Document exactly which `CastKind`s allow which types of casts. Figure out why
-    /// `ArrayToPointer` and `MutToConstPointer` are special.
+    /// **FIXME**: Document exactly which `CastKind`s allow which types of casts.
     Cast(CastKind, Operand<'tcx>, Ty<'tcx>),
 
     /// * `Offset` has the same semantics as [`offset`](pointer::offset), except that the second
@@ -1362,6 +1364,13 @@ pub enum CastKind {
     PointerWithExposedProvenance,
     /// Pointer related casts that are done by coercions. Note that reference-to-raw-ptr casts are
     /// translated into `&raw mut/const *r`, i.e., they are not actually casts.
+    ///
+    /// The following are allowed in [`AnalysisPhase::Initial`] as they're needed for borrowck,
+    /// but after that are forbidden (including in all phases of runtime MIR):
+    /// * [`PointerCoercion::ArrayToPointer`]
+    /// * [`PointerCoercion::MutToConstPointer`]
+    ///
+    /// Both are runtime nops, so should be [`CastKind::PtrToPtr`] instead in runtime MIR.
     PointerCoercion(PointerCoercion),
     /// Cast into a dyn* object.
     DynStar,

compiler/rustc_middle/src/query/mod.rs

@@ -1554,12 +1554,13 @@ rustc_queries! {
         }
     }
 
-    /// The entire set of monomorphizations the local crate can safely link
-    /// to because they are exported from upstream crates. Do not depend on
-    /// this directly, as its value changes anytime a monomorphization gets
-    /// added or removed in any upstream crate. Instead use the narrower
-    /// `upstream_monomorphizations_for`, `upstream_drop_glue_for`, or, even
-    /// better, `Instance::upstream_monomorphization()`.
+    /// The entire set of monomorphizations the local crate can safely
+    /// link to because they are exported from upstream crates. Do
+    /// not depend on this directly, as its value changes anytime
+    /// a monomorphization gets added or removed in any upstream
+    /// crate. Instead use the narrower `upstream_monomorphizations_for`,
+    /// `upstream_drop_glue_for`, `upstream_async_drop_glue_for`, or,
+    /// even better, `Instance::upstream_monomorphization()`.
     query upstream_monomorphizations(_: ()) -> &'tcx DefIdMap<UnordMap<GenericArgsRef<'tcx>, CrateNum>> {
         arena_cache
         desc { "collecting available upstream monomorphizations" }
@@ -1601,6 +1602,26 @@ rustc_queries! {
         desc { "available upstream drop-glue for `{:?}`", args }
     }
 
+    /// Returns the upstream crate that exports async-drop-glue for
+    /// the given type (`args` is expected to be a single-item list
+    /// containing the type one wants async-drop-glue for).
+    ///
+    /// This is a subset of `upstream_monomorphizations_for` in order
+    /// to increase dep-tracking granularity. Otherwise adding or
+    /// removing any type with async-drop-glue in any upstream crate
+    /// would invalidate all functions calling async-drop-glue of an
+    /// upstream type.
+    ///
+    /// You likely want to call `Instance::upstream_monomorphization()`
+    /// instead of invoking this query directly.
+    ///
+    /// NOTE: This query could easily be extended to also support other
+    ///       common functions that have are large set of monomorphizations
+    ///       (like `Clone::clone` for example).
+    query upstream_async_drop_glue_for(args: GenericArgsRef<'tcx>) -> Option<CrateNum> {
+        desc { "available upstream async-drop-glue for `{:?}`", args }
+    }
+
     /// Returns a list of all `extern` blocks of a crate.
     query foreign_modules(_: CrateNum) -> &'tcx FxIndexMap<DefId, ForeignModule> {
         arena_cache

compiler/rustc_middle/src/ty/instance.rs

@@ -219,8 +219,9 @@ impl<'tcx> Instance<'tcx> {
             InstanceKind::Item(def) => tcx
                 .upstream_monomorphizations_for(def)
                 .and_then(|monos| monos.get(&self.args).cloned()),
-            InstanceKind::DropGlue(_, Some(_)) | InstanceKind::AsyncDropGlueCtorShim(_, _) => {
-                tcx.upstream_drop_glue_for(self.args)
+            InstanceKind::DropGlue(_, Some(_)) => tcx.upstream_drop_glue_for(self.args),
+            InstanceKind::AsyncDropGlueCtorShim(_, Some(_)) => {
+                tcx.upstream_async_drop_glue_for(self.args)
             }
             _ => None,
         }
@@ -256,7 +257,7 @@ impl<'tcx> InstanceKind<'tcx> {
         match self {
             ty::InstanceKind::Item(def) => Some(def),
             ty::InstanceKind::DropGlue(def_id, Some(_))
-            | InstanceKind::AsyncDropGlueCtorShim(def_id, _)
+            | InstanceKind::AsyncDropGlueCtorShim(def_id, Some(_))
             | InstanceKind::ThreadLocalShim(def_id) => Some(def_id),
             InstanceKind::VTableShim(..)
             | InstanceKind::ReifyShim(..)
@@ -267,6 +268,7 @@ impl<'tcx> InstanceKind<'tcx> {
             | ty::InstanceKind::ConstructCoroutineInClosureShim { .. }
             | ty::InstanceKind::CoroutineKindShim { .. }
             | InstanceKind::DropGlue(..)
+            | InstanceKind::AsyncDropGlueCtorShim(..)
             | InstanceKind::CloneShim(..)
             | InstanceKind::FnPtrAddrShim(..) => None,
         }
@@ -312,7 +314,9 @@ impl<'tcx> InstanceKind<'tcx> {
         if self.requires_inline(tcx) {
             return true;
         }
-        if let ty::InstanceKind::DropGlue(.., Some(ty)) = *self {
+        if let ty::InstanceKind::DropGlue(.., Some(ty))
+        | ty::InstanceKind::AsyncDropGlueCtorShim(.., Some(ty)) = *self
+        {
             // Drop glue generally wants to be instantiated at every codegen
             // unit, but without an #[inline] hint. We should make this
             // available to normal end-users.
@@ -327,9 +331,14 @@ impl<'tcx> InstanceKind<'tcx> {
             // drops of `Option::None` before LTO. We also respect the intent of
             // `#[inline]` on `Drop::drop` implementations.
             return ty.ty_adt_def().map_or(true, |adt_def| {
-                adt_def
-                    .destructor(tcx)
-                    .map_or_else(|| adt_def.is_enum(), |dtor| tcx.cross_crate_inlinable(dtor.did))
+                match *self {
+                    ty::InstanceKind::DropGlue(..) => adt_def.destructor(tcx).map(|dtor| dtor.did),
+                    ty::InstanceKind::AsyncDropGlueCtorShim(..) => {
+                        adt_def.async_destructor(tcx).map(|dtor| dtor.ctor)
+                    }
+                    _ => unreachable!(),
+                }
+                .map_or_else(|| adt_def.is_enum(), |did| tcx.cross_crate_inlinable(did))
             });
         }
         if let ty::InstanceKind::ThreadLocalShim(..) = *self {

compiler/rustc_middle/src/ty/print/pretty.rs

@@ -7,7 +7,7 @@ use crate::ty::{
     ConstInt, Expr, ParamConst, ScalarInt, Term, TermKind, TypeFoldable, TypeSuperFoldable,
     TypeSuperVisitable, TypeVisitable, TypeVisitableExt,
 };
-use rustc_apfloat::ieee::{Double, Single};
+use rustc_apfloat::ieee::{Double, Half, Quad, Single};
 use rustc_apfloat::Float;
 use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
 use rustc_data_structures::unord::UnordMap;
@@ -1710,6 +1710,10 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
             ty::Bool if int == ScalarInt::FALSE => p!("false"),
             ty::Bool if int == ScalarInt::TRUE => p!("true"),
             // Float
+            ty::Float(ty::FloatTy::F16) => {
+                let val = Half::try_from(int).unwrap();
+                p!(write("{}{}f16", val, if val.is_finite() { "" } else { "_" }))
+            }
             ty::Float(ty::FloatTy::F32) => {
                 let val = Single::try_from(int).unwrap();
                 p!(write("{}{}f32", val, if val.is_finite() { "" } else { "_" }))
@@ -1718,6 +1722,10 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
                 let val = Double::try_from(int).unwrap();
                 p!(write("{}{}f64", val, if val.is_finite() { "" } else { "_" }))
             }
+            ty::Float(ty::FloatTy::F128) => {
+                let val = Quad::try_from(int).unwrap();
+                p!(write("{}{}f128", val, if val.is_finite() { "" } else { "_" }))
+            }
             // Int
             ty::Uint(_) | ty::Int(_) => {
                 let int =

compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs

@@ -18,7 +18,8 @@
 
 use crate::MirPass;
 use rustc_middle::mir::coverage::CoverageKind;
-use rustc_middle::mir::{Body, BorrowKind, Rvalue, StatementKind, TerminatorKind};
+use rustc_middle::mir::{Body, BorrowKind, CastKind, Rvalue, StatementKind, TerminatorKind};
+use rustc_middle::ty::adjustment::PointerCoercion;
 use rustc_middle::ty::TyCtxt;
 
 pub struct CleanupPostBorrowck;
@@ -36,6 +37,22 @@ impl<'tcx> MirPass<'tcx> for CleanupPostBorrowck {
                         CoverageKind::BlockMarker { .. } | CoverageKind::SpanMarker { .. },
                     )
                     | StatementKind::FakeRead(..) => statement.make_nop(),
+                    StatementKind::Assign(box (
+                        _,
+                        Rvalue::Cast(
+                            ref mut cast_kind @ CastKind::PointerCoercion(
+                                PointerCoercion::ArrayToPointer
+                                | PointerCoercion::MutToConstPointer,
+                            ),
+                            ..,
+                        ),
+                    )) => {
+                        // BorrowCk needed to track whether these cases were coercions or casts,
+                        // to know whether to check lifetimes in their pointees,
+                        // but from now on that distinction doesn't matter,
+                        // so just make them ordinary pointer casts instead.
+                        *cast_kind = CastKind::PtrToPtr;
+                    }
                     _ => (),
                 }
             }

compiler/rustc_mir_transform/src/gvn.rs

@@ -571,11 +571,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                     let ret = self.ecx.ptr_to_ptr(&src, to).ok()?;
                     ret.into()
                 }
-                CastKind::PointerCoercion(
-                    ty::adjustment::PointerCoercion::MutToConstPointer
-                    | ty::adjustment::PointerCoercion::ArrayToPointer
-                    | ty::adjustment::PointerCoercion::UnsafeFnPointer,
-                ) => {
+                CastKind::PointerCoercion(ty::adjustment::PointerCoercion::UnsafeFnPointer) => {
                     let src = self.evaluated[value].as_ref()?;
                     let src = self.ecx.read_immediate(src).ok()?;
                     let to = self.ecx.layout_of(to).ok()?;
@@ -1164,10 +1160,10 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             }
         }
 
-        if let PtrToPtr | PointerCoercion(MutToConstPointer) = kind
+        if let PtrToPtr = kind
             && let Value::Cast { kind: inner_kind, value: inner_value, from: inner_from, to: _ } =
                 *self.get(value)
-            && let PtrToPtr | PointerCoercion(MutToConstPointer) = inner_kind
+            && let PtrToPtr = inner_kind
         {
             from = inner_from;
             value = inner_value;

compiler/rustc_mir_transform/src/lib.rs

@@ -51,11 +51,11 @@ mod abort_unwinding_calls;
 mod add_call_guards;
 mod add_moves_for_packed_drops;
 mod add_retag;
+mod add_subtyping_projections;
+mod check_alignment;
 mod check_const_item_mutation;
 mod check_packed_ref;
-mod remove_place_mention;
 // This pass is public to allow external drivers to perform MIR cleanup
-mod add_subtyping_projections;
 pub mod cleanup_post_borrowck;
 mod copy_prop;
 mod coroutine;
@@ -94,6 +94,7 @@ mod prettify;
 mod promote_consts;
 mod ref_prop;
 mod remove_noop_landing_pads;
+mod remove_place_mention;
 mod remove_storage_markers;
 mod remove_uninit_drops;
 mod remove_unneeded_drops;
@@ -103,7 +104,6 @@ mod reveal_all;
 mod shim;
 mod ssa;
 // This pass is public to allow external drivers to perform MIR cleanup
-mod check_alignment;
 pub mod simplify;
 mod simplify_branches;
 mod simplify_comparison_integral;