diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index 7b6abdf1ea99e..149a39edbd01f 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -1508,8 +1508,10 @@ pub enum Variants { /// a struct, and they all have space reserved for the tag. /// For enums, the tag is the sole field of the layout. Multiple { + /// Tag definition tag: Scalar, tag_encoding: TagEncoding, + /// Index of tag among fields tag_field: usize, variants: IndexVec>, }, @@ -1545,6 +1547,7 @@ pub enum TagEncoding { pub struct Niche { pub offset: Size, pub value: Primitive, + /// A range of valid values with both endpoints being inclusive pub valid_range: WrappingRange, } @@ -1555,17 +1558,23 @@ impl Niche { if niche.available(cx) > 0 { Some(niche) } else { None } } + /// Compute how many values are outside the valid range, available for optimisation through niche filling pub fn available(&self, cx: &C) -> u128 { let Self { value, valid_range: v, .. } = *self; let size = value.size(cx); assert!(size.bits() <= 128); let max_value = size.unsigned_int_max(); - // Find out how many values are outside the valid range. let niche = v.end.wrapping_add(1)..v.start; niche.end.wrapping_sub(niche.start) & max_value } + /// Try to enlarge the valid value range to include another `count` values, + /// so that they can be used for niche-filling optimisation. + /// `None` signals impossibility of reservation. + /// Otherwise, `Some((start, scalar))` signifies that a reservation is possible, + /// the first value in the reservation is `start`, and the new scalar including + /// the reserved values is defined in `scalar`. pub fn reserve(&self, cx: &C, count: u128) -> Option<(u128, Scalar)> { assert!(count > 0); diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs index 657e6e0907ca5..bbc46313d968d 100644 --- a/compiler/rustc_borrowck/src/lib.rs +++ b/compiler/rustc_borrowck/src/lib.rs @@ -23,6 +23,7 @@ use std::ops::Deref; use rustc_abi::FieldIdx; use rustc_data_structures::fx::{FxIndexMap, FxIndexSet}; use rustc_data_structures::graph::dominators::Dominators; +use rustc_data_structures::unord::UnordMap; use rustc_errors::Diag; use rustc_hir as hir; use rustc_hir::def_id::LocalDefId; @@ -281,6 +282,7 @@ fn do_mir_borrowck<'tcx>( regioncx: ®ioncx, used_mut: Default::default(), used_mut_upvars: SmallVec::new(), + local_from_upvars: UnordMap::default(), borrow_set: &borrow_set, upvars: &[], local_names: IndexVec::from_elem(None, &promoted_body.local_decls), @@ -306,6 +308,11 @@ fn do_mir_borrowck<'tcx>( } } + let mut local_from_upvars = UnordMap::default(); + for (field, &local) in body.local_upvar_map.iter_enumerated() { + let Some(local) = local else { continue }; + local_from_upvars.insert(local, field); + } let mut mbcx = MirBorrowckCtxt { infcx: &infcx, param_env, @@ -321,6 +328,7 @@ fn do_mir_borrowck<'tcx>( regioncx: ®ioncx, used_mut: Default::default(), used_mut_upvars: SmallVec::new(), + local_from_upvars, borrow_set: &borrow_set, upvars: tcx.closure_captures(def), local_names, @@ -556,6 +564,9 @@ struct MirBorrowckCtxt<'a, 'infcx, 'tcx> { /// If the function we're checking is a closure, then we'll need to report back the list of /// mutable upvars that have been used. This field keeps track of them. used_mut_upvars: SmallVec<[FieldIdx; 8]>, + /// Since upvars are moved to real locals, we need to map mutations to the locals back to + /// the upvars, so that used_mut_upvars is up-to-date. + local_from_upvars: UnordMap, /// Region inference context. This contains the results from region inference and lets us e.g. /// find out which CFG points are contained in each borrow region. regioncx: &'a RegionInferenceContext<'tcx>, @@ -2214,10 +2225,12 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> { // If the local may have been initialized, and it is now currently being // mutated, then it is justified to be annotated with the `mut` // keyword, since the mutation may be a possible reassignment. - if is_local_mutation_allowed != LocalMutationIsAllowed::Yes - && self.is_local_ever_initialized(local, state).is_some() - { - self.used_mut.insert(local); + if !matches!(is_local_mutation_allowed, LocalMutationIsAllowed::Yes) { + if self.is_local_ever_initialized(local, state).is_some() { + self.used_mut.insert(local); + } else if let Some(&field) = self.local_from_upvars.get(&local) { + self.used_mut_upvars.push(field); + } } } RootPlace { @@ -2235,6 +2248,8 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> { projection: place_projection, }) { self.used_mut_upvars.push(field); + } else if let Some(&field) = self.local_from_upvars.get(&place_local) { + self.used_mut_upvars.push(field); } } } diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs index 0fe6a4b5fce9d..02e0bb28cc5ae 100644 --- a/compiler/rustc_borrowck/src/type_check/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/mod.rs @@ -796,15 +796,15 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> { }), }; } - ty::Coroutine(_, args) => { + ty::Coroutine(_def_id, args) => { // Only prefix fields (upvars and current state) are // accessible without a variant index. - return match args.as_coroutine().prefix_tys().get(field.index()) { - Some(ty) => Ok(*ty), - None => Err(FieldAccessError::OutOfRange { - field_count: args.as_coroutine().prefix_tys().len(), - }), - }; + let upvar_tys = args.as_coroutine().upvar_tys(); + if let Some(ty) = upvar_tys.get(field.index()) { + return Ok(*ty); + } else { + return Err(FieldAccessError::OutOfRange { field_count: upvar_tys.len() }); + } } ty::Tuple(tys) => { return match tys.get(field.index()) { @@ -1782,11 +1782,11 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { // It doesn't make sense to look at a field beyond the prefix; // these require a variant index, and are not initialized in // aggregate rvalues. - match args.as_coroutine().prefix_tys().get(field_index.as_usize()) { - Some(ty) => Ok(*ty), - None => Err(FieldAccessError::OutOfRange { - field_count: args.as_coroutine().prefix_tys().len(), - }), + let upvar_tys = &args.as_coroutine().upvar_tys(); + if let Some(ty) = upvar_tys.get(field_index.as_usize()) { + Ok(*ty) + } else { + Err(FieldAccessError::OutOfRange { field_count: upvar_tys.len() }) } } AggregateKind::CoroutineClosure(_, args) => { diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index da3818ca25e9c..ce7b4fa491d44 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -891,6 +891,9 @@ fn codegen_stmt<'tcx>( let variant_dest = lval.downcast_variant(fx, variant_index); (variant_index, variant_dest, active_field_index) } + mir::AggregateKind::Coroutine(_def_id, _args) => { + (FIRST_VARIANT, lval.downcast_variant(fx, FIRST_VARIANT), None) + } _ => (FIRST_VARIANT, lval, None), }; if active_field_index.is_some() { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs index 151923a3bd288..15d042d611869 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -13,8 +13,7 @@ use rustc_hir::def_id::{DefId, LOCAL_CRATE}; use rustc_middle::bug; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{ - self, AdtKind, CoroutineArgsExt, Instance, ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt, - Visibility, + self, AdtKind, Instance, ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt, Visibility, }; use rustc_session::config::{self, DebugInfo, Lto}; use rustc_span::symbol::Symbol; @@ -1098,7 +1097,7 @@ fn build_upvar_field_di_nodes<'ll, 'tcx>( closure_or_coroutine_di_node: &'ll DIType, ) -> SmallVec<&'ll DIType> { let (&def_id, up_var_tys) = match closure_or_coroutine_ty.kind() { - ty::Coroutine(def_id, args) => (def_id, args.as_coroutine().prefix_tys()), + ty::Coroutine(def_id, args) => (def_id, args.as_coroutine().upvar_tys()), ty::Closure(def_id, args) => (def_id, args.as_closure().upvar_tys()), ty::CoroutineClosure(def_id, args) => (def_id, args.as_coroutine_closure().upvar_tys()), _ => { diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs index 100b046cee21b..274b70bb92061 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -672,7 +672,6 @@ fn build_union_fields_for_direct_tag_coroutine<'ll, 'tcx>( let coroutine_layout = cx.tcx.coroutine_layout(coroutine_def_id, coroutine_args.kind_ty()).unwrap(); - let common_upvar_names = cx.tcx.closure_saved_names_of_captured_variables(coroutine_def_id); let variant_range = coroutine_args.variant_range(coroutine_def_id, cx.tcx); let variant_count = (variant_range.start.as_u32()..variant_range.end.as_u32()).len(); @@ -707,7 +706,6 @@ fn build_union_fields_for_direct_tag_coroutine<'ll, 'tcx>( coroutine_type_and_layout, coroutine_type_di_node, coroutine_layout, - common_upvar_names, ); let span = coroutine_layout.variant_source_info[variant_index].span; diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs index b3d4a6642a16c..ac80b68b89b1c 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -4,12 +4,10 @@ use rustc_abi::{FieldIdx, TagEncoding, VariantIdx, Variants}; use rustc_codegen_ssa::debuginfo::type_names::{compute_debuginfo_type_name, cpp_like_debuginfo}; use rustc_codegen_ssa::debuginfo::{tag_base_type, wants_c_like_enum_debuginfo}; use rustc_hir::def::CtorKind; -use rustc_index::IndexSlice; use rustc_middle::bug; use rustc_middle::mir::CoroutineLayout; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; use rustc_middle::ty::{self, AdtDef, CoroutineArgs, CoroutineArgsExt, Ty, VariantDef}; -use rustc_span::Symbol; use super::type_map::{DINodeCreationResult, UniqueTypeId}; use super::{SmallVec, size_and_align_of}; @@ -263,7 +261,6 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( coroutine_type_and_layout: TyAndLayout<'tcx>, coroutine_type_di_node: &'ll DIType, coroutine_layout: &CoroutineLayout<'tcx>, - common_upvar_names: &IndexSlice, ) -> &'ll DIType { let variant_name = CoroutineArgs::variant_name(variant_index); let unique_type_id = UniqueTypeId::for_enum_variant_struct_type( @@ -274,11 +271,6 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( let variant_layout = coroutine_type_and_layout.for_variant(cx, variant_index); - let coroutine_args = match coroutine_type_and_layout.ty.kind() { - ty::Coroutine(_, args) => args.as_coroutine(), - _ => unreachable!(), - }; - type_map::build_type_with_children( cx, type_map::stub( @@ -292,7 +284,7 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( ), |cx, variant_struct_type_di_node| { // Fields that just belong to this variant/state - let state_specific_fields: SmallVec<_> = (0..variant_layout.fields.count()) + (0..variant_layout.fields.count()) .map(|field_index| { let coroutine_saved_local = coroutine_layout.variant_fields[variant_index] [FieldIdx::from_usize(field_index)]; @@ -314,28 +306,7 @@ fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>( type_di_node(cx, field_type), ) }) - .collect(); - - // Fields that are common to all states - let common_fields: SmallVec<_> = coroutine_args - .prefix_tys() - .iter() - .zip(common_upvar_names) - .enumerate() - .map(|(index, (upvar_ty, upvar_name))| { - build_field_di_node( - cx, - variant_struct_type_di_node, - upvar_name.as_str(), - cx.size_and_align_of(upvar_ty), - coroutine_type_and_layout.fields.offset(index), - DIFlags::FlagZero, - type_di_node(cx, upvar_ty), - ) - }) - .collect(); - - state_specific_fields.into_iter().chain(common_fields).collect() + .collect() }, |cx| build_generic_type_param_di_nodes(cx, coroutine_type_and_layout.ty), ) diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs index d4006691d377e..ada9ea056aeb0 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -160,9 +160,6 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( ) }; - let common_upvar_names = - cx.tcx.closure_saved_names_of_captured_variables(coroutine_def_id); - // Build variant struct types let variant_struct_type_di_nodes: SmallVec<_> = variants .indices() @@ -190,7 +187,6 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>( coroutine_type_and_layout, coroutine_type_di_node, coroutine_layout, - common_upvar_names, ), source_info, } diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 19101ec2d1ba3..d6de705aa89cc 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -10,7 +10,7 @@ use rustc_middle::mir::interpret::{Pointer, Scalar, alloc_range}; use rustc_middle::mir::{self, ConstValue}; use rustc_middle::ty::Ty; use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; -use tracing::debug; +use tracing::{debug, instrument}; use super::place::{PlaceRef, PlaceValue}; use super::{FunctionCx, LocalRef}; @@ -554,13 +554,12 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue { } impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { + #[instrument(level = "debug", skip(self, bx), ret)] fn maybe_codegen_consume_direct( &mut self, bx: &mut Bx, place_ref: mir::PlaceRef<'tcx>, ) -> Option> { - debug!("maybe_codegen_consume_direct(place_ref={:?})", place_ref); - match self.locals[place_ref.local] { LocalRef::Operand(mut o) => { // Moves out of scalar and scalar pair fields are trivial. @@ -603,13 +602,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } + #[instrument(level = "debug", skip(self, bx), ret)] pub fn codegen_consume( &mut self, bx: &mut Bx, place_ref: mir::PlaceRef<'tcx>, ) -> OperandRef<'tcx, Bx::Value> { - debug!("codegen_consume(place_ref={:?})", place_ref); - let ty = self.monomorphized_place_ty(place_ref); let layout = bx.cx().layout_of(ty); @@ -628,13 +626,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { bx.load_operand(place) } + #[instrument(level = "debug", skip(self, bx), ret)] pub fn codegen_operand( &mut self, bx: &mut Bx, operand: &mir::Operand<'tcx>, ) -> OperandRef<'tcx, Bx::Value> { - debug!("codegen_operand(operand={:?})", operand); - match *operand { mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => { self.codegen_consume(bx, place.as_ref()) diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index 0e1cd662f9188..42fcae63eceec 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -130,6 +130,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let variant_dest = dest.project_downcast(bx, variant_index); (variant_index, variant_dest, active_field_index) } + mir::AggregateKind::Coroutine(_, _) => { + (FIRST_VARIANT, dest.project_downcast(bx, FIRST_VARIANT), None) + } _ => (FIRST_VARIANT, dest, None), }; if active_field_index.is_some() { diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs index 18cff2c5e0f3d..130dbbb4d93f9 100644 --- a/compiler/rustc_const_eval/src/interpret/step.rs +++ b/compiler/rustc_const_eval/src/interpret/step.rs @@ -294,6 +294,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let variant_dest = self.project_downcast(dest, variant_index)?; (variant_index, variant_dest, active_field_index) } + mir::AggregateKind::Coroutine(_def_id, _args) => { + (FIRST_VARIANT, self.project_downcast(dest, FIRST_VARIANT)?, None) + } mir::AggregateKind::RawPtr(..) => { // Pointers don't have "fields" in the normal sense, so the // projection-based code below would either fail in projection diff --git a/compiler/rustc_feature/src/unstable.rs b/compiler/rustc_feature/src/unstable.rs index 8326d0031ea60..aff3cbd733cf9 100644 --- a/compiler/rustc_feature/src/unstable.rs +++ b/compiler/rustc_feature/src/unstable.rs @@ -434,6 +434,8 @@ declare_features! ( (unstable, const_try, "1.56.0", Some(74935)), /// Allows coroutines to be cloned. (unstable, coroutine_clone, "1.65.0", Some(95360)), + /// Allows aggressive merging coroutine saved slots + (unstable, coroutine_new_layout, "CURRENT_RUSTC_VERSION", Some(99999)), /// Allows defining coroutines. (unstable, coroutines, "1.21.0", Some(43122)), /// Allows function attribute `#[coverage(on/off)]`, to control coverage diff --git a/compiler/rustc_index/src/vec.rs b/compiler/rustc_index/src/vec.rs index 1cb8bc861af98..98ffc8e5b3ef8 100644 --- a/compiler/rustc_index/src/vec.rs +++ b/compiler/rustc_index/src/vec.rs @@ -193,6 +193,11 @@ impl IndexVec { pub fn append(&mut self, other: &mut Self) { self.raw.append(&mut other.raw); } + + #[inline] + pub fn debug_map_view(&self) -> IndexSliceMapView<'_, I, T> { + IndexSliceMapView(self.as_slice()) + } } /// `IndexVec` is often used as a map, so it provides some map-like APIs. @@ -216,14 +221,44 @@ impl IndexVec> { pub fn contains(&self, index: I) -> bool { self.get(index).and_then(Option::as_ref).is_some() } + + #[inline] + pub fn debug_map_view_compact(&self) -> IndexSliceMapViewCompact<'_, I, T> { + IndexSliceMapViewCompact(self.as_slice()) + } } +pub struct IndexSliceMapView<'a, I: Idx, T>(&'a IndexSlice); +pub struct IndexSliceMapViewCompact<'a, I: Idx, T>(&'a IndexSlice>); + impl fmt::Debug for IndexVec { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Debug::fmt(&self.raw, fmt) } } +impl<'a, I: Idx, T: fmt::Debug> fmt::Debug for IndexSliceMapView<'a, I, T> { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut entries = fmt.debug_map(); + for (idx, val) in self.0.iter_enumerated() { + entries.entry(&idx, val); + } + entries.finish() + } +} + +impl<'a, I: Idx, T: fmt::Debug> fmt::Debug for IndexSliceMapViewCompact<'a, I, T> { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut entries = fmt.debug_map(); + for (idx, val) in self.0.iter_enumerated() { + if let Some(val) = val { + entries.entry(&idx, val); + } + } + entries.finish() + } +} + impl Deref for IndexVec { type Target = IndexSlice; diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs index 260c6543f9844..76ece976e3cf9 100644 --- a/compiler/rustc_middle/src/mir/mod.rs +++ b/compiler/rustc_middle/src/mir/mod.rs @@ -368,6 +368,9 @@ pub struct Body<'tcx> { /// If `-Cinstrument-coverage` is not active, or if an individual function /// is not eligible for coverage, then this should always be `None`. pub function_coverage_info: Option>, + + /// Coroutine local-upvar map + pub local_upvar_map: IndexVec>, } impl<'tcx> Body<'tcx> { @@ -411,6 +414,7 @@ impl<'tcx> Body<'tcx> { tainted_by_errors, coverage_info_hi: None, function_coverage_info: None, + local_upvar_map: IndexVec::new(), }; body.is_polymorphic = body.has_non_region_param(); body @@ -442,6 +446,7 @@ impl<'tcx> Body<'tcx> { tainted_by_errors: None, coverage_info_hi: None, function_coverage_info: None, + local_upvar_map: IndexVec::new(), }; body.is_polymorphic = body.has_non_region_param(); body diff --git a/compiler/rustc_middle/src/mir/patch.rs b/compiler/rustc_middle/src/mir/patch.rs index 18c48d99b81ce..336c8682a3971 100644 --- a/compiler/rustc_middle/src/mir/patch.rs +++ b/compiler/rustc_middle/src/mir/patch.rs @@ -155,10 +155,14 @@ impl<'tcx> MirPatch<'tcx> { ty: Ty<'tcx>, span: Span, local_info: LocalInfo<'tcx>, + immutable: bool, ) -> Local { let index = self.next_local; self.next_local += 1; let mut new_decl = LocalDecl::new(ty, span); + if immutable { + new_decl = new_decl.immutable(); + } **new_decl.local_info.as_mut().assert_crate_local() = local_info; self.new_locals.push(new_decl); Local::new(index) diff --git a/compiler/rustc_middle/src/mir/query.rs b/compiler/rustc_middle/src/mir/query.rs index 86abeb5038234..32d5860f7b785 100644 --- a/compiler/rustc_middle/src/mir/query.rs +++ b/compiler/rustc_middle/src/mir/query.rs @@ -22,7 +22,7 @@ use crate::ty::{self, CoroutineArgsExt, OpaqueHiddenType, Ty, TyCtxt}; rustc_index::newtype_index! { #[derive(HashStable)] #[encodable] - #[debug_format = "_{}"] + #[debug_format = "corsl_{}"] pub struct CoroutineSavedLocal {} } diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs index a0eb902931969..ea64722043f5b 100644 --- a/compiler/rustc_middle/src/ty/layout.rs +++ b/compiler/rustc_middle/src/ty/layout.rs @@ -903,9 +903,10 @@ where ), Variants::Multiple { tag, tag_field, .. } => { if i == tag_field { - return TyMaybeWithLayout::TyAndLayout(tag_layout(tag)); + TyMaybeWithLayout::TyAndLayout(tag_layout(tag)) + } else { + TyMaybeWithLayout::Ty(args.as_coroutine().upvar_tys()[i]) } - TyMaybeWithLayout::Ty(args.as_coroutine().prefix_tys()[i]) } }, diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs index 142db8a17f04f..b1f6f69d6bbd1 100644 --- a/compiler/rustc_middle/src/ty/sty.rs +++ b/compiler/rustc_middle/src/ty/sty.rs @@ -148,13 +148,6 @@ impl<'tcx> ty::CoroutineArgs> { }) }) } - - /// This is the types of the fields of a coroutine which are not stored in a - /// variant. - #[inline] - fn prefix_tys(self) -> &'tcx List> { - self.upvar_tys() - } } #[derive(Debug, Copy, Clone, HashStable, TypeFoldable, TypeVisitable)] diff --git a/compiler/rustc_mir_build/src/build/custom/mod.rs b/compiler/rustc_mir_build/src/build/custom/mod.rs index 4815db47b16c1..473d8075e67ba 100644 --- a/compiler/rustc_mir_build/src/build/custom/mod.rs +++ b/compiler/rustc_mir_build/src/build/custom/mod.rs @@ -62,6 +62,7 @@ pub(super) fn build_custom_mir<'tcx>( pass_count: 0, coverage_info_hi: None, function_coverage_info: None, + local_upvar_map: IndexVec::new(), }; body.local_decls.push(LocalDecl::new(return_ty, return_ty_span)); diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs index 063f220501f80..9cf8a08e9bef7 100644 --- a/compiler/rustc_mir_transform/src/coroutine.rs +++ b/compiler/rustc_mir_transform/src/coroutine.rs @@ -51,11 +51,15 @@ //! Otherwise it drops all the values in scope at the last suspension point. mod by_move_body; -use std::{iter, ops}; +mod relocate_upvars; + +use std::ops::Deref; pub(super) use by_move_body::coroutine_by_move_body_def_id; +pub(super) use relocate_upvars::RelocateUpvars; use rustc_abi::{FieldIdx, VariantIdx}; use rustc_data_structures::fx::FxHashSet; +use rustc_data_structures::unord::UnordMap; use rustc_errors::pluralize; use rustc_hir as hir; use rustc_hir::lang_items::LangItem; @@ -65,7 +69,8 @@ use rustc_index::{Idx, IndexVec}; use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::{ - self, CoroutineArgs, CoroutineArgsExt, GenericArgsRef, InstanceKind, Ty, TyCtxt, TypingMode, + self, CapturedPlace, CoroutineArgs, CoroutineArgsExt, GenericArgsRef, InstanceKind, Ty, TyCtxt, + TypingMode, }; use rustc_middle::{bug, span_bug}; use rustc_mir_dataflow::Analysis; @@ -93,6 +98,8 @@ struct RenameLocalVisitor<'tcx> { tcx: TyCtxt<'tcx>, } +const VARIANT_UNRESUMED: VariantIdx = VariantIdx::from_usize(CoroutineArgs::UNRESUMED); + impl<'tcx> MutVisitor<'tcx> for RenameLocalVisitor<'tcx> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx @@ -188,8 +195,11 @@ struct TransformVisitor<'tcx> { // A map from a suspension point in a block to the locals which have live storage at that point storage_liveness: IndexVec>>, + // A rev-lookup from basic blocks with yielding terminator to the suspension point index, + suspension_point_at_block: UnordMap, + // A list of suspension points, generated during the transform - suspension_points: Vec>, + suspension_points: IndexVec>>, // The set of locals that have no `StorageLive`/`StorageDead` annotations. always_live_locals: BitSet, @@ -389,6 +399,16 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { // Replace an Local in the remap with a coroutine struct access if let Some(&Some((ty, variant_index, idx))) = self.remap.get(place.local) { replace_base(place, self.make_field(variant_index, idx, ty), self.tcx); + } else if let Place { local: ty::CAPTURE_STRUCT_LOCAL, projection } = *place + && let [first @ ProjectionElem::Field(..), rest @ ..] = &**projection + { + let projections: Vec<_> = [ProjectionElem::Downcast(None, VARIANT_UNRESUMED), *first] + .into_iter() + .chain(rest.iter().copied()) + .collect(); + let new_place = + Place::from(ty::CAPTURE_STRUCT_LOCAL).project_deeper(&projections, self.tcx); + *place = new_place; } } @@ -416,8 +436,9 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { // We must assign the value first in case it gets declared dead below self.make_state(v, source_info, is_return, &mut data.statements); let state = if let Some((resume, mut resume_arg)) = resume { - // Yield - let state = CoroutineArgs::RESERVED_VARIANTS + self.suspension_points.len(); + // This is a `yield` + let suspension_point_idx = *self.suspension_point_at_block.get(&block).unwrap(); + let state = CoroutineArgs::RESERVED_VARIANTS + suspension_point_idx.as_usize(); // The resume arg target location might itself be remapped if its base local is // live across a yield. @@ -439,12 +460,8 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { } } - self.suspension_points.push(SuspensionPoint { - state, - resume, - resume_arg, - drop, - storage_liveness, + self.suspension_points.get_or_insert_with(suspension_point_idx, || { + SuspensionPoint { state, resume, resume_arg, drop, storage_liveness } }); VariantIdx::new(state) @@ -470,19 +487,19 @@ fn make_aggregate_adt<'tcx>( } fn make_coroutine_state_argument_indirect<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let coroutine_ty = body.local_decls.raw[1].ty; + let coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty; let ref_coroutine_ty = Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, coroutine_ty); // Replace the by value coroutine argument - body.local_decls.raw[1].ty = ref_coroutine_ty; + body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty = ref_coroutine_ty; // Add a deref to accesses of the coroutine state SelfArgVisitor::new(tcx, ProjectionElem::Deref).visit_body(body); } fn make_coroutine_state_argument_pinned<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let ref_coroutine_ty = body.local_decls.raw[1].ty; + let ref_coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty; let pin_did = tcx.require_lang_item(LangItem::Pin, Some(body.span)); let pin_adt_ref = tcx.adt_def(pin_did); @@ -490,7 +507,7 @@ fn make_coroutine_state_argument_pinned<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body let pin_ref_coroutine_ty = Ty::new_adt(tcx, pin_adt_ref, args); // Replace the by ref coroutine argument - body.local_decls.raw[1].ty = pin_ref_coroutine_ty; + body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty = pin_ref_coroutine_ty; // Add the Pin field access to accesses of the coroutine state SelfArgVisitor::new(tcx, ProjectionElem::Field(FieldIdx::ZERO, ref_coroutine_ty)) @@ -629,15 +646,19 @@ fn transform_gen_context<'tcx>(body: &mut Body<'tcx>) { body.arg_count = 1; } +#[derive(Debug)] struct LivenessInfo { /// Which locals are live across any suspension point. saved_locals: CoroutineSavedLocals, + /// Always live locals + always_live_locals: BitSet, + /// The set of saved locals live at each suspension point. - live_locals_at_suspension_points: Vec>, + live_locals_at_suspension_points: IndexVec>, /// Parallel vec to the above with SourceInfo for each yield terminator. - source_info_at_suspension_points: Vec, + source_info_at_suspension_points: IndexVec, /// For every saved local, the set of other saved locals that are /// storage-live at the same time as this local. We cannot overlap locals in @@ -647,6 +668,14 @@ struct LivenessInfo { /// For every suspending block, the locals which are storage-live across /// that suspension point. storage_liveness: IndexVec>>, + + /// A rev-lookup of basic blocks to the suspension point index + suspension_point_at_block: UnordMap, +} + +rustc_index::newtype_index! { + #[debug_format = "suspend_{}"] + struct SuspensionPointIdx {} } /// Computes which locals have to be stored in the state-machine for the @@ -660,12 +689,12 @@ struct LivenessInfo { fn locals_live_across_suspend_points<'tcx>( tcx: TyCtxt<'tcx>, body: &Body<'tcx>, - always_live_locals: &BitSet, + always_live_locals: BitSet, movable: bool, ) -> LivenessInfo { // Calculate when MIR locals have live storage. This gives us an upper bound of their // lifetimes. - let mut storage_live = MaybeStorageLive::new(std::borrow::Cow::Borrowed(always_live_locals)) + let mut storage_live = MaybeStorageLive::new(std::borrow::Cow::Borrowed(&always_live_locals)) .iterate_to_fixpoint(tcx, body, None) .into_results_cursor(body); @@ -688,11 +717,13 @@ fn locals_live_across_suspend_points<'tcx>( MaybeLiveLocals.iterate_to_fixpoint(tcx, body, Some("coroutine")).into_results_cursor(body); let mut storage_liveness_map = IndexVec::from_elem(None, &body.basic_blocks); - let mut live_locals_at_suspension_points = Vec::new(); - let mut source_info_at_suspension_points = Vec::new(); + let mut live_locals_at_suspension_points = IndexVec::::default(); + let mut source_info_at_suspension_points = IndexVec::default(); let mut live_locals_at_any_suspension_point = BitSet::new_empty(body.local_decls.len()); + let mut suspension_point_at_block = UnordMap::default(); - for (block, data) in body.basic_blocks.iter_enumerated() { + for &block in body.basic_blocks.reverse_postorder() { + let data = &body.basic_blocks[block]; if let TerminatorKind::Yield { .. } = data.terminator().kind { let loc = Location { block, statement_index: data.statements.len() }; @@ -727,7 +758,7 @@ fn locals_live_across_suspend_points<'tcx>( live_locals.intersect(requires_storage_cursor.get()); // The coroutine argument is ignored. - live_locals.remove(SELF_ARG); + live_locals.remove(ty::CAPTURE_STRUCT_LOCAL); debug!("loc = {:?}, live_locals = {:?}", loc, live_locals); @@ -735,8 +766,9 @@ fn locals_live_across_suspend_points<'tcx>( // any suspension points live_locals_at_any_suspension_point.union(&live_locals); - live_locals_at_suspension_points.push(live_locals); + let idx = live_locals_at_suspension_points.push(live_locals); source_info_at_suspension_points.push(data.terminator().source_info); + suspension_point_at_block.insert(block, idx); } } @@ -759,10 +791,12 @@ fn locals_live_across_suspend_points<'tcx>( LivenessInfo { saved_locals, + always_live_locals, live_locals_at_suspension_points, source_info_at_suspension_points, storage_conflicts, storage_liveness: storage_liveness_map, + suspension_point_at_block, } } @@ -771,6 +805,7 @@ fn locals_live_across_suspend_points<'tcx>( /// `CoroutineSavedLocal` is indexed in terms of the elements in this set; /// i.e. `CoroutineSavedLocal::new(1)` corresponds to the second local /// included in this set. +#[derive(Debug)] struct CoroutineSavedLocals(BitSet); impl CoroutineSavedLocals { @@ -803,7 +838,7 @@ impl CoroutineSavedLocals { } } -impl ops::Deref for CoroutineSavedLocals { +impl Deref for CoroutineSavedLocals { type Target = BitSet; fn deref(&self) -> &Self::Target { @@ -923,20 +958,26 @@ impl StorageConflictVisitor<'_, '_> { } } +#[instrument[level = "debug", skip(body), fields(body = ?body.source)]] fn compute_layout<'tcx>( liveness: LivenessInfo, body: &Body<'tcx>, + upvar_tys: &[Ty<'tcx>], + upvar_infos: &[&CapturedPlace<'tcx>], ) -> ( IndexVec, VariantIdx, FieldIdx)>>, CoroutineLayout<'tcx>, IndexVec>>, + UnordMap, ) { let LivenessInfo { saved_locals, + always_live_locals, live_locals_at_suspension_points, source_info_at_suspension_points, storage_conflicts, storage_liveness, + suspension_point_at_block, } = liveness; // Gather live local types and their indices. @@ -970,6 +1011,47 @@ fn compute_layout<'tcx>( tys.push(decl); } + // These are the "saved locals" sourced from the UNRESUMED state. + let upvar_saved_locals: IndexVec = upvar_tys + .iter() + .zip(upvar_infos) + .map(|(&ty, info)| { + tys.push(CoroutineSavedTy { + ty, + source_info: SourceInfo::outermost(info.var_ident.span), + ignore_for_traits: false, + }) + }) + .collect(); + debug!(?upvar_saved_locals); + let storage_conflicts = if let Some(&first) = upvar_saved_locals.raw.first() + && let Some(&last) = upvar_saved_locals.raw.last() + { + let mut enlarged_storage_conflicts = BitMatrix::new(tys.len(), tys.len()); + let mut upvars = BitSet::new_empty(tys.len()); + let mut ineligibles = upvars.clone(); + upvars.insert_range(first..=last); + for (saved_local, local) in saved_locals.iter_enumerated() { + if always_live_locals.contains(local) { + ineligibles.insert(saved_local); + } + } + upvars.union(&ineligibles); + for row in storage_conflicts.rows() { + for column in storage_conflicts.iter(row) { + enlarged_storage_conflicts.insert(row, column); + } + } + for &upvar in &upvar_saved_locals { + enlarged_storage_conflicts.union_row_with(&upvars, upvar); + } + for ineligible in ineligibles.iter() { + enlarged_storage_conflicts.union_row_with(&upvars, ineligible); + } + enlarged_storage_conflicts + } else { + storage_conflicts + }; // Leave empty variants for the UNRESUMED, RETURNED, and POISONED states. // In debuginfo, these will correspond to the beginning (UNRESUMED) or end @@ -986,12 +1068,14 @@ fn compute_layout<'tcx>( // Build the coroutine variant field list. // Create a map from local indices to coroutine struct indices. + let variant_fields: [_; CoroutineArgs::RESERVED_VARIANTS] = + [upvar_saved_locals.clone(), IndexVec::new(), IndexVec::new()]; let mut variant_fields: IndexVec> = - iter::repeat(IndexVec::new()).take(CoroutineArgs::RESERVED_VARIANTS).collect(); + variant_fields.into_iter().collect(); let mut remap = IndexVec::from_elem_n(None, saved_locals.domain_size()); - for (suspension_point_idx, live_locals) in live_locals_at_suspension_points.iter().enumerate() { + for (suspension_point_idx, live_locals) in live_locals_at_suspension_points.iter_enumerated() { let variant_index = - VariantIdx::from(CoroutineArgs::RESERVED_VARIANTS + suspension_point_idx); + VariantIdx::from(CoroutineArgs::RESERVED_VARIANTS + suspension_point_idx.as_usize()); let mut fields = IndexVec::new(); for (idx, saved_local) in live_locals.iter().enumerate() { fields.push(saved_local); @@ -1007,18 +1091,24 @@ fn compute_layout<'tcx>( } debug!("coroutine variant_fields = {:?}", variant_fields); debug!("coroutine storage_conflicts = {:#?}", storage_conflicts); + debug!(remap = ?remap.debug_map_view_compact()); + debug!(locals = ?locals.debug_map_view()); let mut field_names = IndexVec::from_elem(None, &tys); for var in &body.var_debug_info { - let VarDebugInfoContents::Place(place) = &var.value else { continue }; - let Some(local) = place.as_local() else { continue }; - let Some(&Some((_, variant, field))) = remap.get(local) else { - continue; - }; - - let saved_local = variant_fields[variant][field]; - field_names.get_or_insert_with(saved_local, || var.name); + debug!(?var); + if let VarDebugInfoContents::Place(place) = &var.value + && let Some(local) = place.local_or_deref_local() + && let Some(&Some((_, variant, field))) = remap.get(local) + { + let saved_local = variant_fields[variant][field]; + field_names.get_or_insert_with(saved_local, || var.name); + } + } + for (capture, saved_local) in upvar_infos.iter().zip(upvar_saved_locals) { + field_names.get_or_insert_with(saved_local, || capture.var_ident.name); } + debug!(field_names = ?field_names.debug_map_view()); let layout = CoroutineLayout { field_tys: tys, @@ -1029,7 +1119,7 @@ fn compute_layout<'tcx>( }; debug!(?layout); - (remap, layout, storage_liveness) + (remap, layout, storage_liveness, suspension_point_at_block) } /// Replaces the entry point of `body` with a block that switches on the coroutine discriminant and @@ -1106,7 +1196,8 @@ fn elaborate_coroutine_drops<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { elaborate_drop( &mut elaborator, *source_info, - Place::from(SELF_ARG), + Place::from(SELF_ARG) + .project_deeper(&[ProjectionElem::Downcast(None, VARIANT_UNRESUMED)], tcx), (), *target, unwind, @@ -1170,7 +1261,7 @@ fn create_coroutine_drop_shim<'tcx>( // Temporary change MirSource to coroutine's instance so that dump_mir produces more sensible // filename. body.source.instance = coroutine_instance; - dump_mir(tcx, false, "coroutine_drop", &0, &body, |_, _| Ok(())); + dump_mir(tcx, false, "coroutine_drop", &0 as _, &body, |_, _| Ok(())); body.source.instance = drop_instance; body @@ -1358,7 +1449,7 @@ fn create_coroutine_resume_function<'tcx>( pm::run_passes_no_validate(tcx, body, &[&abort_unwinding_calls::AbortUnwindingCalls], None); - dump_mir(tcx, false, "coroutine_resume", &0, body, |_, _| Ok(())); + dump_mir(tcx, false, "coroutine_resume", &0 as _, body, |_, _| Ok(())); } fn insert_clean_drop(body: &mut Body<'_>) -> BasicBlock { @@ -1407,6 +1498,7 @@ fn create_cases<'tcx>( .suspension_points .iter() .filter_map(|point| { + let Some(point) = point else { bug!("all suspension points must be resolved now") }; // Find the target for this suspension point, if applicable operation.target_block(point).map(|target| { let mut statements = Vec::new(); @@ -1463,8 +1555,12 @@ pub(crate) fn mir_coroutine_witnesses<'tcx>( // The first argument is the coroutine type passed by value let coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty; - let movable = match *coroutine_ty.kind() { - ty::Coroutine(def_id, _) => tcx.coroutine_movability(def_id) == hir::Movability::Movable, + let (movable, upvar_tys, upvar_infos) = match *coroutine_ty.kind() { + ty::Coroutine(def_id, args) => ( + matches!(tcx.coroutine_movability(def_id), hir::Movability::Movable), + args.as_coroutine().upvar_tys(), + tcx.closure_captures(def_id.expect_local()), + ), ty::Error(_) => return None, _ => span_bug!(body.span, "unexpected coroutine type {}", coroutine_ty), }; @@ -1472,12 +1568,13 @@ pub(crate) fn mir_coroutine_witnesses<'tcx>( // The witness simply contains all locals live across suspend points. let always_live_locals = always_storage_live_locals(body); - let liveness_info = locals_live_across_suspend_points(tcx, body, &always_live_locals, movable); + debug!(?always_live_locals); + let liveness_info = locals_live_across_suspend_points(tcx, body, always_live_locals, movable); // Extract locals which are live across suspension point into `layout` // `remap` gives a mapping from local indices onto coroutine struct indices // `storage_liveness` tells us which locals have live storage at suspension points - let (_, coroutine_layout, _) = compute_layout(liveness_info, body); + let (_, coroutine_layout, _, _) = compute_layout(liveness_info, body, upvar_tys, upvar_infos); check_suspend_tys(tcx, &coroutine_layout, body); check_field_tys_sized(tcx, &coroutine_layout, def_id); @@ -1535,14 +1632,19 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { assert!(body.coroutine_drop().is_none()); // The first argument is the coroutine type passed by value - let coroutine_ty = body.local_decls.raw[1].ty; + let coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty; let coroutine_kind = body.coroutine_kind().unwrap(); // Get the discriminant type and args which typeck computed - let (discr_ty, movable) = match *coroutine_ty.kind() { - ty::Coroutine(_, args) => { + let (discr_ty, movable, upvar_tys, upvar_infos) = match *coroutine_ty.kind() { + ty::Coroutine(def_id, args) => { let args = args.as_coroutine(); - (args.discr_ty(tcx), coroutine_kind.movability() == hir::Movability::Movable) + ( + args.discr_ty(tcx), + matches!(coroutine_kind.movability(), hir::Movability::Movable), + args.upvar_tys(), + tcx.closure_captures(def_id.expect_local()), + ) } _ => { tcx.dcx().span_bug(body.span, format!("unexpected coroutine type {coroutine_ty}")); @@ -1612,7 +1714,7 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { let always_live_locals = always_storage_live_locals(body); let liveness_info = - locals_live_across_suspend_points(tcx, body, &always_live_locals, movable); + locals_live_across_suspend_points(tcx, body, always_live_locals.clone(), movable); if tcx.sess.opts.unstable_opts.validate_mir { let mut vis = EnsureCoroutineFieldAssignmentsNeverAlias { @@ -1627,7 +1729,8 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { // Extract locals which are live across suspension point into `layout` // `remap` gives a mapping from local indices onto coroutine struct indices // `storage_liveness` tells us which locals have live storage at suspension points - let (remap, layout, storage_liveness) = compute_layout(liveness_info, body); + let (remap, layout, storage_liveness, suspension_point_at_block) = + compute_layout(liveness_info, body, upvar_tys, upvar_infos); let can_return = can_return(tcx, body, tcx.param_env(body.source.def_id())); @@ -1642,11 +1745,12 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { remap, storage_liveness, always_live_locals, - suspension_points: Vec::new(), + suspension_points: IndexVec::default(), old_ret_local, discr_ty, old_ret_ty, old_yield_ty, + suspension_point_at_block, }; transform.visit_body(body); @@ -1675,14 +1779,14 @@ impl<'tcx> crate::MirPass<'tcx> for StateTransform { // This is expanded to a drop ladder in `elaborate_coroutine_drops`. let drop_clean = insert_clean_drop(body); - dump_mir(tcx, false, "coroutine_pre-elab", &0, body, |_, _| Ok(())); + dump_mir(tcx, false, "coroutine_pre-elab", &0 as _, body, |_, _| Ok(())); // Expand `drop(coroutine_struct)` to a drop ladder which destroys upvars. // If any upvars are moved out of, drop elaboration will handle upvar destruction. // However we need to also elaborate the code generated by `insert_clean_drop`. elaborate_coroutine_drops(tcx, body); - dump_mir(tcx, false, "coroutine_post-transform", &0, body, |_, _| Ok(())); + dump_mir(tcx, false, "coroutine_post-transform", &0 as _, body, |_, _| Ok(())); // Create a copy of our MIR and use it to create the drop shim for the coroutine let drop_shim = create_coroutine_drop_shim(tcx, &transform, coroutine_ty, body, drop_clean); diff --git a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs index 36eb435c63a10..8d98c8825a4b2 100644 --- a/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs +++ b/compiler/rustc_mir_transform/src/coroutine/by_move_body.rs @@ -71,7 +71,6 @@ use rustc_abi::{FieldIdx, VariantIdx}; use rustc_data_structures::steal::Steal; -use rustc_data_structures::unord::UnordMap; use rustc_hir as hir; use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LocalDefId}; @@ -81,6 +80,16 @@ use rustc_middle::mir::visit::MutVisitor; use rustc_middle::mir::{self, dump_mir}; use rustc_middle::ty::{self, InstanceKind, Ty, TyCtxt, TypeVisitableExt}; use rustc_span::symbol::kw; +use rustc_type_ir::data_structures::IndexMap; + +struct CaptureInfo<'tcx> { + /// Field index of the capture in the parent coroutine structure + remapped_idx: FieldIdx, + /// Type of the capture in the parent coroutine structure + remapped_ty: Ty<'tcx>, + peel_deref: bool, + bridging_projections: Vec>, +} pub(crate) fn coroutine_by_move_body_def_id<'tcx>( tcx: TyCtxt<'tcx>, @@ -128,23 +137,27 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( .tuple_fields() .len(); - let field_remapping: UnordMap<_, _> = ty::analyze_coroutine_closure_captures( + let field_remapping: IndexMap<_, _> = ty::analyze_coroutine_closure_captures( tcx.closure_captures(parent_def_id).iter().copied(), tcx.closure_captures(coroutine_def_id).iter().skip(num_args).copied(), |(parent_field_idx, parent_capture), (child_field_idx, child_capture)| { // Store this set of additional projections (fields and derefs). // We need to re-apply them later. - let mut child_precise_captures = - child_capture.place.projections[parent_capture.place.projections.len()..].to_vec(); + let child_precise_captures = child_capture.place.projections + [parent_capture.place.projections.len()..] + .iter() + .copied(); // If the parent capture is by-ref, then we need to apply an additional // deref before applying any further projections to this place. - if parent_capture.is_by_ref() { - child_precise_captures.insert(0, Projection { - ty: parent_capture.place.ty(), - kind: ProjectionKind::Deref, - }); - } + let bridging_projections = if parent_capture.is_by_ref() { + [Projection { ty: parent_capture.place.ty(), kind: ProjectionKind::Deref }] + .into_iter() + .chain(child_precise_captures) + .collect() + } else { + child_precise_captures.collect() + }; // If the child capture is by-ref, then we need to apply a "ref" // projection (i.e. `&`) at the end. But wait! We don't have that // as a projection kind. So instead, we can apply its dual and @@ -170,8 +183,8 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( // Finally, store the type of the parent's captured place. We need // this when building the field projection in the MIR body later on. - let mut parent_capture_ty = parent_capture.place.ty(); - parent_capture_ty = match parent_capture.info.capture_kind { + let parent_capture_ty = parent_capture.place.ty(); + let remapped_ty = match parent_capture.info.capture_kind { ty::UpvarCapture::ByValue => parent_capture_ty, ty::UpvarCapture::ByRef(kind) => Ty::new_ref( tcx, @@ -181,20 +194,17 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( ), }; - ( - FieldIdx::from_usize(child_field_idx + num_args), - ( - FieldIdx::from_usize(parent_field_idx + num_args), - parent_capture_ty, - peel_deref, - child_precise_captures, - ), - ) + (FieldIdx::from_usize(child_field_idx + num_args), CaptureInfo { + remapped_idx: FieldIdx::from_usize(parent_field_idx + num_args), + remapped_ty, + peel_deref, + bridging_projections, + }) }, ) .collect(); - if coroutine_kind == ty::ClosureKind::FnOnce { + if matches!(coroutine_kind, ty::ClosureKind::FnOnce) { assert_eq!(field_remapping.len(), tcx.closure_captures(parent_def_id).len()); // The by-move body is just the body :) return coroutine_def_id.to_def_id(); @@ -213,6 +223,7 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( ); let mut by_move_body = body.clone(); + dump_mir(tcx, false, "built", &"before", &by_move_body, |_, _| Ok(())); MakeByMoveBody { tcx, field_remapping, by_move_coroutine_ty }.visit_body(&mut by_move_body); // This will always be `{closure#1}`, since the original coroutine is `{closure#0}`. @@ -243,7 +254,7 @@ pub(crate) fn coroutine_by_move_body_def_id<'tcx>( struct MakeByMoveBody<'tcx> { tcx: TyCtxt<'tcx>, - field_remapping: UnordMap, bool, Vec>)>, + field_remapping: IndexMap>, by_move_coroutine_ty: Ty<'tcx>, } @@ -264,8 +275,12 @@ impl<'tcx> MutVisitor<'tcx> for MakeByMoveBody<'tcx> { if place.local == ty::CAPTURE_STRUCT_LOCAL && let Some((&mir::ProjectionElem::Field(idx, _), projection)) = place.projection.split_first() - && let Some(&(remapped_idx, remapped_ty, peel_deref, ref bridging_projections)) = - self.field_remapping.get(&idx) + && let Some(&CaptureInfo { + remapped_idx, + remapped_ty, + peel_deref, + ref bridging_projections, + }) = self.field_remapping.get(&idx) { // As noted before, if the parent closure captures a field by value, and // the child captures a field by ref, then for the by-move body we're diff --git a/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs b/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs new file mode 100644 index 0000000000000..66c5a6c1c138c --- /dev/null +++ b/compiler/rustc_mir_transform/src/coroutine/relocate_upvars.rs @@ -0,0 +1,310 @@ +//! MIR rewrite pass to promote upvars into native locals in the coroutine body +//! +//! # Summary +//! This pass performs the following transformations. +//! 1. It generates a fresh batch of locals for each captured upvars. +//! +//! For each upvar, whether used or not, a fresh local is created with the same type. +//! +//! 2. It replaces the places pointing into those upvars with places pointing into those locals instead +//! +//! Each place that starts with access into the coroutine structure `_1` is replaced with the fresh local as +//! the base. For instance, `(_1.4 as Some).0` is rewritten into `(_34 as Some).0` when `_34` is the fresh local +//! corresponding to the captured upvar stored in `_1.4`. +//! +//! 3. It assembles an prologue to replace the current entry block. +//! +//! This prologue block transfers every captured upvar into its corresponding fresh local, *via scratch locals*. +//! The upvars are first completely moved into the scratch locals in batch, and then moved into the destination +//! locals in batch. +//! The reason is that it is possible that coroutine layout may change and the source memory location of +//! an upvar may not necessarily be mapped exactly to the same place as in the `Unresumed` state. +//! While coroutine layout ensures that the same saved local has stable offsets throughout its lifetime, +//! technically the upvar in `Unresumed` state and their fresh locals are different saved locals. +//! This scratch locals re-estabilish safety so that the correct data permutation can take place. + +use rustc_index::{IndexSlice, IndexVec}; +use rustc_middle::bug; +use rustc_middle::mir::patch::MirPatch; +use rustc_middle::mir::visit::MutVisitor; +use rustc_middle::mir::{ + self, BasicBlock, BasicBlockData, Body, Local, Place, ProjectionElem, START_BLOCK, SourceInfo, + Statement, StatementKind, Terminator, TerminatorKind, UnwindAction, +}; +use rustc_middle::ty::{self, TyCtxt}; +use rustc_span::Span; +use rustc_target::abi::FieldIdx; + +use crate::pass_manager::MirPass; + +pub(crate) struct RelocateUpvars; + +struct UpvarSubstitution<'tcx> { + /// Newly minted local into which the upvar is moved + local: Local, + reloc: Local, + /// Place into the capture structure where this upvar is found + upvar_place: Place<'tcx>, + span: Span, +} + +struct SubstituteUpvarVisitor<'tcx, 'a> { + tcx: TyCtxt<'tcx>, + mappings: &'a IndexSlice>, +} + +impl<'tcx, 'a> MutVisitor<'tcx> for SubstituteUpvarVisitor<'tcx, 'a> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_place( + &mut self, + place: &mut Place<'tcx>, + _context: mir::visit::PlaceContext, + location: mir::Location, + ) { + if let Place { local: ty::CAPTURE_STRUCT_LOCAL, projection } = place + && let [ProjectionElem::Field(field_idx, _ty), rest @ ..] = &***projection + { + let Some(&UpvarSubstitution { local, .. }) = self.mappings.get(*field_idx) else { + bug!( + "SubstituteUpvar: found {field_idx:?} @ {location:?} but there is no upvar for it" + ) + }; + let new_place = Place::from(local); + let new_place = new_place.project_deeper(rest, self.tcx); + *place = new_place; + } + } + + fn visit_terminator( + &mut self, + terminator: &mut mir::Terminator<'tcx>, + location: mir::Location, + ) { + if let TerminatorKind::Drop { place, .. } = &terminator.kind + && let Some(ty::CAPTURE_STRUCT_LOCAL) = place.as_local() + { + // This is a drop on the whole coroutine state, which we will processed later + return; + } + self.super_terminator(terminator, location) + } +} + +impl<'tcx> MirPass<'tcx> for RelocateUpvars { + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let Some(coroutine_def_id) = body.source.def_id().as_local() else { + return; + }; + if tcx.coroutine_kind(coroutine_def_id).is_none() { + return; + } + + // The first argument is the coroutine type passed by value + let coroutine_ty = if let Some(decl) = body.local_decls.get(ty::CAPTURE_STRUCT_LOCAL) { + decl.ty + } else { + return; + }; + + // We only care when there is at least one upvar + let (def_id, upvar_tys) = if let ty::Coroutine(def_id, args) = *coroutine_ty.kind() { + let args = args.as_coroutine(); + (def_id, args.upvar_tys()) + } else { + return; + }; + if upvar_tys.is_empty() { + return; + } + + let upvar_infos = tcx.closure_captures(def_id.expect_local()).iter(); + + let mut substitution_mapping = IndexVec::new(); + let mut patch = MirPatch::new(body); + for (field_idx, (upvar_ty, &captured)) in upvar_tys.iter().zip(upvar_infos).enumerate() { + let span = captured.var_ident.span; + + let local = patch.new_local_with_info( + upvar_ty, + span, + mir::LocalInfo::Boring, + matches!(captured.mutability, ty::Mutability::Not), + ); + let reloc = patch.new_local_with_info(upvar_ty, span, mir::LocalInfo::Boring, true); + + let field_idx = FieldIdx::from_usize(field_idx); + let upvar_place = Place::from(ty::CAPTURE_STRUCT_LOCAL) + .project_deeper(&[ProjectionElem::Field(field_idx, upvar_ty)], tcx); + + substitution_mapping.push(UpvarSubstitution { local, reloc, upvar_place, span }); + } + patch.apply(body); + body.local_upvar_map = substitution_mapping.iter().map(|sub| Some(sub.local)).collect(); + SubstituteUpvarVisitor { tcx, mappings: &substitution_mapping }.visit_body(body); + + rewrite_drop_coroutine_struct(body, &substitution_mapping); + insert_substitution_prologue(body, &substitution_mapping); + } +} + +fn rewrite_one_drop_coroutine_struct<'tcx>( + patch: &mut MirPatch<'tcx>, + body: &Body<'tcx>, + block: BasicBlock, + substitution_mapping: &IndexSlice>, +) { + let data = &body.basic_blocks[block]; + let source_info = data.terminator().source_info; + let TerminatorKind::Drop { place: _, mut target, mut unwind, replace } = data.terminator().kind + else { + unreachable!() + }; + let mut cleanup = match unwind { + UnwindAction::Cleanup(tgt) => tgt, + UnwindAction::Continue => patch.resume_block(), + UnwindAction::Unreachable => patch.unreachable_cleanup_block(), + UnwindAction::Terminate(reason) => patch.terminate_block(reason), + }; + for &UpvarSubstitution { local, .. } in substitution_mapping { + let place = local.into(); + let mut unwind_one = patch.new_block(BasicBlockData { + statements: vec![Statement { source_info, kind: StatementKind::StorageDead(local) }], + terminator: Some(Terminator { + source_info, + kind: TerminatorKind::Goto { + target: if data.is_cleanup { target } else { cleanup }, + }, + }), + is_cleanup: true, + }); + unwind_one = patch.new_block(BasicBlockData { + statements: vec![], + terminator: Some(Terminator { + source_info, + kind: TerminatorKind::Drop { + place, + target: unwind_one, + unwind: UnwindAction::Terminate(mir::UnwindTerminateReason::InCleanup), + replace, + }, + }), + is_cleanup: true, + }); + if data.is_cleanup { + unwind = UnwindAction::Cleanup(unwind_one); + cleanup = unwind_one; + target = unwind_one; + } else { + let mut drop_one = patch.new_block(BasicBlockData { + statements: vec![Statement { + source_info, + kind: StatementKind::StorageDead(local), + }], + terminator: Some(Terminator { source_info, kind: TerminatorKind::Goto { target } }), + is_cleanup: false, + }); + drop_one = patch.new_block(BasicBlockData { + terminator: Some(Terminator { + source_info, + kind: TerminatorKind::Drop { place, target: drop_one, unwind, replace }, + }), + statements: vec![], + is_cleanup: false, + }); + target = drop_one; + unwind = UnwindAction::Cleanup(unwind_one); + cleanup = unwind_one; + } + } + patch.patch_terminator(block, TerminatorKind::Goto { target }); +} + +fn rewrite_drop_coroutine_struct<'tcx>( + body: &mut Body<'tcx>, + substitution_mapping: &IndexSlice>, +) { + let mut blocks = vec![]; + for (block, block_data) in body.basic_blocks.iter_enumerated() { + let Terminator { source_info: _, kind: TerminatorKind::Drop { place, .. } } = + block_data.terminator() + else { + continue; + }; + let Some(local) = place.as_local() else { continue }; + if local == ty::CAPTURE_STRUCT_LOCAL { + blocks.push(block) + } + } + let mut patch = MirPatch::new(body); + for block in blocks { + rewrite_one_drop_coroutine_struct(&mut patch, body, block, substitution_mapping); + } + patch.apply(body); +} + +fn insert_substitution_prologue<'tcx>( + body: &mut Body<'tcx>, + substitution_mapping: &IndexSlice>, +) { + let mut patch = MirPatch::new(body); + let mut stmts = Vec::with_capacity(2 * substitution_mapping.len()); + for &UpvarSubstitution { local, reloc, upvar_place, span } in substitution_mapping { + // For each upvar-local _$i + let source_info = SourceInfo::outermost(span); + // StorageLive(_$i) + stmts.push(Statement { source_info, kind: StatementKind::StorageLive(local) }); + // Use a fresh local _$i' here, so as to avoid potential field permutation + // StorageLive(_$i') + stmts.push(Statement { source_info, kind: StatementKind::StorageLive(reloc) }); + // _$i' = move $ + stmts.push(Statement { + source_info, + kind: StatementKind::Assign(Box::new(( + reloc.into(), + mir::Rvalue::Use(mir::Operand::Move(upvar_place)), + ))), + }); + } + for &UpvarSubstitution { local, reloc, upvar_place: _, span } in substitution_mapping { + let source_info = SourceInfo::outermost(span); + // _$i = move $i' + stmts.push(Statement { + source_info, + kind: StatementKind::Assign(Box::new(( + local.into(), + mir::Rvalue::Use(mir::Operand::Move(reloc.into())), + ))), + }); + stmts.push(Statement { source_info, kind: StatementKind::StorageDead(reloc) }); + } + let source_info = SourceInfo::outermost(body.span); + let prologue = patch.new_block(BasicBlockData { + statements: stmts, + terminator: Some(Terminator { + source_info, + kind: TerminatorKind::Drop { + place: Place::from(ty::CAPTURE_STRUCT_LOCAL), + target: START_BLOCK, + unwind: UnwindAction::Continue, + replace: false, + }, + }), + is_cleanup: false, + }); + patch.apply(body); + + // Manually patch so that prologue is the new entry + let preds = body.basic_blocks.predecessors()[START_BLOCK].clone(); + let basic_blocks = body.basic_blocks.as_mut(); + for pred in preds { + for target in &mut basic_blocks[pred].terminator_mut().successors_mut() { + if *target == START_BLOCK { + *target = prologue; + } + } + } + basic_blocks.swap(START_BLOCK, prologue); +} diff --git a/compiler/rustc_mir_transform/src/deref_separator.rs b/compiler/rustc_mir_transform/src/deref_separator.rs index ad7ccef4976aa..fa969be115736 100644 --- a/compiler/rustc_mir_transform/src/deref_separator.rs +++ b/compiler/rustc_mir_transform/src/deref_separator.rs @@ -39,6 +39,7 @@ impl<'a, 'tcx> MutVisitor<'tcx> for DerefChecker<'a, 'tcx> { ty, self.local_decls[p_ref.local].source_info.span, LocalInfo::DerefTemp, + false, ); // We are adding current p_ref's projections to our diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index d184328748f84..257982e7118dd 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -355,7 +355,12 @@ fn mir_promoted( pm::run_passes( tcx, &mut body, - &[&promote_pass, &simplify::SimplifyCfg::PromoteConsts, &coverage::InstrumentCoverage], + &[ + &coroutine::RelocateUpvars, + &promote_pass, + &simplify::SimplifyCfg::PromoteConsts, + &coverage::InstrumentCoverage, + ], Some(MirPhase::Analysis(AnalysisPhase::Initial)), ); diff --git a/compiler/rustc_mir_transform/src/pass_manager.rs b/compiler/rustc_mir_transform/src/pass_manager.rs index 29f8b4f6e4ddf..b2bcec1cdd291 100644 --- a/compiler/rustc_mir_transform/src/pass_manager.rs +++ b/compiler/rustc_mir_transform/src/pass_manager.rs @@ -230,10 +230,26 @@ fn run_passes_inner<'tcx>( dump_mir_for_pass(tcx, body, name, true); } if validate { - validate_body(tcx, body, format!("after pass {name}")); + validate_body( + tcx, + body, + format!( + "after pass {name} {:03}-{:03}", + body.phase.phase_index(), + body.pass_count + ), + ); } if lint { - lint_body(tcx, body, format!("after pass {name}")); + lint_body( + tcx, + body, + format!( + "after pass {name} {:03}-{:03}", + body.phase.phase_index(), + body.pass_count + ), + ); } body.pass_count += 1; diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs index 7ed43547e1127..43f8c06715881 100644 --- a/compiler/rustc_mir_transform/src/simplify.rs +++ b/compiler/rustc_mir_transform/src/simplify.rs @@ -399,9 +399,16 @@ impl<'tcx> crate::MirPass<'tcx> for SimplifyLocals { // Only bother running the `LocalUpdater` if we actually found locals to remove. if map.iter().any(Option::is_none) { // Update references to all vars and tmps now - let mut updater = LocalUpdater { map, tcx }; + let mut updater = LocalUpdater { map: &map, tcx }; updater.visit_body_preserves_cfg(body); + // Update mapping for local to upvar + for local in &mut body.local_upvar_map { + if let Some(idx) = local { + *local = *map.get(*idx).unwrap_or(&None); + } + } + body.local_decls.shrink_to_fit(); } } @@ -577,12 +584,12 @@ fn remove_unused_definitions_helper(used_locals: &mut UsedLocals, body: &mut Bod } } -struct LocalUpdater<'tcx> { - map: IndexVec>, +struct LocalUpdater<'tcx, 'a> { + map: &'a IndexSlice>, tcx: TyCtxt<'tcx>, } -impl<'tcx> MutVisitor<'tcx> for LocalUpdater<'tcx> { +impl<'tcx> MutVisitor<'tcx> for LocalUpdater<'tcx, '_> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs index ae4e6ea6a7497..5055f954ce605 100644 --- a/compiler/rustc_mir_transform/src/validate.rs +++ b/compiler/rustc_mir_transform/src/validate.rs @@ -12,8 +12,7 @@ use rustc_middle::mir::visit::{NonUseContext, PlaceContext, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::adjustment::PointerCoercion; use rustc_middle::ty::{ - self, CoroutineArgsExt, InstanceKind, ParamEnv, ScalarInt, Ty, TyCtxt, TypeVisitableExt, - Variance, + self, InstanceKind, ParamEnv, ScalarInt, Ty, TyCtxt, TypeVisitableExt, Variance, }; use rustc_middle::{bug, span_bug}; use rustc_trait_selection::traits::ObligationCtxt; @@ -782,14 +781,11 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { }; ty::EarlyBinder::bind(f_ty.ty).instantiate(self.tcx, args) - } else { - let Some(&f_ty) = args.as_coroutine().prefix_tys().get(f.index()) - else { - fail_out_of_bounds(self, location); - return; - }; - + } else if let Some(&f_ty) = args.as_coroutine().upvar_tys().get(f.index()) { f_ty + } else { + fail_out_of_bounds(self, location); + return; }; check_equal(self, location, f_ty); diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index c04c793ba4692..6bca20f1e8981 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -661,6 +661,7 @@ symbols! { core_panic_macro, coroutine, coroutine_clone, + coroutine_new_layout, coroutine_resume, coroutine_return, coroutine_state, diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs index 63421dfdce6cd..c29de06d1f7c1 100644 --- a/compiler/rustc_ty_utils/src/layout.rs +++ b/compiler/rustc_ty_utils/src/layout.rs @@ -31,6 +31,7 @@ use crate::errors::{ MultipleArrayFieldsSimdType, NonPrimitiveSimdType, OversizedSimdType, ZeroLengthSimdType, }; +mod coroutine; mod invariant; pub(crate) fn provide(providers: &mut Providers) { @@ -393,7 +394,13 @@ fn layout_of_uncached<'tcx>( tcx.mk_layout(unit) } - ty::Coroutine(def_id, args) => coroutine_layout(cx, ty, def_id, args)?, + ty::Coroutine(def_id, args) => { + if tcx.features().coroutine_new_layout() { + coroutine::coroutine_layout(cx, ty, def_id, args)? + } else { + coroutine_layout(cx, ty, def_id, args)? + } + } ty::Closure(_, args) => { let tys = args.as_closure().upvar_tys(); @@ -826,7 +833,7 @@ fn coroutine_layout<'tcx>( // Build a prefix layout, including "promoting" all ineligible // locals as part of the prefix. We compute the layout of all of // these fields at once to get optimal packing. - let tag_index = args.as_coroutine().prefix_tys().len(); + let tag_index = 0; // `info.variant_fields` already accounts for the reserved variants, so no need to add them. let max_discr = (info.variant_fields.len() - 1) as u128; @@ -845,14 +852,8 @@ fn coroutine_layout<'tcx>( let uninit_ty = Ty::new_maybe_uninit(tcx, field_ty); cx.spanned_layout_of(uninit_ty, info.field_tys[local].source_info.span) }); - let prefix_layouts = args - .as_coroutine() - .prefix_tys() - .iter() - .map(|ty| cx.layout_of(ty)) - .chain(iter::once(Ok(tag_layout))) - .chain(promoted_layouts) - .try_collect::>()?; + let prefix_layouts: IndexVec<_, _> = + [Ok(tag_layout)].into_iter().chain(promoted_layouts).try_collect()?; let prefix = univariant_uninterned( cx, ty, @@ -863,10 +864,10 @@ fn coroutine_layout<'tcx>( let (prefix_size, prefix_align) = (prefix.size, prefix.align); - // Split the prefix layout into the "outer" fields (upvars and - // discriminant) and the "promoted" fields. Promoted fields will - // get included in each variant that requested them in - // CoroutineLayout. + // Split the prefix layout into the discriminant and + // the "promoted" fields. + // Promoted fields will get included in each variant + // that requested them in CoroutineLayout. debug!("prefix = {:#?}", prefix); let (outer_fields, promoted_offsets, promoted_memory_index) = match prefix.fields { FieldsShape::Arbitrary { mut offsets, memory_index } => { @@ -1162,8 +1163,10 @@ fn variant_info_for_coroutine<'tcx>( .zip_eq(upvar_names) .enumerate() .map(|(field_idx, (_, name))| { - let field_layout = layout.field(cx, field_idx); - let offset = layout.fields.offset(field_idx); + // Upvars occupies the Unresumed variant at index zero + let variant_layout = layout.for_variant(cx, VariantIdx::ZERO); + let field_layout = variant_layout.field(cx, field_idx); + let offset = variant_layout.fields.offset(field_idx); upvars_size = upvars_size.max(offset + field_layout.size); FieldInfo { kind: FieldKind::Upvar, @@ -1183,12 +1186,11 @@ fn variant_info_for_coroutine<'tcx>( let variant_layout = layout.for_variant(cx, variant_idx); let mut variant_size = Size::ZERO; let fields = variant_def - .iter() - .enumerate() + .iter_enumerated() .map(|(field_idx, local)| { let field_name = coroutine.field_names[*local]; - let field_layout = variant_layout.field(cx, field_idx); - let offset = variant_layout.fields.offset(field_idx); + let field_layout = variant_layout.field(cx, field_idx.index()); + let offset = variant_layout.fields.offset(field_idx.index()); // The struct is as large as the last field's end variant_size = variant_size.max(offset + field_layout.size); FieldInfo { @@ -1206,7 +1208,11 @@ fn variant_info_for_coroutine<'tcx>( .then(|| Symbol::intern(&field_layout.ty.to_string())), } }) - .chain(upvar_fields.iter().copied()) + .chain( + if variant_idx.as_usize() == 0 { &upvar_fields[..] } else { &[] } + .iter() + .copied(), + ) .collect(); // If the variant has no state-specific fields, then it's the size of the upvars. diff --git a/compiler/rustc_ty_utils/src/layout/coroutine.rs b/compiler/rustc_ty_utils/src/layout/coroutine.rs new file mode 100644 index 0000000000000..a91d94140cc5a --- /dev/null +++ b/compiler/rustc_ty_utils/src/layout/coroutine.rs @@ -0,0 +1,289 @@ +use std::cmp::Reverse; +use std::collections::BTreeSet; +use std::fmt::Debug; +use std::ops::Bound; + +use rustc_abi::{ + BackendRepr, FieldIdx, FieldsShape, Integer, Layout, LayoutData, Primitive, Scalar, Size, + TagEncoding, TyAndLayout, Variants, WrappingRange, +}; +use rustc_index::bit_set::BitSet; +use rustc_index::{Idx, IndexVec}; +use rustc_middle::mir::CoroutineSavedLocal; +use rustc_middle::ty::layout::{HasTyCtxt, IntegerExt, LayoutCx, LayoutError, LayoutOf}; +use rustc_middle::ty::{EarlyBinder, GenericArgsRef, Ty}; +use tracing::{debug, instrument}; + +use super::error; + +#[instrument(level = "debug", skip(cx))] +pub(super) fn coroutine_layout<'tcx>( + cx: &LayoutCx<'tcx>, + ty: Ty<'tcx>, + def_id: rustc_hir::def_id::DefId, + args: GenericArgsRef<'tcx>, +) -> Result, &'tcx LayoutError<'tcx>> { + let tcx = cx.tcx(); + let Some(info) = tcx.coroutine_layout(def_id, args.as_coroutine().kind_ty()) else { + return Err(error(cx, LayoutError::Unknown(ty))); + }; + + let tcx = cx.tcx(); + let field_layouts: IndexVec = info + .field_tys + .iter_enumerated() + .map(|(saved_local, ty)| { + let ty = EarlyBinder::bind(ty.ty).instantiate(tcx, args); + cx.spanned_layout_of(ty, info.field_tys[saved_local].source_info.span) + }) + .try_collect()?; + let layouts: IndexVec = + field_layouts.iter().map(|data| data.layout.clone()).collect(); + + let field_sort_keys: IndexVec; + let mut saved_locals: Vec<_>; + // ## The heuristic on which saved locals get allocation first ## + // 1. the alignment + // Intuitively data with a larger alignment asks for a larger block of contiguous memory. + // It is easier to get large blocks early in the beginning, but it will get harder to + // recover them as fragmentation creeps in when data with smaller alignment occupies + // the large chunks. + // 2. the size + // The size also poses restriction on layout, but not as potent as alignment. + // 3. the degree of conflicts + // This metric is the number of confliciting saved locals with a given saved local. + // Preferring allocating highly conflicting data over those that are less and more + // transient in nature will keep the fragmentation contained in neighbourhoods of a layout. + (saved_locals, field_sort_keys) = field_layouts + .iter_enumerated() + .map(|(saved_local, ty)| { + ( + saved_local, + ( + Reverse(ty.align.abi), + Reverse(ty.size), + Reverse(info.storage_conflicts.count(saved_local)), + ), + ) + }) + .unzip(); + let mut uninhabited_or_zst = BitSet::new_empty(field_layouts.len()); + for (saved_local, ty) in field_layouts.iter_enumerated() { + if ty.backend_repr.is_uninhabited() || ty.is_zst() { + uninhabited_or_zst.insert(saved_local); + } + } + saved_locals.sort_by_key(|&idx| &field_sort_keys[idx]); + let max_discr = (info.variant_fields.len() - 1) as u128; + let discr_int = Integer::fit_unsigned(max_discr); + let tag = Scalar::Initialized { + value: Primitive::Int(discr_int, false), + valid_range: WrappingRange { start: 0, end: max_discr }, + }; + let tag_layout = TyAndLayout { + ty: discr_int.to_ty(tcx, false), + layout: tcx.mk_layout(LayoutData::scalar(cx, tag)), + }; + // This will be *the* align of the entire coroutine, + // which is the maximal alignment of all saved locals. + // We need to also consider the tag layout alignment. + let align = saved_locals + .get(0) + .map(|&idx| layouts[idx].align.max(tag_layout.align)) + .unwrap_or(tag_layout.align); + + // ## The blocked map, or the reservation map ## + // This map from saved locals to memory layout records the reservation + // status of the coroutine state memory, down to the byte granularity. + // `Slot`s are inserted to mark ranges of memory that a particular saved local + // shall not have overlapping memory allocation, due to the liveness of + // other conflicting saved locals. + // Therefore, we can try to make reservation for this saved local + // by inspecting the gaps before, between, and after those blocked-out memory ranges. + let mut blocked: IndexVec> = + IndexVec::from_elem_n(BTreeSet::new(), saved_locals.len()); + let mut tag_blocked = BTreeSet::new(); + let mut assignment = IndexVec::from_elem_n(Slot { start: 0, end: 0 }, saved_locals.len()); + for (idx, ¤t_local) in saved_locals.iter().enumerate() { + if uninhabited_or_zst.contains(current_local) { + // Do not bother to compute on uninhabited data. + // They will not get allocation after all. + // By default, a ZST occupies the beginning of the coroutine state. + continue; + } + let layout_data = &field_layouts[current_local]; + + let candidate = find_lowest_viable_allocation(&layout_data, &blocked[current_local]); + // The discriminant is certainly conflicting with all the saved locals + merge_slot_in(&mut tag_blocked, candidate); + for &other_local in &saved_locals[idx + 1..] { + if info.storage_conflicts.contains(current_local, other_local) { + merge_slot_in(&mut blocked[other_local], candidate); + } + } + // Adjustment to the layout of this field by shifting them into the chosen slot + assignment[current_local] = candidate; + } + debug!(assignment = ?assignment.debug_map_view()); + + // Find a slot for discriminant, also known as the tag. + let tag_candidate = find_lowest_viable_allocation(&tag_layout, &tag_blocked); + debug!(tag = ?tag_candidate); + + // Assemble the layout for each coroutine state + let variants: IndexVec<_, LayoutData<_, _>> = info + .variant_fields + .iter_enumerated() + .map(|(index, fields)| { + let size = Size::from_bytes( + fields + .iter() + .map(|&saved_local| assignment[saved_local].end) + .max() + .unwrap_or(0) + .max(tag_candidate.end), + ) + .align_to(align.abi); + let offsets: IndexVec<_, _> = fields + .iter() + .map(|&saved_local| Size::from_bytes(assignment[saved_local].start)) + .collect(); + let memory_index = IndexVec::from_fn_n(|n: FieldIdx| (n.index() as u32), offsets.len()); + LayoutData { + // We are aware of specialized layouts such as scalar pairs but this is still + // in development. + // Let us hold off from further optimisation until more information is available. + fields: FieldsShape::Arbitrary { offsets, memory_index }, + variants: Variants::Single { index }, + backend_repr: BackendRepr::Memory { sized: true }, + largest_niche: None, + align, + size, + max_repr_align: None, + unadjusted_abi_align: align.abi, + } + }) + .collect(); + let size = variants + .iter() + .map(|data| data.size) + .max() + .unwrap_or(Size::ZERO) + .max(Size::from_bytes(tag_candidate.end)) + .align_to(align.abi); + let layout = tcx.mk_layout(LayoutData { + fields: FieldsShape::Arbitrary { + offsets: [Size::from_bytes(tag_candidate.start)].into(), + memory_index: [0].into(), + }, + variants: Variants::Multiple { + tag, + tag_encoding: TagEncoding::Direct, + tag_field: 0, + variants, + }, + backend_repr: BackendRepr::Memory { sized: true }, + // Suppress niches inside coroutines. If the niche is inside a field that is aliased (due to + // self-referentiality), getting the discriminant can cause aliasing violations. + // `UnsafeCell` blocks niches for the same reason, but we don't yet have `UnsafePinned` that + // would do the same for us here. + // See , . + // FIXME(#125735): Remove when + // is implemented and aliased coroutine fields are wrapped in `UnsafePinned`. + // NOTE(@dingxiangfei2009): I believe there is still niche, which is the tag, + // but I am not sure how much benefit is there for us to grab. + largest_niche: None, + align, + size, + max_repr_align: None, + unadjusted_abi_align: align.abi, + }); + debug!("coroutine layout ({:?}): {:#?}", ty, layout); + Ok(layout) +} + +/// An occupied slot in the coroutine memory at some yield point +#[derive(PartialEq, Eq, Copy, Clone)] +struct Slot { + /// Beginning of the memory slot, inclusive + start: u64, + /// End of the memory slot, exclusive or one byte past the data + end: u64, +} + +impl PartialOrd for Slot { + fn partial_cmp(&self, other: &Self) -> Option { + (self.start, self.end).partial_cmp(&(other.start, other.end)) + } +} + +impl Ord for Slot { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + (self.start, self.end).cmp(&(other.start, other.end)) + } +} + +impl Debug for Slot { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("Slot").field(&self.start).field(&self.end).finish() + } +} + +impl Slot { + fn overlap_with(&self, other: &Self) -> bool { + if self.start == self.end || other.start == other.end { + return false; + } + self.contains_point(other.start) || other.contains_point(self.start) + } + fn contains_point(&self, point: u64) -> bool { + self.start <= point && point < self.end + } +} + +fn merge_slot_in(slots: &mut BTreeSet, slot: Slot) { + let start = Slot { start: slot.start, end: slot.start }; + let end = Slot { start: slot.end, end: slot.end }; + let one_past_end = Slot { start: slot.end + 1, end: slot.end + 1 }; + let (range_start, replace_start) = if let Some(prev) = slots.range(..start).next_back() + && (prev.end == slot.start || prev.contains_point(slot.start)) + { + (Bound::Included(prev), prev.start) + } else { + (Bound::Included(&start), slot.start) + }; + let (range_end, replace_end) = if let Some(next) = slots.range(..one_past_end).next_back() + && next.start == slot.end + { + (Bound::Included(next), next.end) + } else if let Some(prev) = slots.range(..end).next_back() + && prev.contains_point(slot.end) + { + (Bound::Included(prev), prev.end) + } else { + (Bound::Included(&end), slot.end) + }; + let to_remove: Vec<_> = slots.range((range_start, range_end)).copied().collect(); + for slot in to_remove { + slots.remove(&slot); + } + slots.insert(Slot { start: replace_start, end: replace_end }); +} + +fn find_lowest_viable_allocation( + layout: &LayoutData, + blocked: &BTreeSet, +) -> Slot { + let size = layout.size.bytes(); + let align = layout.align.abi; + let mut candidate = Slot { start: 0, end: size }; + for slot in blocked { + if slot.overlap_with(&candidate) { + let start = Size::from_bytes(slot.end).align_to(align).bytes(); + candidate = Slot { start, end: start + size }; + } else { + break; + } + } + candidate +} diff --git a/tests/debuginfo/coroutine-objects.rs b/tests/debuginfo/coroutine-objects.rs index 242c76c2989e3..2142e8f4b859f 100644 --- a/tests/debuginfo/coroutine-objects.rs +++ b/tests/debuginfo/coroutine-objects.rs @@ -10,22 +10,22 @@ // gdb-command:run // gdb-command:print b -// gdb-check:$1 = coroutine_objects::main::{coroutine_env#0}::Unresumed{_ref__a: 0x[...]} +// gdb-check:$1 = coroutine_objects::main::{coroutine_env#0}::Unresumed{a: 0x[...]} // gdb-command:continue // gdb-command:print b -// gdb-check:$2 = coroutine_objects::main::{coroutine_env#0}::Suspend0{c: 6, d: 7, _ref__a: 0x[...]} +// gdb-check:$2 = coroutine_objects::main::{coroutine_env#0}::Suspend0{c: 6, d: 7, a: 0x[...]} // gdb-command:continue // gdb-command:print b -// gdb-check:$3 = coroutine_objects::main::{coroutine_env#0}::Suspend1{c: 7, d: 8, _ref__a: 0x[...]} +// gdb-check:$3 = coroutine_objects::main::{coroutine_env#0}::Suspend1{c: 7, d: 8, a: 0x[...]} // gdb-command:continue // gdb-command:print b -// gdb-check:$4 = coroutine_objects::main::{coroutine_env#0}::Returned{_ref__a: 0x[...]} +// gdb-check:$4 = coroutine_objects::main::{coroutine_env#0}::Returned // === LLDB TESTS ================================================================================== // lldb-command:run // lldb-command:v b -// lldb-check:(coroutine_objects::main::{coroutine_env#0}) b = { value = { _ref__a = 0x[...] } $discr$ = [...] } +// lldb-check:(coroutine_objects::main::{coroutine_env#0}) b = { value = { a = 0x[...] } $discr$ = [...] } // === CDB TESTS =================================================================================== diff --git a/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir b/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir index 109a41d1ef907..fc73c7ba15456 100644 --- a/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir +++ b/tests/mir-opt/building/async_await.b-{closure#0}.coroutine_resume.0.mir @@ -1,7 +1,7 @@ // MIR for `b::{closure#0}` 0 coroutine_resume /* coroutine_layout = CoroutineLayout { field_tys: { - _0: CoroutineSavedTy { + corsl_0: CoroutineSavedTy { ty: Coroutine( DefId(0:5 ~ async_await[ccf8]::a::{closure#0}), [ @@ -22,7 +22,7 @@ }, ignore_for_traits: false, }, - _1: CoroutineSavedTy { + corsl_1: CoroutineSavedTy { ty: Coroutine( DefId(0:5 ~ async_await[ccf8]::a::{closure#0}), [ @@ -48,12 +48,12 @@ Unresumed(0): [], Returned (1): [], Panicked (2): [], - Suspend0 (3): [_0], - Suspend1 (4): [_1], + Suspend0 (3): [corsl_1], + Suspend1 (4): [corsl_0], }, storage_conflicts: BitMatrix(2x2) { - (_0, _0), - (_1, _1), + (corsl_0, corsl_0), + (corsl_1, corsl_1), }, } */ @@ -96,14 +96,14 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> let mut _38: &mut std::task::Context<'_>; let mut _39: u32; scope 1 { - debug __awaitee => (((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()}); + debug __awaitee => (((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()}); let _17: (); scope 2 { debug result => _17; } } scope 3 { - debug __awaitee => (((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()}); + debug __awaitee => (((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()}); let _33: (); scope 4 { debug result => _33; @@ -131,7 +131,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_5); PlaceMention(_4); nop; - (((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()}) = move _4; + (((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()}) = move _4; goto -> bb4; } @@ -141,7 +141,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageLive(_10); StorageLive(_11); StorageLive(_12); - _12 = &mut (((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()}); + _12 = &mut (((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()}); _11 = &mut (*_12); _10 = Pin::<&mut {async fn body of a()}>::new_unchecked(move _11) -> [return: bb5, unwind unreachable]; } @@ -188,7 +188,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_4); StorageDead(_19); StorageDead(_20); - discriminant((*(_1.0: &mut {async fn body of b()}))) = 3; + discriminant((*(_1.0: &mut {async fn body of b()}))) = 4; return; } @@ -201,7 +201,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_12); StorageDead(_9); StorageDead(_8); - drop((((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()})) -> [return: bb12, unwind unreachable]; + drop((((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()})) -> [return: bb12, unwind unreachable]; } bb11: { @@ -233,7 +233,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_22); PlaceMention(_21); nop; - (((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()}) = move _21; + (((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()}) = move _21; goto -> bb16; } @@ -243,7 +243,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageLive(_26); StorageLive(_27); StorageLive(_28); - _28 = &mut (((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()}); + _28 = &mut (((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()}); _27 = &mut (*_28); _26 = Pin::<&mut {async fn body of a()}>::new_unchecked(move _27) -> [return: bb17, unwind unreachable]; } @@ -285,7 +285,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_21); StorageDead(_35); StorageDead(_36); - discriminant((*(_1.0: &mut {async fn body of b()}))) = 4; + discriminant((*(_1.0: &mut {async fn body of b()}))) = 3; return; } @@ -298,7 +298,7 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> StorageDead(_28); StorageDead(_25); StorageDead(_24); - drop((((*(_1.0: &mut {async fn body of b()})) as variant#4).0: {async fn body of a()})) -> [return: bb23, unwind unreachable]; + drop((((*(_1.0: &mut {async fn body of b()})) as variant#3).0: {async fn body of a()})) -> [return: bb23, unwind unreachable]; } bb22: { @@ -330,6 +330,14 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> } bb27: { + StorageLive(_21); + StorageLive(_35); + StorageLive(_36); + _35 = move _2; + goto -> bb22; + } + + bb28: { StorageLive(_3); StorageLive(_4); StorageLive(_19); @@ -338,14 +346,6 @@ fn b::{closure#0}(_1: Pin<&mut {async fn body of b()}>, _2: &mut Context<'_>) -> goto -> bb11; } - bb28: { - StorageLive(_21); - StorageLive(_35); - StorageLive(_36); - _35 = move _2; - goto -> bb22; - } - bb29: { assert(const false, "`async fn` resumed after completion") -> [success: bb29, unwind unreachable]; } diff --git a/tests/mir-opt/coroutine_tiny.main-{closure#0}.coroutine_resume.0.mir b/tests/mir-opt/coroutine_tiny.main-{closure#0}.coroutine_resume.0.mir index f8b3f68d21e63..fe01ea115cd2b 100644 --- a/tests/mir-opt/coroutine_tiny.main-{closure#0}.coroutine_resume.0.mir +++ b/tests/mir-opt/coroutine_tiny.main-{closure#0}.coroutine_resume.0.mir @@ -1,7 +1,7 @@ // MIR for `main::{closure#0}` 0 coroutine_resume /* coroutine_layout = CoroutineLayout { field_tys: { - _0: CoroutineSavedTy { + corsl_0: CoroutineSavedTy { ty: HasDrop, source_info: SourceInfo { span: $DIR/coroutine_tiny.rs:22:13: 22:15 (#0), @@ -14,10 +14,10 @@ Unresumed(0): [], Returned (1): [], Panicked (2): [], - Suspend0 (3): [_0], + Suspend0 (3): [corsl_0], }, storage_conflicts: BitMatrix(1x1) { - (_0, _0), + (corsl_0, corsl_0), }, } */ diff --git a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff index 1e33e222b27f6..f4df5290d35a3 100644 --- a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff +++ b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-abort.diff @@ -38,17 +38,19 @@ + let mut _27: !; + let mut _28: &mut std::task::Context<'_>; + let mut _29: (); -+ let mut _30: (); -+ let mut _31: &mut std::task::Context<'_>; -+ let mut _32: u32; -+ let mut _33: &mut {async fn body of ActionPermit<'_, T>::perform()}; -+ let mut _34: &mut {async fn body of ActionPermit<'_, T>::perform()}; ++ let _30: ActionPermit<'_, T>; ++ let _31: ActionPermit<'_, T>; ++ let mut _32: (); ++ let mut _33: &mut std::task::Context<'_>; ++ let mut _34: u32; + let mut _35: &mut {async fn body of ActionPermit<'_, T>::perform()}; + let mut _36: &mut {async fn body of ActionPermit<'_, T>::perform()}; + let mut _37: &mut {async fn body of ActionPermit<'_, T>::perform()}; + let mut _38: &mut {async fn body of ActionPermit<'_, T>::perform()}; + let mut _39: &mut {async fn body of ActionPermit<'_, T>::perform()}; + let mut _40: &mut {async fn body of ActionPermit<'_, T>::perform()}; ++ let mut _41: &mut {async fn body of ActionPermit<'_, T>::perform()}; ++ let mut _42: &mut {async fn body of ActionPermit<'_, T>::perform()}; + scope 7 { + let mut _15: std::future::Ready<()>; + scope 8 { @@ -57,7 +59,7 @@ + } + } + scope 10 (inlined ready::<()>) { -+ let mut _41: std::option::Option<()>; ++ let mut _43: std::option::Option<()>; + } + } + } @@ -100,8 +102,6 @@ + StorageLive(_16); + StorageLive(_25); + StorageLive(_27); -+ StorageLive(_30); -+ StorageLive(_31); + StorageLive(_32); + StorageLive(_33); + StorageLive(_34); @@ -111,9 +111,11 @@ + StorageLive(_38); + StorageLive(_39); + StorageLive(_40); -+ _33 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); -+ _32 = discriminant((*_33)); -+ switchInt(move _32) -> [0: bb3, 1: bb13, 3: bb12, otherwise: bb8]; ++ StorageLive(_41); ++ StorageLive(_42); ++ _35 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); ++ _34 = discriminant((*_35)); ++ switchInt(move _34) -> [0: bb3, 1: bb13, 3: bb12, otherwise: bb8]; } - bb3: { @@ -123,6 +125,8 @@ + } + + bb2: { ++ StorageDead(_42); ++ StorageDead(_41); + StorageDead(_40); + StorageDead(_39); + StorageDead(_38); @@ -132,8 +136,6 @@ + StorageDead(_34); + StorageDead(_33); + StorageDead(_32); -+ StorageDead(_31); -+ StorageDead(_30); + StorageDead(_27); + StorageDead(_25); + StorageDead(_16); @@ -151,18 +153,23 @@ } + bb3: { -+ _31 = move _9; -+ _34 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); -+ _35 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); -+ (((*_34) as variant#3).0: ActionPermit<'_, T>) = move ((*_35).0: ActionPermit<'_, T>); ++ _33 = move _9; ++ StorageLive(_30); ++ StorageLive(_31); ++ _36 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); ++ _31 = move (((*_36) as variant#0).0: ActionPermit<'_, T>); ++ _30 = move _31; ++ StorageDead(_31); ++ _42 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); ++ (((*_42) as variant#3).0: ActionPermit<'_, T>) = move _30; + StorageLive(_12); + StorageLive(_13); + StorageLive(_14); + _14 = (); -+ StorageLive(_41); -+ _41 = Option::<()>::Some(copy _14); -+ _13 = std::future::Ready::<()>(move _41); -+ StorageDead(_41); ++ StorageLive(_43); ++ _43 = Option::<()>::Some(copy _14); ++ _13 = std::future::Ready::<()>(move _43); ++ StorageDead(_43); + StorageDead(_14); + _12 = as IntoFuture>::into_future(move _13) -> [return: bb4, unwind unreachable]; + } @@ -171,8 +178,8 @@ - StorageDead(_2); - return; + StorageDead(_13); -+ _36 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); -+ (((*_36) as variant#3).1: std::future::Ready<()>) = move _12; ++ _37 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); ++ (((*_37) as variant#3).1: std::future::Ready<()>) = move _12; + goto -> bb5; + } + @@ -182,8 +189,8 @@ + StorageLive(_19); + StorageLive(_20); + StorageLive(_21); -+ _37 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); -+ _21 = &mut (((*_37) as variant#3).1: std::future::Ready<()>); ++ _38 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); ++ _21 = &mut (((*_38) as variant#3).1: std::future::Ready<()>); + _20 = &mut (*_21); + _19 = Pin::<&mut std::future::Ready<()>>::new_unchecked(move _20) -> [return: bb6, unwind unreachable]; + } @@ -193,7 +200,7 @@ + StorageLive(_22); + StorageLive(_23); + StorageLive(_24); -+ _24 = copy _31; ++ _24 = copy _33; + _23 = move _24; + _22 = &mut (*_23); + StorageDead(_24); @@ -224,29 +231,31 @@ + StorageDead(_12); + StorageDead(_28); + StorageDead(_29); -+ _38 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); -+ discriminant((*_38)) = 3; ++ StorageDead(_30); ++ _39 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); ++ discriminant((*_39)) = 3; + goto -> bb2; + } + + bb10: { + StorageLive(_26); + _26 = copy ((_18 as Ready).0: ()); -+ _30 = copy _26; ++ _32 = copy _26; + StorageDead(_26); + StorageDead(_23); + StorageDead(_21); + StorageDead(_18); + StorageDead(_17); + StorageDead(_12); -+ _39 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); -+ drop((((*_39) as variant#3).0: ActionPermit<'_, T>)) -> [return: bb11, unwind unreachable]; ++ _40 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); ++ drop((((*_40) as variant#3).0: ActionPermit<'_, T>)) -> [return: bb11, unwind unreachable]; + } + + bb11: { -+ _7 = Poll::<()>::Ready(move _30); -+ _40 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); -+ discriminant((*_40)) = 1; ++ StorageDead(_30); ++ _7 = Poll::<()>::Ready(move _32); ++ _41 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); ++ discriminant((*_41)) = 1; + goto -> bb2; + } + @@ -254,9 +263,10 @@ + StorageLive(_12); + StorageLive(_28); + StorageLive(_29); ++ StorageLive(_30); + _28 = move _9; + StorageDead(_29); -+ _31 = move _28; ++ _33 = move _28; + StorageDead(_28); + _16 = const (); + goto -> bb5; diff --git a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff index b1840beb3efcc..006d24f3ef0c0 100644 --- a/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline_coroutine_body.run2-{closure#0}.Inline.panic-unwind.diff @@ -38,11 +38,11 @@ + let mut _27: !; + let mut _28: &mut std::task::Context<'_>; + let mut _29: (); -+ let mut _30: (); -+ let mut _31: &mut std::task::Context<'_>; -+ let mut _32: u32; -+ let mut _33: &mut {async fn body of ActionPermit<'_, T>::perform()}; -+ let mut _34: &mut {async fn body of ActionPermit<'_, T>::perform()}; ++ let _30: ActionPermit<'_, T>; ++ let _31: ActionPermit<'_, T>; ++ let mut _32: (); ++ let mut _33: &mut std::task::Context<'_>; ++ let mut _34: u32; + let mut _35: &mut {async fn body of ActionPermit<'_, T>::perform()}; + let mut _36: &mut {async fn body of ActionPermit<'_, T>::perform()}; + let mut _37: &mut {async fn body of ActionPermit<'_, T>::perform()}; @@ -51,6 +51,8 @@ + let mut _40: &mut {async fn body of ActionPermit<'_, T>::perform()}; + let mut _41: &mut {async fn body of ActionPermit<'_, T>::perform()}; + let mut _42: &mut {async fn body of ActionPermit<'_, T>::perform()}; ++ let mut _43: &mut {async fn body of ActionPermit<'_, T>::perform()}; ++ let mut _44: &mut {async fn body of ActionPermit<'_, T>::perform()}; + scope 7 { + let mut _15: std::future::Ready<()>; + scope 8 { @@ -59,7 +61,7 @@ + } + } + scope 10 (inlined ready::<()>) { -+ let mut _43: std::option::Option<()>; ++ let mut _45: std::option::Option<()>; + } + } + } @@ -102,8 +104,6 @@ + StorageLive(_16); + StorageLive(_25); + StorageLive(_27); -+ StorageLive(_30); -+ StorageLive(_31); + StorageLive(_32); + StorageLive(_33); + StorageLive(_34); @@ -115,9 +115,11 @@ + StorageLive(_40); + StorageLive(_41); + StorageLive(_42); -+ _33 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); -+ _32 = discriminant((*_33)); -+ switchInt(move _32) -> [0: bb5, 1: bb22, 2: bb21, 3: bb20, otherwise: bb10]; ++ StorageLive(_43); ++ StorageLive(_44); ++ _35 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); ++ _34 = discriminant((*_35)); ++ switchInt(move _34) -> [0: bb5, 1: bb22, 2: bb21, 3: bb20, otherwise: bb10]; } - bb3: { @@ -135,6 +137,8 @@ + } + + bb4: { ++ StorageDead(_44); ++ StorageDead(_43); + StorageDead(_42); + StorageDead(_41); + StorageDead(_40); @@ -146,8 +150,6 @@ + StorageDead(_34); + StorageDead(_33); + StorageDead(_32); -+ StorageDead(_31); -+ StorageDead(_30); + StorageDead(_27); + StorageDead(_25); + StorageDead(_16); @@ -168,28 +170,33 @@ - StorageDead(_2); - return; + bb5: { -+ _31 = move _9; -+ _34 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); -+ _35 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); -+ (((*_34) as variant#3).0: ActionPermit<'_, T>) = move ((*_35).0: ActionPermit<'_, T>); ++ _33 = move _9; ++ StorageLive(_30); ++ StorageLive(_31); ++ _36 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); ++ _31 = move (((*_36) as variant#0).0: ActionPermit<'_, T>); ++ _30 = move _31; ++ StorageDead(_31); ++ _43 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); ++ (((*_43) as variant#3).0: ActionPermit<'_, T>) = move _30; + StorageLive(_12); + StorageLive(_13); + StorageLive(_14); + _14 = (); -+ StorageLive(_43); -+ _43 = Option::<()>::Some(copy _14); -+ _13 = std::future::Ready::<()>(move _43); -+ StorageDead(_43); ++ StorageLive(_45); ++ _45 = Option::<()>::Some(copy _14); ++ _13 = std::future::Ready::<()>(move _45); ++ StorageDead(_45); + StorageDead(_14); -+ _12 = as IntoFuture>::into_future(move _13) -> [return: bb6, unwind: bb17]; ++ _12 = as IntoFuture>::into_future(move _13) -> [return: bb6, unwind: bb16]; } - bb5 (cleanup): { - drop(_2) -> [return: bb6, unwind terminate(cleanup)]; + bb6: { + StorageDead(_13); -+ _36 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); -+ (((*_36) as variant#3).1: std::future::Ready<()>) = move _12; ++ _37 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); ++ (((*_37) as variant#3).1: std::future::Ready<()>) = move _12; + goto -> bb7; } @@ -201,10 +208,10 @@ + StorageLive(_19); + StorageLive(_20); + StorageLive(_21); -+ _37 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); -+ _21 = &mut (((*_37) as variant#3).1: std::future::Ready<()>); ++ _38 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); ++ _21 = &mut (((*_38) as variant#3).1: std::future::Ready<()>); + _20 = &mut (*_21); -+ _19 = Pin::<&mut std::future::Ready<()>>::new_unchecked(move _20) -> [return: bb8, unwind: bb15]; ++ _19 = Pin::<&mut std::future::Ready<()>>::new_unchecked(move _20) -> [return: bb8, unwind: bb14]; + } + + bb8: { @@ -212,11 +219,11 @@ + StorageLive(_22); + StorageLive(_23); + StorageLive(_24); -+ _24 = copy _31; ++ _24 = copy _33; + _23 = move _24; + _22 = &mut (*_23); + StorageDead(_24); -+ _18 = as Future>::poll(move _19, move _22) -> [return: bb9, unwind: bb14]; ++ _18 = as Future>::poll(move _19, move _22) -> [return: bb9, unwind: bb13]; + } + + bb9: { @@ -243,66 +250,68 @@ + StorageDead(_12); + StorageDead(_28); + StorageDead(_29); -+ _38 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); -+ discriminant((*_38)) = 3; ++ StorageDead(_30); ++ _39 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); ++ discriminant((*_39)) = 3; + goto -> bb4; + } + + bb12: { + StorageLive(_26); + _26 = copy ((_18 as Ready).0: ()); -+ _30 = copy _26; ++ _32 = copy _26; + StorageDead(_26); + StorageDead(_23); + StorageDead(_21); + StorageDead(_18); + StorageDead(_17); + StorageDead(_12); -+ _39 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); -+ drop((((*_39) as variant#3).0: ActionPermit<'_, T>)) -> [return: bb13, unwind: bb19]; -+ } -+ -+ bb13: { -+ _7 = Poll::<()>::Ready(move _30); + _40 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); -+ discriminant((*_40)) = 1; -+ goto -> bb4; ++ drop((((*_40) as variant#3).0: ActionPermit<'_, T>)) -> [return: bb18, unwind: bb19]; + } + -+ bb14 (cleanup): { ++ bb13 (cleanup): { + StorageDead(_22); + StorageDead(_19); + StorageDead(_23); -+ goto -> bb16; ++ goto -> bb15; + } + -+ bb15 (cleanup): { ++ bb14 (cleanup): { + StorageDead(_20); + StorageDead(_19); -+ goto -> bb16; ++ goto -> bb15; + } + -+ bb16 (cleanup): { ++ bb15 (cleanup): { + StorageDead(_21); + StorageDead(_18); + StorageDead(_17); -+ goto -> bb18; ++ goto -> bb17; + } + -+ bb17 (cleanup): { ++ bb16 (cleanup): { + StorageDead(_13); -+ goto -> bb18; ++ goto -> bb17; + } + -+ bb18 (cleanup): { ++ bb17 (cleanup): { + StorageDead(_12); + _41 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); + drop((((*_41) as variant#3).0: ActionPermit<'_, T>)) -> [return: bb19, unwind terminate(cleanup)]; + } + -+ bb19 (cleanup): { ++ bb18: { ++ StorageDead(_30); ++ _7 = Poll::<()>::Ready(move _32); + _42 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); -+ discriminant((*_42)) = 2; ++ discriminant((*_42)) = 1; ++ goto -> bb4; ++ } ++ ++ bb19 (cleanup): { ++ _44 = deref_copy (_8.0: &mut {async fn body of ActionPermit<'_, T>::perform()}); ++ discriminant((*_44)) = 2; + goto -> bb2; + } + @@ -310,9 +319,10 @@ + StorageLive(_12); + StorageLive(_28); + StorageLive(_29); ++ StorageLive(_30); + _28 = move _9; + StorageDead(_29); -+ _31 = move _28; ++ _33 = move _28; + StorageDead(_28); + _16 = const (); + goto -> bb7; diff --git a/tests/ui/async-await/async-drop.rs b/tests/ui/async-await/async-drop.rs index 4e60598661faf..daca511d90597 100644 --- a/tests/ui/async-await/async-drop.rs +++ b/tests/ui/async-await/async-drop.rs @@ -11,10 +11,10 @@ //@ edition: 2021 // FIXME(zetanumbers): consider AsyncDestruct::async_drop cleanup tests -use core::future::{async_drop_in_place, AsyncDrop, Future}; +use core::future::{AsyncDrop, Future, async_drop_in_place}; use core::hint::black_box; use core::mem::{self, ManuallyDrop}; -use core::pin::{pin, Pin}; +use core::pin::{Pin, pin}; use core::task::{Context, Poll, Waker}; async fn test_async_drop(x: T, _size: usize) { @@ -26,8 +26,8 @@ async fn test_async_drop(x: T, _size: usize) { // async functions. #[cfg(target_pointer_width = "64")] assert_eq!( - mem::size_of_val(&*dtor), _size, + mem::size_of_val(&*dtor), "sizes did not match for async destructor of type {}", core::any::type_name::(), ); @@ -53,20 +53,18 @@ fn main() { let i = 13; let fut = pin!(async { test_async_drop(Int(0), 0).await; - // FIXME(#63818): niches in coroutines are disabled. - // Some of these sizes should be smaller, as indicated in comments. - test_async_drop(AsyncInt(0), /*104*/ 112).await; - test_async_drop([AsyncInt(1), AsyncInt(2)], /*152*/ 168).await; - test_async_drop((AsyncInt(3), AsyncInt(4)), /*488*/ 528).await; + test_async_drop(AsyncInt(0), 56).await; + test_async_drop([AsyncInt(1), AsyncInt(2)], 96).await; + test_async_drop((AsyncInt(3), AsyncInt(4)), 144).await; test_async_drop(5, 0).await; let j = 42; test_async_drop(&i, 0).await; test_async_drop(&j, 0).await; - test_async_drop(AsyncStruct { b: AsyncInt(8), a: AsyncInt(7), i: 6 }, /*1688*/ 1792).await; + test_async_drop(AsyncStruct { b: AsyncInt(8), a: AsyncInt(7), i: 6 }, 200).await; test_async_drop(ManuallyDrop::new(AsyncInt(9)), 0).await; let foo = AsyncInt(10); - test_async_drop(AsyncReference { foo: &foo }, /*104*/ 112).await; + test_async_drop(AsyncReference { foo: &foo }, 56).await; let foo = AsyncInt(11); test_async_drop( @@ -75,17 +73,17 @@ fn main() { let foo = AsyncInt(10); foo }, - /*120*/ 136, + 72, ) .await; - test_async_drop(AsyncEnum::A(AsyncInt(12)), /*680*/ 736).await; - test_async_drop(AsyncEnum::B(SyncInt(13)), /*680*/ 736).await; + test_async_drop(AsyncEnum::A(AsyncInt(12)), 168).await; + test_async_drop(AsyncEnum::B(SyncInt(13)), 168).await; test_async_drop(SyncInt(14), /*16*/ 24).await; test_async_drop( SyncThenAsync { i: 15, a: AsyncInt(16), b: SyncInt(17), c: AsyncInt(18) }, - /*3064*/ 3296, + 232, ) .await; @@ -101,11 +99,11 @@ fn main() { black_box(core::future::ready(())).await; foo }, - /*120*/ 136, + 72, ) .await; - test_async_drop(AsyncUnion { signed: 21 }, /*32*/ 40).await; + test_async_drop(AsyncUnion { signed: 21 }, 32).await; }); let res = fut.poll(&mut cx); assert_eq!(res, Poll::Ready(())); @@ -149,7 +147,10 @@ struct AsyncReference<'a> { } impl AsyncDrop for AsyncReference<'_> { - type Dropper<'a> = impl Future where Self: 'a; + type Dropper<'a> + = impl Future + where + Self: 'a; fn async_drop(self: Pin<&mut Self>) -> Self::Dropper<'_> { async move { @@ -212,11 +213,9 @@ impl AsyncDrop for AsyncUnion { fn async_drop(self: Pin<&mut Self>) -> Self::Dropper<'_> { async move { - println!( - "AsyncUnion::Dropper::poll: {}, {}", - unsafe { self.signed }, - unsafe { self.unsigned }, - ); + println!("AsyncUnion::Dropper::poll: {}, {}", unsafe { self.signed }, unsafe { + self.unsigned + },); } } } diff --git a/tests/ui/async-await/awaiting-unsized-param.rs b/tests/ui/async-await/awaiting-unsized-param.rs index 45611eae41f5c..b00b23c1139ee 100644 --- a/tests/ui/async-await/awaiting-unsized-param.rs +++ b/tests/ui/async-await/awaiting-unsized-param.rs @@ -7,6 +7,7 @@ use std::future::Future; async fn bug(mut f: dyn Future + Unpin) -> T { //~^ ERROR the size for values of type `(dyn Future + Unpin + 'static)` cannot be known at compilation time + //~| ERROR the size for values of type `dyn Future + Unpin` cannot be known at compilation time (&mut f).await } diff --git a/tests/ui/async-await/awaiting-unsized-param.stderr b/tests/ui/async-await/awaiting-unsized-param.stderr index 0104736976d5d..4095ca21007b3 100644 --- a/tests/ui/async-await/awaiting-unsized-param.stderr +++ b/tests/ui/async-await/awaiting-unsized-param.stderr @@ -16,6 +16,15 @@ LL | async fn bug(mut f: dyn Future + Unpin) -> T { = help: the trait `Sized` is not implemented for `(dyn Future + Unpin + 'static)` = note: all values captured by value by a closure must have a statically known size -error: aborting due to 1 previous error; 1 warning emitted +error[E0277]: the size for values of type `dyn Future + Unpin` cannot be known at compilation time + --> $DIR/awaiting-unsized-param.rs:8:21 + | +LL | async fn bug(mut f: dyn Future + Unpin) -> T { + | ^ doesn't have a size known at compile-time + | + = help: the trait `Sized` is not implemented for `dyn Future + Unpin` + = note: all values live across `await` must have a statically known size + +error: aborting due to 2 previous errors; 1 warning emitted For more information about this error, try `rustc --explain E0277`. diff --git a/tests/ui/async-await/future-sizes/async-awaiting-fut.stdout b/tests/ui/async-await/future-sizes/async-awaiting-fut.stdout index 642e27b2a57d6..99506c95e6203 100644 --- a/tests/ui/async-await/future-sizes/async-awaiting-fut.stdout +++ b/tests/ui/async-await/future-sizes/async-awaiting-fut.stdout @@ -1,39 +1,36 @@ -print-type-size type: `{async fn body of test()}`: 3078 bytes, alignment: 1 bytes +print-type-size type: `{async fn body of test()}`: 2053 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 0 bytes -print-type-size variant `Suspend0`: 3077 bytes -print-type-size local `.__awaitee`: 3077 bytes, type: {async fn body of calls_fut<{async fn body of big_fut()}>()} +print-type-size variant `Suspend0`: 2052 bytes +print-type-size local `.__awaitee`: 2052 bytes, type: {async fn body of calls_fut<{async fn body of big_fut()}>()} print-type-size variant `Returned`: 0 bytes print-type-size variant `Panicked`: 0 bytes -print-type-size type: `std::mem::ManuallyDrop<{async fn body of calls_fut<{async fn body of big_fut()}>()}>`: 3077 bytes, alignment: 1 bytes -print-type-size field `.value`: 3077 bytes -print-type-size type: `std::mem::MaybeUninit<{async fn body of calls_fut<{async fn body of big_fut()}>()}>`: 3077 bytes, alignment: 1 bytes -print-type-size variant `MaybeUninit`: 3077 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of calls_fut<{async fn body of big_fut()}>()}>`: 2052 bytes, alignment: 1 bytes +print-type-size field `.value`: 2052 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of calls_fut<{async fn body of big_fut()}>()}>`: 2052 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 2052 bytes print-type-size field `.uninit`: 0 bytes -print-type-size field `.value`: 3077 bytes -print-type-size type: `{async fn body of calls_fut<{async fn body of big_fut()}>()}`: 3077 bytes, alignment: 1 bytes +print-type-size field `.value`: 2052 bytes +print-type-size type: `{async fn body of calls_fut<{async fn body of big_fut()}>()}`: 2052 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes -print-type-size variant `Unresumed`: 1025 bytes -print-type-size upvar `.fut`: 1025 bytes -print-type-size variant `Suspend0`: 2052 bytes -print-type-size upvar `.fut`: 1025 bytes +print-type-size variant `Unresumed`: 2051 bytes +print-type-size padding: 1026 bytes +print-type-size local `.fut`: 1025 bytes, alignment: 1 bytes +print-type-size upvar `.fut`: 1025 bytes, offset: 1027 bytes, alignment: 1 bytes +print-type-size variant `Suspend0`: 1027 bytes print-type-size local `.fut`: 1025 bytes print-type-size local `..coroutine_field4`: 1 bytes, type: bool print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} -print-type-size variant `Suspend1`: 3076 bytes -print-type-size upvar `.fut`: 1025 bytes +print-type-size variant `Suspend1`: 2051 bytes print-type-size padding: 1025 bytes print-type-size local `..coroutine_field4`: 1 bytes, alignment: 1 bytes, type: bool print-type-size local `.__awaitee`: 1025 bytes, type: {async fn body of big_fut()} -print-type-size variant `Suspend2`: 2052 bytes -print-type-size upvar `.fut`: 1025 bytes +print-type-size variant `Suspend2`: 1027 bytes print-type-size local `.fut`: 1025 bytes print-type-size local `..coroutine_field4`: 1 bytes, type: bool print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} -print-type-size variant `Returned`: 1025 bytes -print-type-size upvar `.fut`: 1025 bytes -print-type-size variant `Panicked`: 1025 bytes -print-type-size upvar `.fut`: 1025 bytes +print-type-size variant `Returned`: 2051 bytes +print-type-size variant `Panicked`: 2051 bytes print-type-size type: `std::mem::ManuallyDrop<{async fn body of big_fut()}>`: 1025 bytes, alignment: 1 bytes print-type-size field `.value`: 1025 bytes print-type-size type: `std::mem::MaybeUninit<{async fn body of big_fut()}>`: 1025 bytes, alignment: 1 bytes @@ -43,11 +40,16 @@ print-type-size field `.value`: 1025 bytes print-type-size type: `{async fn body of big_fut()}`: 1025 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 1024 bytes -print-type-size upvar `.arg`: 1024 bytes +print-type-size local `.arg`: 1024 bytes +print-type-size upvar `.arg`: 1024 bytes, offset: 1 bytes, alignment: 1 bytes print-type-size variant `Returned`: 1024 bytes -print-type-size upvar `.arg`: 1024 bytes print-type-size variant `Panicked`: 1024 bytes -print-type-size upvar `.arg`: 1024 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1024 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1024 bytes print-type-size type: `std::mem::ManuallyDrop`: 1 bytes, alignment: 1 bytes print-type-size field `.value`: 1 bytes print-type-size type: `std::mem::ManuallyDrop<{async fn body of wait()}>`: 1 bytes, alignment: 1 bytes diff --git a/tests/ui/async-await/future-sizes/future-as-arg.rs b/tests/ui/async-await/future-sizes/future-as-arg.rs index 317c17b9b3e59..3bbbadc2099df 100644 --- a/tests/ui/async-await/future-sizes/future-as-arg.rs +++ b/tests/ui/async-await/future-sizes/future-as-arg.rs @@ -8,9 +8,10 @@ async fn use_future(fut: impl std::future::Future) { } fn main() { - let actual = std::mem::size_of_val( - &use_future(use_future(use_future(use_future(use_future(test([0; 16]))))))); + let actual = std::mem::size_of_val(&use_future(use_future(use_future(use_future( + use_future(test([0; 16])), + ))))); // Not using an exact number in case it slightly changes over different commits - let expected = 550; - assert!(actual > expected, "expected: >{expected}, actual: {actual}"); + let expected = 21; + assert!(actual >= expected, "expected: >{expected}, actual: {actual}"); } diff --git a/tests/ui/async-await/future-sizes/large-arg.stdout b/tests/ui/async-await/future-sizes/large-arg.stdout index 67168a3d6ef74..03d4125f916f1 100644 --- a/tests/ui/async-await/future-sizes/large-arg.stdout +++ b/tests/ui/async-await/future-sizes/large-arg.stdout @@ -1,44 +1,40 @@ -print-type-size type: `{async fn body of test()}`: 3076 bytes, alignment: 1 bytes +print-type-size type: `{async fn body of test()}`: 1028 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 0 bytes -print-type-size variant `Suspend0`: 3075 bytes -print-type-size local `.__awaitee`: 3075 bytes, type: {async fn body of a<[u8; 1024]>()} +print-type-size variant `Suspend0`: 1027 bytes +print-type-size local `.__awaitee`: 1027 bytes, type: {async fn body of a<[u8; 1024]>()} print-type-size variant `Returned`: 0 bytes print-type-size variant `Panicked`: 0 bytes -print-type-size type: `std::mem::ManuallyDrop<{async fn body of a<[u8; 1024]>()}>`: 3075 bytes, alignment: 1 bytes -print-type-size field `.value`: 3075 bytes -print-type-size type: `std::mem::MaybeUninit<{async fn body of a<[u8; 1024]>()}>`: 3075 bytes, alignment: 1 bytes -print-type-size variant `MaybeUninit`: 3075 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of a<[u8; 1024]>()}>`: 1027 bytes, alignment: 1 bytes +print-type-size field `.value`: 1027 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of a<[u8; 1024]>()}>`: 1027 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1027 bytes print-type-size field `.uninit`: 0 bytes -print-type-size field `.value`: 3075 bytes -print-type-size type: `{async fn body of a<[u8; 1024]>()}`: 3075 bytes, alignment: 1 bytes +print-type-size field `.value`: 1027 bytes +print-type-size type: `{async fn body of a<[u8; 1024]>()}`: 1027 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size variant `Suspend0`: 3074 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size local `.__awaitee`: 2050 bytes, type: {async fn body of b<[u8; 1024]>()} +print-type-size local `.t`: 1024 bytes +print-type-size upvar `.t`: 1024 bytes, offset: 1 bytes, alignment: 1 bytes +print-type-size variant `Suspend0`: 1026 bytes +print-type-size local `.__awaitee`: 1026 bytes, type: {async fn body of b<[u8; 1024]>()} print-type-size variant `Returned`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes print-type-size variant `Panicked`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size type: `std::mem::ManuallyDrop<{async fn body of b<[u8; 1024]>()}>`: 2050 bytes, alignment: 1 bytes -print-type-size field `.value`: 2050 bytes -print-type-size type: `std::mem::MaybeUninit<{async fn body of b<[u8; 1024]>()}>`: 2050 bytes, alignment: 1 bytes -print-type-size variant `MaybeUninit`: 2050 bytes +print-type-size type: `std::mem::ManuallyDrop<{async fn body of b<[u8; 1024]>()}>`: 1026 bytes, alignment: 1 bytes +print-type-size field `.value`: 1026 bytes +print-type-size type: `std::mem::MaybeUninit<{async fn body of b<[u8; 1024]>()}>`: 1026 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1026 bytes print-type-size field `.uninit`: 0 bytes -print-type-size field `.value`: 2050 bytes -print-type-size type: `{async fn body of b<[u8; 1024]>()}`: 2050 bytes, alignment: 1 bytes +print-type-size field `.value`: 1026 bytes +print-type-size type: `{async fn body of b<[u8; 1024]>()}`: 1026 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes -print-type-size variant `Suspend0`: 2049 bytes -print-type-size upvar `.t`: 1024 bytes +print-type-size local `.t`: 1024 bytes +print-type-size upvar `.t`: 1024 bytes, offset: 1 bytes, alignment: 1 bytes +print-type-size variant `Suspend0`: 1025 bytes print-type-size local `.__awaitee`: 1025 bytes, type: {async fn body of c<[u8; 1024]>()} print-type-size variant `Returned`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes print-type-size variant `Panicked`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes print-type-size type: `std::mem::ManuallyDrop<{async fn body of c<[u8; 1024]>()}>`: 1025 bytes, alignment: 1 bytes print-type-size field `.value`: 1025 bytes print-type-size type: `std::mem::MaybeUninit<{async fn body of c<[u8; 1024]>()}>`: 1025 bytes, alignment: 1 bytes @@ -53,8 +49,13 @@ print-type-size variant `Pending`: 0 bytes print-type-size type: `{async fn body of c<[u8; 1024]>()}`: 1025 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes +print-type-size local `.t`: 1024 bytes +print-type-size upvar `.t`: 1024 bytes, offset: 1 bytes, alignment: 1 bytes print-type-size variant `Returned`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes print-type-size variant `Panicked`: 1024 bytes -print-type-size upvar `.t`: 1024 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size field `.value`: 1024 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 1024]>`: 1024 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 1024 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 1024 bytes diff --git a/tests/ui/async-await/in-trait/async-recursive-generic.stderr b/tests/ui/async-await/in-trait/async-recursive-generic.stderr index d085747bc4bf2..169f5b8bcd2fd 100644 --- a/tests/ui/async-await/in-trait/async-recursive-generic.stderr +++ b/tests/ui/async-await/in-trait/async-recursive-generic.stderr @@ -3,9 +3,6 @@ error[E0733]: recursion in an async fn requires boxing | LL | async fn foo_recursive(&self, n: usize) -> T { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -... -LL | self.foo_recursive(n - 1).await - | ------------------------------- recursive call here | = note: a recursive `async fn` call must introduce indirection such as `Box::pin` to avoid an infinitely sized future diff --git a/tests/ui/async-await/in-trait/async-recursive.stderr b/tests/ui/async-await/in-trait/async-recursive.stderr index 25ebc6e77c4fb..5e8ed60ec1b81 100644 --- a/tests/ui/async-await/in-trait/async-recursive.stderr +++ b/tests/ui/async-await/in-trait/async-recursive.stderr @@ -3,9 +3,6 @@ error[E0733]: recursion in an async fn requires boxing | LL | async fn foo_recursive(&self, n: usize) -> i32 { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -... -LL | self.foo_recursive(n - 1).await - | ------------------------------- recursive call here | = note: a recursive `async fn` call must introduce indirection such as `Box::pin` to avoid an infinitely sized future diff --git a/tests/ui/async-await/issue-70818.rs b/tests/ui/async-await/issue-70818.rs index 36295a84e7ad7..837d532e067d1 100644 --- a/tests/ui/async-await/issue-70818.rs +++ b/tests/ui/async-await/issue-70818.rs @@ -3,6 +3,7 @@ use std::future::Future; fn foo(ty: T, ty1: U) -> impl Future + Send { //~^ Error future cannot be sent between threads safely + //~| Error future cannot be sent between threads safely async { (ty, ty1) } } diff --git a/tests/ui/async-await/issue-70818.stderr b/tests/ui/async-await/issue-70818.stderr index 317c04d2c7478..c68c35eb1586c 100644 --- a/tests/ui/async-await/issue-70818.stderr +++ b/tests/ui/async-await/issue-70818.stderr @@ -5,7 +5,7 @@ LL | fn foo(ty: T, ty1: U) -> impl Future + Send { | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ future created by async block is not `Send` | note: captured value is not `Send` - --> $DIR/issue-70818.rs:6:18 + --> $DIR/issue-70818.rs:7:18 | LL | async { (ty, ty1) } | ^^^ has type `U` which is not `Send` @@ -14,5 +14,24 @@ help: consider restricting type parameter `U` LL | fn foo(ty: T, ty1: U) -> impl Future + Send { | +++++++++++++++++++ -error: aborting due to 1 previous error +error: future cannot be sent between threads safely + --> $DIR/issue-70818.rs:4:38 + | +LL | fn foo(ty: T, ty1: U) -> impl Future + Send { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ future created by async block is not `Send` + | +note: future is not `Send` as this value is used across an await + --> $DIR/issue-70818.rs:7:5 + | +LL | fn foo(ty: T, ty1: U) -> impl Future + Send { + | --- has type `U` which is not `Send` +... +LL | async { (ty, ty1) } + | ^ await occurs here, with `ty1` maybe used later +help: consider restricting type parameter `U` + | +LL | fn foo(ty: T, ty1: U) -> impl Future + Send { + | +++++++++++++++++++ + +error: aborting due to 2 previous errors diff --git a/tests/ui/async-await/issue-74072-lifetime-name-annotations.rs b/tests/ui/async-await/issue-74072-lifetime-name-annotations.rs index 58509642b10bc..97cc0b2ebb88a 100644 --- a/tests/ui/async-await/issue-74072-lifetime-name-annotations.rs +++ b/tests/ui/async-await/issue-74072-lifetime-name-annotations.rs @@ -10,27 +10,27 @@ pub async fn async_fn(x: &mut i32) -> &i32 { y } -pub fn async_closure(x: &mut i32) -> impl Future { +pub fn async_closure(x: &mut i32) -> impl Future { (async move || { //~^ ERROR lifetime may not live long enough //~| ERROR temporary value dropped while borrowed let y = &*x; - *x += 1; //~ ERROR cannot assign to `*x` because it is borrowed + *x += 1; //~ ERROR cannot assign to value because it is borrowed y })() } -pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future { +pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future { (async move || -> &i32 { //~^ ERROR lifetime may not live long enough //~| ERROR temporary value dropped while borrowed let y = &*x; - *x += 1; //~ ERROR cannot assign to `*x` because it is borrowed + *x += 1; //~ ERROR cannot assign to value because it is borrowed y })() } -pub fn async_block(x: &mut i32) -> impl Future { +pub fn async_block(x: &mut i32) -> impl Future { async move { let y = &*x; *x += 1; //~ ERROR cannot assign to `*x` because it is borrowed diff --git a/tests/ui/async-await/issue-74072-lifetime-name-annotations.stderr b/tests/ui/async-await/issue-74072-lifetime-name-annotations.stderr index 9d963686dea0e..df15e380f863e 100644 --- a/tests/ui/async-await/issue-74072-lifetime-name-annotations.stderr +++ b/tests/ui/async-await/issue-74072-lifetime-name-annotations.stderr @@ -10,18 +10,18 @@ LL | *x += 1; LL | y | - returning this value requires that `*x` is borrowed for `'1` -error[E0506]: cannot assign to `*x` because it is borrowed +error[E0506]: cannot assign to value because it is borrowed --> $DIR/issue-74072-lifetime-name-annotations.rs:18:9 | LL | (async move || { | - return type of async closure is &'1 i32 ... LL | let y = &*x; - | --- `*x` is borrowed here + | --- value is borrowed here LL | *x += 1; - | ^^^^^^^ `*x` is assigned to here but it was already borrowed + | ^^^^^^^ value is assigned to here but it was already borrowed LL | y - | - returning this value requires that `*x` is borrowed for `'1` + | - returning this value requires that borrow lasts for `'1` error: lifetime may not live long enough --> $DIR/issue-74072-lifetime-name-annotations.rs:14:20 @@ -44,7 +44,7 @@ LL | | })() error[E0716]: temporary value dropped while borrowed --> $DIR/issue-74072-lifetime-name-annotations.rs:14:5 | -LL | pub fn async_closure(x: &mut i32) -> impl Future { +LL | pub fn async_closure(x: &mut i32) -> impl Future { | - let's call the lifetime of this reference `'1` LL | // (async move || { LL | || @@ -59,18 +59,18 @@ LL | || })() LL | } | - temporary value is freed at the end of this statement -error[E0506]: cannot assign to `*x` because it is borrowed +error[E0506]: cannot assign to value because it is borrowed --> $DIR/issue-74072-lifetime-name-annotations.rs:28:9 | LL | (async move || -> &i32 { | - return type of async closure is &'1 i32 ... LL | let y = &*x; - | --- `*x` is borrowed here + | --- value is borrowed here LL | *x += 1; - | ^^^^^^^ `*x` is assigned to here but it was already borrowed + | ^^^^^^^ value is assigned to here but it was already borrowed LL | y - | - returning this value requires that `*x` is borrowed for `'1` + | - returning this value requires that borrow lasts for `'1` error: lifetime may not live long enough --> $DIR/issue-74072-lifetime-name-annotations.rs:24:28 @@ -93,7 +93,7 @@ LL | | })() error[E0716]: temporary value dropped while borrowed --> $DIR/issue-74072-lifetime-name-annotations.rs:24:5 | -LL | pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future { +LL | pub fn async_closure_explicit_return_type(x: &mut i32) -> impl Future { | - let's call the lifetime of this reference `'1` LL | // (async move || -> &i32 { LL | || diff --git a/tests/ui/async-await/issue-86507.rs b/tests/ui/async-await/issue-86507.rs index 484122a1ddcfd..e959f2c1166f4 100644 --- a/tests/ui/async-await/issue-86507.rs +++ b/tests/ui/async-await/issue-86507.rs @@ -15,6 +15,7 @@ impl Foo for () { -> Pin + Send + 'async_trait>> where 'me:'async_trait { Box::pin( //~ ERROR future cannot be sent between threads safely + //~^ ERROR future cannot be sent between threads safely async move { let x = x; } diff --git a/tests/ui/async-await/issue-86507.stderr b/tests/ui/async-await/issue-86507.stderr index f4cd7c42706c8..0e5ce6c0226bc 100644 --- a/tests/ui/async-await/issue-86507.stderr +++ b/tests/ui/async-await/issue-86507.stderr @@ -2,6 +2,7 @@ error: future cannot be sent between threads safely --> $DIR/issue-86507.rs:17:13 | LL | / Box::pin( +LL | | LL | | async move { LL | | let x = x; LL | | } @@ -9,15 +10,40 @@ LL | | ) | |_____________^ future created by async block is not `Send` | note: captured value is not `Send` because `&` references cannot be sent unless their referent is `Sync` - --> $DIR/issue-86507.rs:19:29 + --> $DIR/issue-86507.rs:20:29 | LL | let x = x; | ^ has type `&T` which is not `Send`, because `T` is not `Sync` - = note: required for the cast from `Pin>` to `Pin + Send + 'async_trait)>>` + = note: required for the cast from `Pin>` to `Pin + Send + 'async_trait)>>` help: consider further restricting this bound | LL | fn bar<'me, 'async_trait, T: Send + std::marker::Sync>(x: &'me T) | +++++++++++++++++++ -error: aborting due to 1 previous error +error: future cannot be sent between threads safely + --> $DIR/issue-86507.rs:17:13 + | +LL | / Box::pin( +LL | | +LL | | async move { +LL | | let x = x; +LL | | } +LL | | ) + | |_____________^ future created by async block is not `Send` + | +note: future is not `Send` as this value is used across an await + --> $DIR/issue-86507.rs:19:17 + | +LL | fn bar<'me, 'async_trait, T: Send>(x: &'me T) + | - has type `&T` which is not `Send` +... +LL | async move { + | ^ await occurs here, with `x` maybe used later + = note: required for the cast from `Pin>` to `Pin + Send>>` +help: consider further restricting this bound + | +LL | fn bar<'me, 'async_trait, T: Send + std::marker::Sync>(x: &'me T) + | +++++++++++++++++++ + +error: aborting due to 2 previous errors diff --git a/tests/ui/coroutine/clone-impl.stderr b/tests/ui/coroutine/clone-impl.stderr index ed933fe784edd..604061d99d95b 100644 --- a/tests/ui/coroutine/clone-impl.stderr +++ b/tests/ui/coroutine/clone-impl.stderr @@ -10,8 +10,11 @@ LL | check_copy(&gen_clone_0); note: captured value does not implement `Copy` --> $DIR/clone-impl.rs:48:14 | -LL | drop(clonable_0); - | ^^^^^^^^^^ has type `Vec` which does not implement `Copy` +LL | let clonable_0: Vec = Vec::new(); + | ---------- has type `Vec` which does not implement `Copy` +LL | let gen_clone_0 = #[coroutine] +LL | move || { + | ^ yield occurs here, with `clonable_0` maybe used later note: required by a bound in `check_copy` --> $DIR/clone-impl.rs:90:18 | @@ -52,8 +55,11 @@ LL | check_copy(&gen_clone_1); note: captured value does not implement `Copy` --> $DIR/clone-impl.rs:69:14 | -LL | drop(clonable_1); - | ^^^^^^^^^^ has type `Vec` which does not implement `Copy` +LL | let clonable_1: Vec = Vec::new(); + | ---------- has type `Vec` which does not implement `Copy` +LL | let gen_clone_1 = #[coroutine] +LL | move || { + | ^ yield occurs here, with `clonable_1` maybe used later note: required by a bound in `check_copy` --> $DIR/clone-impl.rs:90:18 | @@ -95,8 +101,11 @@ LL | check_copy(&gen_non_clone); note: captured value does not implement `Copy` --> $DIR/clone-impl.rs:82:14 | -LL | drop(non_clonable); - | ^^^^^^^^^^^^ has type `NonClone` which does not implement `Copy` +LL | let non_clonable: NonClone = NonClone; + | ------------ has type `NonClone` which does not implement `Copy` +LL | let gen_non_clone = #[coroutine] +LL | move || { + | ^ yield occurs here, with `non_clonable` maybe used later note: required by a bound in `check_copy` --> $DIR/clone-impl.rs:90:18 | @@ -120,8 +129,11 @@ LL | check_clone(&gen_non_clone); note: captured value does not implement `Clone` --> $DIR/clone-impl.rs:82:14 | -LL | drop(non_clonable); - | ^^^^^^^^^^^^ has type `NonClone` which does not implement `Clone` +LL | let non_clonable: NonClone = NonClone; + | ------------ has type `NonClone` which does not implement `Clone` +LL | let gen_non_clone = #[coroutine] +LL | move || { + | ^ yield occurs here, with `non_clonable` maybe used later note: required by a bound in `check_clone` --> $DIR/clone-impl.rs:91:19 | diff --git a/tests/ui/coroutine/ref-escapes-but-not-over-yield.rs b/tests/ui/coroutine/ref-escapes-but-not-over-yield.rs index 0f9c56786da06..def6fc0600729 100644 --- a/tests/ui/coroutine/ref-escapes-but-not-over-yield.rs +++ b/tests/ui/coroutine/ref-escapes-but-not-over-yield.rs @@ -1,16 +1,13 @@ #![feature(coroutines, stmt_expr_attributes)] fn foo(x: &i32) { - // In this case, a reference to `b` escapes the coroutine, but not - // because of a yield. We see that there is no yield in the scope of - // `b` and give the more generic error message. - let mut a = &3; + let a = &mut &3; let mut b = #[coroutine] move || { yield (); let b = 5; - a = &b; - //~^ ERROR borrowed data escapes outside of coroutine + *a = &b; + //~^ ERROR: borrowed data escapes outside of coroutine }; } diff --git a/tests/ui/coroutine/ref-escapes-but-not-over-yield.stderr b/tests/ui/coroutine/ref-escapes-but-not-over-yield.stderr index 6fa7082c0b8b7..4e4577c9ceb50 100644 --- a/tests/ui/coroutine/ref-escapes-but-not-over-yield.stderr +++ b/tests/ui/coroutine/ref-escapes-but-not-over-yield.stderr @@ -1,13 +1,13 @@ error[E0521]: borrowed data escapes outside of coroutine - --> $DIR/ref-escapes-but-not-over-yield.rs:12:9 + --> $DIR/ref-escapes-but-not-over-yield.rs:9:9 | -LL | let mut a = &3; - | ----- `a` declared here, outside of the coroutine body +LL | let a = &mut &3; + | - `a` declared here, outside of the coroutine body ... -LL | a = &b; - | ^^^^-- - | | | - | | borrow is only valid in the coroutine body +LL | *a = &b; + | ^^^^^-- + | | | + | | borrow is only valid in the coroutine body | reference to `b` escapes the coroutine body here error: aborting due to 1 previous error diff --git a/tests/ui/coroutine/ref-upvar-not-send.rs b/tests/ui/coroutine/ref-upvar-not-send.rs index 89bb5e5495f45..ecae6f1258760 100644 --- a/tests/ui/coroutine/ref-upvar-not-send.rs +++ b/tests/ui/coroutine/ref-upvar-not-send.rs @@ -8,24 +8,44 @@ fn assert_send(_: T) {} //~| NOTE required by this bound in `assert_send` //~| NOTE required by a bound in `assert_send` //~| NOTE required by this bound in `assert_send` +//~| NOTE required by a bound in `assert_send` +//~| NOTE required by this bound in `assert_send` +//~| NOTE required by a bound in `assert_send` +//~| NOTE required by this bound in `assert_send` fn main() { let x: &*mut () = &std::ptr::null_mut(); + //~^ NOTE has type `&*mut ()` which is not `Send` let y: &mut *mut () = &mut std::ptr::null_mut(); - assert_send(#[coroutine] move || { + //~^ NOTE has type `&mut *mut ()` which is not `Send` + assert_send( //~^ ERROR coroutine cannot be sent between threads safely //~| NOTE coroutine is not `Send` - yield; - let _x = x; - }); - //~^^ NOTE captured value is not `Send` because `&` references cannot be sent unless their referent is `Sync` - //~| NOTE has type `&*mut ()` which is not `Send`, because `*mut ()` is not `Sync` - assert_send(#[coroutine] move || { + #[coroutine] + move || { + //~^ ERROR coroutine cannot be sent between threads safely + //~| NOTE coroutine is not `Send` + //~| NOTE coroutine is not `Send` + //~| yield occurs here + yield; + let _x = x; + //~^ NOTE captured value is not `Send` + //~| NOTE has type `&*mut ()` which is not `Send`, because `*mut ()` is not `Sync` + }, + ); + assert_send( //~^ ERROR coroutine cannot be sent between threads safely //~| NOTE coroutine is not `Send` - yield; - let _y = y; - }); - //~^^ NOTE captured value is not `Send` because `&mut` references cannot be sent unless their referent is `Send` - //~| NOTE has type `&mut *mut ()` which is not `Send`, because `*mut ()` is not `Send` + #[coroutine] + move || { + //~^ ERROR coroutine cannot be sent between threads safely + //~| NOTE coroutine is not `Send` + //~| NOTE coroutine is not `Send` + //~| yield occurs here + yield; + let _y = y; + //~^ NOTE captured value is not `Send` + //~| NOTE has type `&mut *mut ()` which is not `Send`, because `*mut ()` is not `Send` + }, + ); } diff --git a/tests/ui/coroutine/ref-upvar-not-send.stderr b/tests/ui/coroutine/ref-upvar-not-send.stderr index 892b5d261c2b8..5f671c5bf1408 100644 --- a/tests/ui/coroutine/ref-upvar-not-send.stderr +++ b/tests/ui/coroutine/ref-upvar-not-send.stderr @@ -1,21 +1,21 @@ error: coroutine cannot be sent between threads safely - --> $DIR/ref-upvar-not-send.rs:15:30 + --> $DIR/ref-upvar-not-send.rs:25:9 | -LL | assert_send(#[coroutine] move || { - | ______________________________^ +LL | / move || { LL | | LL | | -LL | | yield; -LL | | let _x = x; -LL | | }); - | |_____^ coroutine is not `Send` +LL | | +... | +LL | | +LL | | }, + | |_________^ coroutine is not `Send` | = help: the trait `Sync` is not implemented for `*mut ()` note: captured value is not `Send` because `&` references cannot be sent unless their referent is `Sync` - --> $DIR/ref-upvar-not-send.rs:19:18 + --> $DIR/ref-upvar-not-send.rs:31:22 | -LL | let _x = x; - | ^ has type `&*mut ()` which is not `Send`, because `*mut ()` is not `Sync` +LL | let _x = x; + | ^ has type `&*mut ()` which is not `Send`, because `*mut ()` is not `Sync` note: required by a bound in `assert_send` --> $DIR/ref-upvar-not-send.rs:6:19 | @@ -23,28 +23,82 @@ LL | fn assert_send(_: T) {} | ^^^^ required by this bound in `assert_send` error: coroutine cannot be sent between threads safely - --> $DIR/ref-upvar-not-send.rs:23:30 + --> $DIR/ref-upvar-not-send.rs:40:9 | -LL | assert_send(#[coroutine] move || { - | ______________________________^ +LL | / move || { LL | | LL | | -LL | | yield; -LL | | let _y = y; -LL | | }); - | |_____^ coroutine is not `Send` +LL | | +... | +LL | | +LL | | }, + | |_________^ coroutine is not `Send` | - = help: within `{coroutine@$DIR/ref-upvar-not-send.rs:23:30: 23:37}`, the trait `Send` is not implemented for `*mut ()` + = help: within `{coroutine@$DIR/ref-upvar-not-send.rs:40:9: 40:16}`, the trait `Send` is not implemented for `*mut ()` note: captured value is not `Send` because `&mut` references cannot be sent unless their referent is `Send` - --> $DIR/ref-upvar-not-send.rs:27:18 + --> $DIR/ref-upvar-not-send.rs:46:22 + | +LL | let _y = y; + | ^ has type `&mut *mut ()` which is not `Send`, because `*mut ()` is not `Send` +note: required by a bound in `assert_send` + --> $DIR/ref-upvar-not-send.rs:6:19 + | +LL | fn assert_send(_: T) {} + | ^^^^ required by this bound in `assert_send` + +error: coroutine cannot be sent between threads safely + --> $DIR/ref-upvar-not-send.rs:21:5 + | +LL | / assert_send( +LL | | +LL | | +LL | | #[coroutine] +... | +LL | | }, +LL | | ); + | |_____^ coroutine is not `Send` + | + = help: the trait `Sync` is not implemented for `*mut ()` +note: coroutine is not `Send` as this value is used across a yield + --> $DIR/ref-upvar-not-send.rs:25:9 + | +LL | let x: &*mut () = &std::ptr::null_mut(); + | - has type `&*mut ()` which is not `Send` +... +LL | move || { + | ^ yield occurs here, with `x` maybe used later +note: required by a bound in `assert_send` + --> $DIR/ref-upvar-not-send.rs:6:19 + | +LL | fn assert_send(_: T) {} + | ^^^^ required by this bound in `assert_send` + +error: coroutine cannot be sent between threads safely + --> $DIR/ref-upvar-not-send.rs:36:5 + | +LL | / assert_send( +LL | | +LL | | +LL | | #[coroutine] +... | +LL | | }, +LL | | ); + | |_____^ coroutine is not `Send` + | + = help: within `{coroutine@$DIR/ref-upvar-not-send.rs:40:9: 40:16}`, the trait `Send` is not implemented for `*mut ()` +note: coroutine is not `Send` as this value is used across a yield + --> $DIR/ref-upvar-not-send.rs:40:9 | -LL | let _y = y; - | ^ has type `&mut *mut ()` which is not `Send`, because `*mut ()` is not `Send` +LL | let y: &mut *mut () = &mut std::ptr::null_mut(); + | - has type `&mut *mut ()` which is not `Send` +... +LL | move || { + | ^ yield occurs here, with `y` maybe used later note: required by a bound in `assert_send` --> $DIR/ref-upvar-not-send.rs:6:19 | LL | fn assert_send(_: T) {} | ^^^^ required by this bound in `assert_send` -error: aborting due to 2 previous errors +error: aborting due to 4 previous errors diff --git a/tests/ui/coroutine/unsized-capture-across-yield.rs b/tests/ui/coroutine/unsized-capture-across-yield.rs index c86b1823aafff..27b0581e8d7b8 100644 --- a/tests/ui/coroutine/unsized-capture-across-yield.rs +++ b/tests/ui/coroutine/unsized-capture-across-yield.rs @@ -7,6 +7,7 @@ use std::ops::Coroutine; fn capture() -> impl Coroutine { let b: [u8] = *(Box::new([]) as Box<[u8]>); + //~^ ERROR the size for values of type `[u8]` cannot be known at compilation time #[coroutine] move || { println!("{:?}", &b); diff --git a/tests/ui/coroutine/unsized-capture-across-yield.stderr b/tests/ui/coroutine/unsized-capture-across-yield.stderr index 03551f1bbff51..6faa9226f9123 100644 --- a/tests/ui/coroutine/unsized-capture-across-yield.stderr +++ b/tests/ui/coroutine/unsized-capture-across-yield.stderr @@ -8,7 +8,7 @@ LL | #![feature(unsized_locals)] = note: `#[warn(incomplete_features)]` on by default error[E0277]: the size for values of type `[u8]` cannot be known at compilation time - --> $DIR/unsized-capture-across-yield.rs:12:27 + --> $DIR/unsized-capture-across-yield.rs:13:27 | LL | move || { | -- this closure captures all values by move @@ -18,6 +18,15 @@ LL | println!("{:?}", &b); = help: the trait `Sized` is not implemented for `[u8]` = note: all values captured by value by a closure must have a statically known size -error: aborting due to 1 previous error; 1 warning emitted +error[E0277]: the size for values of type `[u8]` cannot be known at compilation time + --> $DIR/unsized-capture-across-yield.rs:9:9 + | +LL | let b: [u8] = *(Box::new([]) as Box<[u8]>); + | ^ doesn't have a size known at compile-time + | + = help: the trait `Sized` is not implemented for `[u8]` + = note: all values live across `yield` must have a statically known size + +error: aborting due to 2 previous errors; 1 warning emitted For more information about this error, try `rustc --explain E0277`. diff --git a/tests/ui/feature-gates/feature-gate-coroutine-new-layout.rs b/tests/ui/feature-gates/feature-gate-coroutine-new-layout.rs new file mode 100644 index 0000000000000..58210c7c3957c --- /dev/null +++ b/tests/ui/feature-gates/feature-gate-coroutine-new-layout.rs @@ -0,0 +1,3 @@ +fn main() { + compile_error!("this is an experimental flag that has no obvious user-facing changes"); +} diff --git a/tests/ui/impl-trait/recursive-impl-trait-type-indirect.stderr b/tests/ui/impl-trait/recursive-impl-trait-type-indirect.stderr index 2d2731e4368f5..63d373088eca9 100644 --- a/tests/ui/impl-trait/recursive-impl-trait-type-indirect.stderr +++ b/tests/ui/impl-trait/recursive-impl-trait-type-indirect.stderr @@ -92,6 +92,12 @@ error[E0720]: cannot resolve opaque type | LL | fn coroutine_capture() -> impl Sized { | ^^^^^^^^^^ recursive opaque type +LL | +LL | let x = coroutine_capture(); + | - + | | + | coroutine captures itself here + | coroutine captures itself here ... LL | / move || { LL | | yield; diff --git a/tests/ui/layout/post-mono-layout-cycle-2.stderr b/tests/ui/layout/post-mono-layout-cycle-2.stderr index 2e8d237844e8e..7b0bf4f0140c5 100644 --- a/tests/ui/layout/post-mono-layout-cycle-2.stderr +++ b/tests/ui/layout/post-mono-layout-cycle-2.stderr @@ -6,9 +6,6 @@ LL | | LL | | where LL | | T: IntoIterator, | |___________________________________^ -LL | { -LL | Blah::iter(self, iterator).await - | -------------------------------- recursive call here | = note: a recursive `async fn` call must introduce indirection such as `Box::pin` to avoid an infinitely sized future diff --git a/tests/ui/mir/lint/storage-live.stderr b/tests/ui/mir/lint/storage-live.stderr index 7d4c3f0832a97..a06194a45c86d 100644 --- a/tests/ui/mir/lint/storage-live.stderr +++ b/tests/ui/mir/lint/storage-live.stderr @@ -1,4 +1,4 @@ -error: internal compiler error: broken MIR in Item(DefId(0:8 ~ storage_live[HASH]::multiple_storage)) (after pass CheckPackedRef) at bb0[1]: +error: internal compiler error: broken MIR in Item(DefId(0:8 ~ storage_live[HASH]::multiple_storage)) (after pass CheckPackedRef 001-000) at bb0[1]: StorageLive(_1) which already has storage here --> $DIR/storage-live.rs:23:13 | diff --git a/tests/ui/nll/coroutine-upvar-mutability.stderr b/tests/ui/nll/coroutine-upvar-mutability.stderr index 02c011301761c..6c8382e7a2c8d 100644 --- a/tests/ui/nll/coroutine-upvar-mutability.stderr +++ b/tests/ui/nll/coroutine-upvar-mutability.stderr @@ -1,14 +1,12 @@ -error[E0594]: cannot assign to `x`, as it is not declared as mutable +error[E0384]: cannot assign twice to immutable variable `x` --> $DIR/coroutine-upvar-mutability.rs:10:9 | +LL | let x = 0; + | - first assignment to `x` +... LL | x = 1; - | ^^^^^ cannot assign - | -help: consider changing this to be mutable - | -LL | let mut x = 0; - | +++ + | ^^^^^ cannot assign twice to immutable variable error: aborting due to 1 previous error -For more information about this error, try `rustc --explain E0594`. +For more information about this error, try `rustc --explain E0384`. diff --git a/tests/ui/print_type_sizes/async.stdout b/tests/ui/print_type_sizes/async.stdout index 83a6962e4cd13..690b377629958 100644 --- a/tests/ui/print_type_sizes/async.stdout +++ b/tests/ui/print_type_sizes/async.stdout @@ -1,15 +1,13 @@ -print-type-size type: `{async fn body of test()}`: 16386 bytes, alignment: 1 bytes +print-type-size type: `{async fn body of test()}`: 8194 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 8192 bytes -print-type-size upvar `.arg`: 8192 bytes -print-type-size variant `Suspend0`: 16385 bytes -print-type-size upvar `.arg`: 8192 bytes -print-type-size local `.arg`: 8192 bytes +print-type-size upvar `.arg`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `..coroutine_field2`: 8192 bytes, type: [u8; 8192] +print-type-size variant `Suspend0`: 8193 bytes print-type-size local `.__awaitee`: 1 bytes, type: {async fn body of wait()} -print-type-size variant `Returned`: 8192 bytes -print-type-size upvar `.arg`: 8192 bytes -print-type-size variant `Panicked`: 8192 bytes -print-type-size upvar `.arg`: 8192 bytes +print-type-size local `.arg`: 8192 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes print-type-size type: `std::mem::ManuallyDrop<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes print-type-size field `.value`: 8192 bytes print-type-size type: `std::mem::MaybeUninit<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes diff --git a/tests/ui/print_type_sizes/coroutine.stdout b/tests/ui/print_type_sizes/coroutine.stdout index 339bbddfc2a93..6abee21d415b9 100644 --- a/tests/ui/print_type_sizes/coroutine.stdout +++ b/tests/ui/print_type_sizes/coroutine.stdout @@ -1,10 +1,15 @@ print-type-size type: `{coroutine@$DIR/coroutine.rs:11:5: 11:14}`: 8193 bytes, alignment: 1 bytes print-type-size discriminant: 1 bytes print-type-size variant `Unresumed`: 8192 bytes -print-type-size upvar `.array`: 8192 bytes +print-type-size upvar `.array`: 1 bytes, offset: 0 bytes, alignment: 1 bytes +print-type-size local `.array`: 8192 bytes print-type-size variant `Suspend0`: 8192 bytes -print-type-size upvar `.array`: 8192 bytes -print-type-size variant `Returned`: 8192 bytes -print-type-size upvar `.array`: 8192 bytes -print-type-size variant `Panicked`: 8192 bytes -print-type-size upvar `.array`: 8192 bytes +print-type-size local `.array`: 8192 bytes +print-type-size variant `Returned`: 0 bytes +print-type-size variant `Panicked`: 0 bytes +print-type-size type: `std::mem::ManuallyDrop<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size field `.value`: 8192 bytes +print-type-size type: `std::mem::MaybeUninit<[u8; 8192]>`: 8192 bytes, alignment: 1 bytes +print-type-size variant `MaybeUninit`: 8192 bytes +print-type-size field `.uninit`: 0 bytes +print-type-size field `.value`: 8192 bytes diff --git a/tests/ui/type-alias-impl-trait/hkl_forbidden4.stderr b/tests/ui/type-alias-impl-trait/hkl_forbidden4.stderr index 0c2772683a918..154f2607c597a 100644 --- a/tests/ui/type-alias-impl-trait/hkl_forbidden4.stderr +++ b/tests/ui/type-alias-impl-trait/hkl_forbidden4.stderr @@ -36,13 +36,13 @@ LL | type FutNothing<'a> = impl 'a + Future; = note: `FutNothing` must be used in combination with a concrete type within the same module error[E0792]: expected generic lifetime parameter, found `'any` - --> $DIR/hkl_forbidden4.rs:15:5 + --> $DIR/hkl_forbidden4.rs:15:21 | LL | async fn operation(_: &mut ()) -> () { | - this generic parameter must be used with a generic lifetime parameter LL | LL | call(operation).await - | ^^^^^^^^^^^^^^^ + | ^^^^^ error: concrete type differs from previous defining opaque type use --> $DIR/hkl_forbidden4.rs:13:1 diff --git a/tests/ui/type-alias-impl-trait/indirect-recursion-issue-112047.stderr b/tests/ui/type-alias-impl-trait/indirect-recursion-issue-112047.stderr index 6cbffaaed4d35..a125d8b65c4d0 100644 --- a/tests/ui/type-alias-impl-trait/indirect-recursion-issue-112047.stderr +++ b/tests/ui/type-alias-impl-trait/indirect-recursion-issue-112047.stderr @@ -2,15 +2,8 @@ error[E0733]: recursion in an async block requires boxing --> $DIR/indirect-recursion-issue-112047.rs:22:9 | LL | async move { recur(self).await; } - | ^^^^^^^^^^ ----------------- recursive call here + | ^^^^^^^^^^ | -note: which leads to this async fn - --> $DIR/indirect-recursion-issue-112047.rs:14:1 - | -LL | async fn recur(t: impl Recur) { - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -LL | t.recur().await; - | --------------- ...leading to this recursive call = note: a recursive `async fn` call must introduce indirection such as `Box::pin` to avoid an infinitely sized future error: aborting due to 1 previous error