Create definitions for promoted constants.

compiler/rustc_borrowck/src/consumers.rs

@@ -3,7 +3,6 @@
 //! This file provides API for compiler consumers.
 
 use rustc_hir::def_id::LocalDefId;
-use rustc_index::IndexSlice;
 use rustc_infer::infer::{DefiningAnchor, TyCtxtInferExt};
 use rustc_middle::mir::Body;
 use rustc_middle::ty::TyCtxt;
@@ -29,9 +28,8 @@ pub use super::{
 ///
 /// *   Polonius is highly unstable, so expect regular changes in its signature or other details.
 pub fn get_body_with_borrowck_facts(tcx: TyCtxt<'_>, def: LocalDefId) -> BodyWithBorrowckFacts<'_> {
-    let (input_body, promoted) = tcx.mir_promoted(def);
+    let (input_body, _) = tcx.mir_promoted(def);
     let infcx = tcx.infer_ctxt().with_opaque_type_inference(DefiningAnchor::Bind(def)).build();
     let input_body: &Body<'_> = &input_body.borrow();
-    let promoted: &IndexSlice<_, _> = &promoted.borrow();
-    *super::do_mir_borrowck(&infcx, input_body, promoted, true).1.unwrap()
+    *super::do_mir_borrowck(&infcx, input_body, true).1.unwrap()
 }

compiler/rustc_borrowck/src/dataflow.rs

@@ -125,15 +125,9 @@ pub struct Borrows<'a, 'tcx> {
     borrows_out_of_scope_at_location: FxIndexMap<Location, Vec<BorrowIndex>>,
 }
 
-struct StackEntry {
-    bb: mir::BasicBlock,
-    lo: usize,
-    hi: usize,
-}
-
 struct OutOfScopePrecomputer<'a, 'tcx> {
     visited: BitSet<mir::BasicBlock>,
-    visit_stack: Vec<StackEntry>,
+    visit_stack: Vec<mir::BasicBlock>,
     body: &'a Body<'tcx>,
     regioncx: &'a RegionInferenceContext<'tcx>,
     borrows_out_of_scope_at_location: FxIndexMap<Location, Vec<BorrowIndex>>,
@@ -158,68 +152,66 @@ impl<'tcx> OutOfScopePrecomputer<'_, 'tcx> {
         borrow_region: RegionVid,
         first_location: Location,
     ) {
-        // We visit one BB at a time. The complication is that we may start in the
-        // middle of the first BB visited (the one containing `first_location`), in which
-        // case we may have to later on process the first part of that BB if there
-        // is a path back to its start.
-
-        // For visited BBs, we record the index of the first statement processed.
-        // (In fully processed BBs this index is 0.) Note also that we add BBs to
-        // `visited` once they are added to `stack`, before they are actually
-        // processed, because this avoids the need to look them up again on
-        // completion.
-        self.visited.insert(first_location.block);
-
         let first_block = first_location.block;
-        let mut first_lo = first_location.statement_index;
-        let first_hi = self.body[first_block].statements.len();
+        let first_bb_data = &self.body.basic_blocks[first_block];
+
+        // This is the first block, we only want to visit it from the creation of the borrow at
+        // `first_location`.
+        let first_lo = first_location.statement_index;
+        let first_hi = first_bb_data.statements.len();
+
+        if let Some(kill_stmt) = self.regioncx.first_non_contained_inclusive(
+            borrow_region,
+            first_block,
+            first_lo,
+            first_hi,
+        ) {
+            let kill_location = Location { block: first_block, statement_index: kill_stmt };
+            // If region does not contain a point at the location, then add to list and skip
+            // successor locations.
+            debug!("borrow {:?} gets killed at {:?}", borrow_index, kill_location);
+            self.borrows_out_of_scope_at_location
+                .entry(kill_location)
+                .or_default()
+                .push(borrow_index);
+
+            // The borrow is already dead, there is no need to visit other blocks.
+            return;
+        }
 
-        self.visit_stack.push(StackEntry { bb: first_block, lo: first_lo, hi: first_hi });
+        // The borrow is not dead. Add successor BBs to the work list, if necessary.
+        for succ_bb in first_bb_data.terminator().successors() {
+            if self.visited.insert(succ_bb) {
+                self.visit_stack.push(succ_bb);
+            }
+        }
 
-        'preorder: while let Some(StackEntry { bb, lo, hi }) = self.visit_stack.pop() {
+        // We may end up visiting `first_block` again. This is not an issue: we know at this point
+        // that it does not kill the borrow in the `first_lo..=first_hi` range, so checking the
+        // `0..first_lo` range and the `0..first_hi` range give the same result.
+        while let Some(block) = self.visit_stack.pop() {
+            let bb_data = &self.body[block];
+            let num_stmts = bb_data.statements.len();
             if let Some(kill_stmt) =
-                self.regioncx.first_non_contained_inclusive(borrow_region, bb, lo, hi)
+                self.regioncx.first_non_contained_inclusive(borrow_region, block, 0, num_stmts)
             {
-                let kill_location = Location { block: bb, statement_index: kill_stmt };
+                let kill_location = Location { block, statement_index: kill_stmt };
                 // If region does not contain a point at the location, then add to list and skip
                 // successor locations.
                 debug!("borrow {:?} gets killed at {:?}", borrow_index, kill_location);
                 self.borrows_out_of_scope_at_location
                     .entry(kill_location)
                     .or_default()
                     .push(borrow_index);
-                continue 'preorder;
-            }
 
-            // If we process the first part of the first basic block (i.e. we encounter that block
-            // for the second time), we no longer have to visit its successors again.
-            if bb == first_block && hi != first_hi {
+                // We killed the borrow, so we do not visit this block's successors.
                 continue;
             }
 
             // Add successor BBs to the work list, if necessary.
-            let bb_data = &self.body[bb];
-            debug_assert!(hi == bb_data.statements.len());
             for succ_bb in bb_data.terminator().successors() {
-                if !self.visited.insert(succ_bb) {
-                    if succ_bb == first_block && first_lo > 0 {
-                        // `succ_bb` has been seen before. If it wasn't
-                        // fully processed, add its first part to `stack`
-                        // for processing.
-                        self.visit_stack.push(StackEntry { bb: succ_bb, lo: 0, hi: first_lo - 1 });
-
-                        // And update this entry with 0, to represent the
-                        // whole BB being processed.
-                        first_lo = 0;
-                    }
-                } else {
-                    // succ_bb hasn't been seen before. Add it to
-                    // `stack` for processing.
-                    self.visit_stack.push(StackEntry {
-                        bb: succ_bb,
-                        lo: 0,
-                        hi: self.body[succ_bb].statements.len(),
-                    });
+                if self.visited.insert(succ_bb) {
+                    self.visit_stack.push(succ_bb);
                 }
             }
         }

compiler/rustc_borrowck/src/invalidation.rs

@@ -351,15 +351,14 @@ impl<'cx, 'tcx> InvalidationGenerator<'cx, 'tcx> {
         let tcx = self.tcx;
         let body = self.body;
         let borrow_set = self.borrow_set;
-        let indices = self.borrow_set.indices();
         each_borrow_involving_path(
             self,
             tcx,
             body,
             location,
             (sd, place),
             borrow_set,
-            indices,
+            |_| true,
             |this, borrow_index, borrow| {
                 match (rw, borrow.kind) {
                     // Obviously an activation is compatible with its own

compiler/rustc_borrowck/src/lib.rs

@@ -23,25 +23,18 @@ use rustc_errors::{Diagnostic, DiagnosticBuilder, DiagnosticMessage, Subdiagnost
 use rustc_fluent_macro::fluent_messages;
 use rustc_hir as hir;
 use rustc_hir::def_id::LocalDefId;
-use rustc_index::bit_set::ChunkedBitSet;
-use rustc_index::{IndexSlice, IndexVec};
+use rustc_index::bit_set::{BitSet, ChunkedBitSet};
+use rustc_index::IndexVec;
 use rustc_infer::infer::{
     DefiningAnchor, InferCtxt, NllRegionVariableOrigin, RegionVariableOrigin, TyCtxtInferExt,
 };
-use rustc_middle::mir::{
-    traversal, Body, ClearCrossCrate, Local, Location, Mutability, NonDivergingIntrinsic, Operand,
-    Place, PlaceElem, PlaceRef, VarDebugInfoContents,
-};
-use rustc_middle::mir::{AggregateKind, BasicBlock, BorrowCheckResult, BorrowKind};
-use rustc_middle::mir::{InlineAsmOperand, Terminator, TerminatorKind};
-use rustc_middle::mir::{ProjectionElem, Promoted, Rvalue, Statement, StatementKind};
+use rustc_middle::mir::*;
 use rustc_middle::query::Providers;
 use rustc_middle::ty::{self, CapturedPlace, ParamEnv, RegionVid, TyCtxt};
 use rustc_session::lint::builtin::UNUSED_MUT;
 use rustc_span::{Span, Symbol};
 use rustc_target::abi::FieldIdx;
 
-use either::Either;
 use smallvec::SmallVec;
 use std::cell::OnceCell;
 use std::cell::RefCell;
@@ -123,8 +116,8 @@ pub fn provide(providers: &mut Providers) {
 }
 
 fn mir_borrowck(tcx: TyCtxt<'_>, def: LocalDefId) -> &BorrowCheckResult<'_> {
-    let (input_body, promoted) = tcx.mir_promoted(def);
-    debug!("run query mir_borrowck: {}", tcx.def_path_str(def));
+    let (input_body, _) = tcx.mir_promoted(def);
+    debug!("run query mir_borrowck: {def:?}");
 
     if input_body.borrow().should_skip() {
         debug!("Skipping borrowck because of injected body");
@@ -138,14 +131,12 @@ fn mir_borrowck(tcx: TyCtxt<'_>, def: LocalDefId) -> &BorrowCheckResult<'_> {
         return tcx.arena.alloc(result);
     }
 
-    let hir_owner = tcx.hir().local_def_id_to_hir_id(def).owner;
-
+    let typeck_root = tcx.typeck_root_def_id(def.to_def_id()).expect_local();
     let infcx =
-        tcx.infer_ctxt().with_opaque_type_inference(DefiningAnchor::Bind(hir_owner.def_id)).build();
+        tcx.infer_ctxt().with_opaque_type_inference(DefiningAnchor::Bind(typeck_root)).build();
     let input_body: &Body<'_> = &input_body.borrow();
-    let promoted: &IndexSlice<_, _> = &promoted.borrow();
-    let opt_closure_req = do_mir_borrowck(&infcx, input_body, promoted, false).0;
-    debug!("mir_borrowck done");
+    let opt_closure_req = do_mir_borrowck(&infcx, input_body, false).0;
+    debug!("mir_borrowck done: {def:?}");
 
     tcx.arena.alloc(opt_closure_req)
 }
@@ -155,11 +146,10 @@ fn mir_borrowck(tcx: TyCtxt<'_>, def: LocalDefId) -> &BorrowCheckResult<'_> {
 /// If `return_body_with_facts` is true, then return the body with non-erased
 /// region ids on which the borrow checking was performed together with Polonius
 /// facts.
-#[instrument(skip(infcx, input_body, input_promoted), fields(id=?input_body.source.def_id()), level = "debug")]
+#[instrument(skip(infcx, input_body), fields(id=?input_body.source.def_id()), level = "debug")]
 fn do_mir_borrowck<'tcx>(
     infcx: &InferCtxt<'tcx>,
     input_body: &Body<'tcx>,
-    input_promoted: &IndexSlice<Promoted, Body<'tcx>>,
     return_body_with_facts: bool,
 ) -> (BorrowCheckResult<'tcx>, Option<Box<BodyWithBorrowckFacts<'tcx>>>) {
     let def = input_body.source.def_id().expect_local();
@@ -212,9 +202,7 @@ fn do_mir_borrowck<'tcx>(
     // be modified (in place) to contain non-lexical lifetimes. It
     // will have a lifetime tied to the inference context.
     let mut body_owned = input_body.clone();
-    let mut promoted = input_promoted.to_owned();
-    let free_regions =
-        nll::replace_regions_in_mir(&infcx, param_env, &mut body_owned, &mut promoted);
+    let free_regions = nll::replace_regions_in_mir(&infcx, param_env, &mut body_owned);
     let body = &body_owned; // no further changes
 
     let location_table_owned = LocationTable::new(body);
@@ -225,9 +213,6 @@ fn do_mir_borrowck<'tcx>(
             Ok((_, move_data)) => (move_data, Vec::new()),
             Err((move_data, move_errors)) => (move_data, move_errors),
         };
-    let promoted_errors = promoted
-        .iter_enumerated()
-        .map(|(idx, body)| (idx, MoveData::gather_moves(&body, tcx, param_env)));
 
     let mdpe = MoveDataParamEnv { move_data, param_env };
 
@@ -255,7 +240,6 @@ fn do_mir_borrowck<'tcx>(
         &infcx,
         free_regions,
         body,
-        &promoted,
         location_table,
         param_env,
         &mut flow_inits,
@@ -311,39 +295,6 @@ fn do_mir_borrowck<'tcx>(
         true
     };
 
-    for (idx, move_data_results) in promoted_errors {
-        let promoted_body = &promoted[idx];
-
-        if let Err((move_data, move_errors)) = move_data_results {
-            let mut promoted_mbcx = MirBorrowckCtxt {
-                infcx: &infcx,
-                param_env,
-                body: promoted_body,
-                move_data: &move_data,
-                location_table, // no need to create a real one for the promoted, it is not used
-                movable_generator,
-                fn_self_span_reported: Default::default(),
-                locals_are_invalidated_at_exit,
-                access_place_error_reported: Default::default(),
-                reservation_error_reported: Default::default(),
-                uninitialized_error_reported: Default::default(),
-                regioncx: regioncx.clone(),
-                used_mut: Default::default(),
-                used_mut_upvars: SmallVec::new(),
-                borrow_set: Rc::clone(&borrow_set),
-                dominators: Default::default(),
-                upvars: Vec::new(),
-                local_names: IndexVec::from_elem(None, &promoted_body.local_decls),
-                region_names: RefCell::default(),
-                next_region_name: RefCell::new(1),
-                polonius_output: None,
-                errors,
-            };
-            promoted_mbcx.report_move_errors(move_errors);
-            errors = promoted_mbcx.errors;
-        };
-    }
-
     let mut mbcx = MirBorrowckCtxt {
         infcx: &infcx,
         param_env,
@@ -737,7 +688,6 @@ impl<'cx, 'tcx> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtx
             }
             TerminatorKind::Assert { cond, expected: _, msg, target: _, unwind: _ } => {
                 self.consume_operand(loc, (cond, span), flow_state);
-                use rustc_middle::mir::AssertKind;
                 if let AssertKind::BoundsCheck { len, index } = &**msg {
                     self.consume_operand(loc, (len, span), flow_state);
                     self.consume_operand(loc, (index, span), flow_state);
@@ -1049,12 +999,16 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         let borrow_set = self.borrow_set.clone();
 
         // Use polonius output if it has been enabled.
-        let polonius_output = self.polonius_output.clone();
-        let borrows_in_scope = if let Some(polonius) = &polonius_output {
+        let mut polonius_output;
+        let borrows_in_scope = if let Some(polonius) = &self.polonius_output {
             let location = self.location_table.start_index(location);
-            Either::Left(polonius.errors_at(location).iter().copied())
+            polonius_output = BitSet::new_empty(borrow_set.len());
+            for &idx in polonius.errors_at(location) {
+                polonius_output.insert(idx);
+            }
+            &polonius_output
         } else {
-            Either::Right(flow_state.borrows.iter())
+            &flow_state.borrows
         };
 
         each_borrow_involving_path(
@@ -1064,7 +1018,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
             location,
             (sd, place_span.0),
             &borrow_set,
-            borrows_in_scope,
+            |borrow_index| borrows_in_scope.contains(borrow_index),
             |this, borrow_index, borrow| match (rw, borrow.kind) {
                 // Obviously an activation is compatible with its own
                 // reservation (or even prior activating uses of same

compiler/rustc_borrowck/src/nll.rs

@@ -4,11 +4,9 @@
 
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_hir::def_id::LocalDefId;
-use rustc_index::IndexSlice;
 use rustc_middle::mir::{create_dump_file, dump_enabled, dump_mir, PassWhere};
 use rustc_middle::mir::{
-    Body, ClosureOutlivesSubject, ClosureRegionRequirements, LocalKind, Location, Promoted,
-    START_BLOCK,
+    Body, ClosureOutlivesSubject, ClosureRegionRequirements, LocalKind, Location, START_BLOCK,
 };
 use rustc_middle::ty::{self, OpaqueHiddenType, TyCtxt};
 use rustc_span::symbol::sym;
@@ -54,12 +52,11 @@ pub(crate) struct NllOutput<'tcx> {
 /// Rewrites the regions in the MIR to use NLL variables, also scraping out the set of universal
 /// regions (e.g., region parameters) declared on the function. That set will need to be given to
 /// `compute_regions`.
-#[instrument(skip(infcx, param_env, body, promoted), level = "debug")]
+#[instrument(skip(infcx, param_env, body), level = "debug")]
 pub(crate) fn replace_regions_in_mir<'tcx>(
     infcx: &BorrowckInferCtxt<'_, 'tcx>,
     param_env: ty::ParamEnv<'tcx>,
     body: &mut Body<'tcx>,
-    promoted: &mut IndexSlice<Promoted, Body<'tcx>>,
 ) -> UniversalRegions<'tcx> {
     let def = body.source.def_id().expect_local();
 
@@ -69,7 +66,7 @@ pub(crate) fn replace_regions_in_mir<'tcx>(
     let universal_regions = UniversalRegions::new(infcx, def, param_env);
 
     // Replace all remaining regions with fresh inference variables.
-    renumber::renumber_mir(infcx, body, promoted);
+    renumber::renumber_mir(infcx, body);
 
     dump_mir(infcx.tcx, false, "renumber", &0, body, |_, _| Ok(()));
 
@@ -158,7 +155,6 @@ pub(crate) fn compute_regions<'cx, 'tcx>(
     infcx: &BorrowckInferCtxt<'_, 'tcx>,
     universal_regions: UniversalRegions<'tcx>,
     body: &Body<'tcx>,
-    promoted: &IndexSlice<Promoted, Body<'tcx>>,
     location_table: &LocationTable,
     param_env: ty::ParamEnv<'tcx>,
     flow_inits: &mut ResultsCursor<'cx, 'tcx, MaybeInitializedPlaces<'cx, 'tcx>>,
@@ -180,7 +176,6 @@ pub(crate) fn compute_regions<'cx, 'tcx>(
             infcx,
             param_env,
             body,
-            promoted,
             &universal_regions,
             location_table,
             borrow_set,