Auto merge of #156046 - Amanieu:move-elimination, r=<try>

[WIP] MIR move elimination pass

Author: bors

compiler/rustc_index/src/interval.rs

@@ -1,6 +1,7 @@
 use std::iter::Step;
 use std::marker::PhantomData;
-use std::ops::{Bound, Range, RangeBounds};
+use std::ops::{Bound, RangeBounds};
+use std::range::RangeInclusive;
 
 use smallvec::SmallVec;
 
@@ -59,11 +60,14 @@ impl<I: Idx> IntervalSet<I> {
     }
 
     /// Iterates through intervals stored in the set, in order.
-    pub fn iter_intervals(&self) -> impl Iterator<Item = std::ops::Range<I>>
+    pub fn iter_intervals(&self) -> impl Iterator<Item = RangeInclusive<I>>
     where
         I: Step,
     {
-        self.map.iter().map(|&(start, end)| I::new(start as usize)..I::new(end as usize + 1))
+        self.map.iter().map(|&(start, end)| RangeInclusive {
+            start: I::new(start as usize),
+            last: I::new(end as usize),
+        })
     }
 
     /// Returns true if we increased the number of elements present.
@@ -164,6 +168,35 @@ impl<I: Idx> IntervalSet<I> {
         );
     }
 
+    /// Specialized version of `insert` when we know that the inserted range is
+    /// *after* any contained.
+    pub fn append_range(&mut self, range: impl RangeBounds<I> + Clone) {
+        let start = inclusive_start(range.clone());
+        let Some(end) = inclusive_end(self.domain, range) else {
+            // empty range
+            return;
+        };
+        if start > end {
+            return;
+        }
+
+        if let Some((_, last_end)) = self.map.last_mut() {
+            assert!(*last_end < start);
+            if start == *last_end + 1 {
+                *last_end = end;
+            } else {
+                self.map.push((start, end));
+            }
+        } else {
+            self.map.push((start, end));
+        }
+
+        debug_assert!(
+            self.check_invariants(),
+            "wrong intervals after append {start:?}..={end:?} to {self:?}"
+        );
+    }
+
     pub fn contains(&self, needle: I) -> bool {
         let needle = needle.index() as u32;
         let Some(last) = self.map.partition_point(|r| r.0 <= needle).checked_sub(1) else {
@@ -180,11 +213,12 @@ impl<I: Idx> IntervalSet<I> {
     {
         let mut sup_iter = self.iter_intervals();
         let mut current = None;
-        let contains = |sup: Range<I>, sub: Range<I>, current: &mut Option<Range<I>>| {
-            if sup.end < sub.start {
-                // if `sup.end == sub.start`, the next sup doesn't contain `sub.start`
+        let contains = |sup: RangeInclusive<I>,
+                        sub: RangeInclusive<I>,
+                        current: &mut Option<RangeInclusive<I>>| {
+            if sup.last < sub.start {
                 None // continue to the next sup
-            } else if sup.end >= sub.end && sup.start <= sub.start {
+            } else if sup.last >= sub.last && sup.start <= sub.start {
                 *current = Some(sup); // save the current sup
                 Some(true)
             } else {
@@ -194,8 +228,8 @@ impl<I: Idx> IntervalSet<I> {
         other.iter_intervals().all(|sub| {
             current
                 .take()
-                .and_then(|sup| contains(sup, sub.clone(), &mut current))
-                .or_else(|| sup_iter.find_map(|sup| contains(sup, sub.clone(), &mut current)))
+                .and_then(|sup| contains(sup, sub, &mut current))
+                .or_else(|| sup_iter.find_map(|sup| contains(sup, sub, &mut current)))
                 .unwrap_or(false)
         })
     }
@@ -212,11 +246,11 @@ impl<I: Idx> IntervalSet<I> {
             let mut other_current = other_iter.next()?;
 
             loop {
-                if self_current.end <= other_current.start {
+                if self_current.last < other_current.start {
                     self_current = self_iter.next()?;
                     continue;
                 }
-                if other_current.end <= self_current.start {
+                if other_current.last < self_current.start {
                     other_current = other_iter.next()?;
                     continue;
                 }
@@ -340,6 +374,12 @@ impl<R: Idx, C: Step + Idx> SparseIntervalMatrix<R, C> {
         self.rows.get(row)
     }
 
+    pub fn clear_row(&mut self, row: R) {
+        if let Some(row) = self.rows.get_mut(row) {
+            row.clear();
+        }
+    }
+
     fn ensure_row(&mut self, row: R) -> &mut IntervalSet<C> {
         self.rows.ensure_contains_elem(row, || IntervalSet::new(self.column_size))
     }
@@ -363,6 +403,16 @@ impl<R: Idx, C: Step + Idx> SparseIntervalMatrix<R, C> {
         write_row.union(read_row)
     }
 
+    pub fn disjoint_rows(&self, a: R, b: R) -> bool
+    where
+        C: Step,
+    {
+        let (Some(a), Some(b)) = (self.rows.get(a), self.rows.get(b)) else {
+            return true;
+        };
+        a.disjoint(b)
+    }
+
     pub fn insert_all_into_row(&mut self, row: R) {
         self.ensure_row(row).insert_all();
     }
@@ -379,6 +429,10 @@ impl<R: Idx, C: Step + Idx> SparseIntervalMatrix<R, C> {
         self.ensure_row(row).append(point)
     }
 
+    pub fn append_range(&mut self, row: R, range: impl RangeBounds<C> + Clone) {
+        self.ensure_row(row).append_range(range)
+    }
+
     pub fn contains(&self, row: R, point: C) -> bool {
         self.row(row).is_some_and(|r| r.contains(point))
     }

compiler/rustc_index/src/interval/tests.rs

@@ -5,7 +5,7 @@ fn insert_collapses() {
     let mut set = IntervalSet::<u32>::new(10000);
     set.insert_range(9831..=9837);
     set.insert_range(43..=9830);
-    assert_eq!(set.iter_intervals().collect::<Vec<_>>(), [43..9838]);
+    assert_eq!(set.iter_intervals().collect::<Vec<_>>(), [(43..=9837).into()]);
 }
 
 #[test]

compiler/rustc_middle/src/mir/syntax.rs

@@ -128,7 +128,6 @@ pub enum RuntimePhase {
     /// disallowed:
     /// * [`TerminatorKind::Yield`]
     /// * [`TerminatorKind::CoroutineDrop`]
-    /// * [`Rvalue::Aggregate`] for any `AggregateKind` except `Array`
     /// * [`Rvalue::CopyForDeref`]
     /// * [`PlaceElem::OpaqueCast`]
     /// * [`LocalInfo::DerefTemp`](super::LocalInfo::DerefTemp)
@@ -1452,9 +1451,6 @@ pub enum Rvalue<'tcx> {
     /// This is needed because dataflow analysis needs to distinguish
     /// `dest = Foo { x: ..., y: ... }` from `dest.x = ...; dest.y = ...;` in the case that `Foo`
     /// has a destructor.
-    ///
-    /// Disallowed after deaggregation for all aggregate kinds except `Array` and `Coroutine`. After
-    /// coroutine lowering, `Coroutine` aggregate kinds are disallowed too.
     Aggregate(Box<AggregateKind<'tcx>>, IndexVec<FieldIdx, Operand<'tcx>>),
 
     /// A CopyForDeref is equivalent to a read from a place at the

compiler/rustc_mir_dataflow/src/framework/direction.rs

@@ -7,6 +7,7 @@ use rustc_middle::mir::{
 
 use super::visitor::ResultsVisitor;
 use super::{Analysis, Effect, EffectIndex};
+use crate::Results;
 
 pub trait Direction {
     const IS_FORWARD: bool;
@@ -40,7 +41,7 @@ pub trait Direction {
     /// all locations in a basic block (starting from `entry_state` and to
     /// visit them with `vis`.
     fn visit_results_in_block<'mir, 'tcx, A>(
-        analysis: &A,
+        results: &Results<'tcx, A>,
         state: &mut A::Domain,
         block: BasicBlock,
         block_data: &'mir mir::BasicBlockData<'tcx>,
@@ -206,32 +207,32 @@ impl Direction for Backward {
     }
 
     fn visit_results_in_block<'mir, 'tcx, A>(
-        analysis: &A,
+        results: &Results<'tcx, A>,
         state: &mut A::Domain,
         block: BasicBlock,
         block_data: &'mir mir::BasicBlockData<'tcx>,
         vis: &mut impl ResultsVisitor<'tcx, A>,
     ) where
         A: Analysis<'tcx>,
     {
-        vis.visit_block_end(state);
+        vis.visit_block_end(results, state, block);
 
         let loc = Location { block, statement_index: block_data.statements.len() };
         let term = block_data.terminator();
-        analysis.apply_early_terminator_effect(state, term, loc);
-        vis.visit_after_early_terminator_effect(analysis, state, term, loc);
-        analysis.apply_primary_terminator_effect(state, term, loc);
-        vis.visit_after_primary_terminator_effect(analysis, state, term, loc);
+        results.analysis.apply_early_terminator_effect(state, term, loc);
+        vis.visit_after_early_terminator_effect(&results.analysis, state, term, loc);
+        results.analysis.apply_primary_terminator_effect(state, term, loc);
+        vis.visit_after_primary_terminator_effect(&results.analysis, state, term, loc);
 
         for (statement_index, stmt) in block_data.statements.iter().enumerate().rev() {
             let loc = Location { block, statement_index };
-            analysis.apply_early_statement_effect(state, stmt, loc);
-            vis.visit_after_early_statement_effect(analysis, state, stmt, loc);
-            analysis.apply_primary_statement_effect(state, stmt, loc);
-            vis.visit_after_primary_statement_effect(analysis, state, stmt, loc);
+            results.analysis.apply_early_statement_effect(state, stmt, loc);
+            vis.visit_after_early_statement_effect(&results.analysis, state, stmt, loc);
+            results.analysis.apply_primary_statement_effect(state, stmt, loc);
+            vis.visit_after_primary_statement_effect(&results.analysis, state, stmt, loc);
         }
 
-        vis.visit_block_start(state);
+        vis.visit_block_start(results, state, block);
     }
 }
 
@@ -386,31 +387,31 @@ impl Direction for Forward {
     }
 
     fn visit_results_in_block<'mir, 'tcx, A>(
-        analysis: &A,
+        results: &Results<'tcx, A>,
         state: &mut A::Domain,
         block: BasicBlock,
         block_data: &'mir mir::BasicBlockData<'tcx>,
         vis: &mut impl ResultsVisitor<'tcx, A>,
     ) where
         A: Analysis<'tcx>,
     {
-        vis.visit_block_start(state);
+        vis.visit_block_start(results, state, block);
 
         for (statement_index, stmt) in block_data.statements.iter().enumerate() {
             let loc = Location { block, statement_index };
-            analysis.apply_early_statement_effect(state, stmt, loc);
-            vis.visit_after_early_statement_effect(analysis, state, stmt, loc);
-            analysis.apply_primary_statement_effect(state, stmt, loc);
-            vis.visit_after_primary_statement_effect(analysis, state, stmt, loc);
+            results.analysis.apply_early_statement_effect(state, stmt, loc);
+            vis.visit_after_early_statement_effect(&results.analysis, state, stmt, loc);
+            results.analysis.apply_primary_statement_effect(state, stmt, loc);
+            vis.visit_after_primary_statement_effect(&results.analysis, state, stmt, loc);
         }
 
         let loc = Location { block, statement_index: block_data.statements.len() };
         let term = block_data.terminator();
-        analysis.apply_early_terminator_effect(state, term, loc);
-        vis.visit_after_early_terminator_effect(analysis, state, term, loc);
-        analysis.apply_primary_terminator_effect(state, term, loc);
-        vis.visit_after_primary_terminator_effect(analysis, state, term, loc);
+        results.analysis.apply_early_terminator_effect(state, term, loc);
+        vis.visit_after_early_terminator_effect(&results.analysis, state, term, loc);
+        results.analysis.apply_primary_terminator_effect(state, term, loc);
+        vis.visit_after_primary_terminator_effect(&results.analysis, state, term, loc);
 
-        vis.visit_block_end(state);
+        vis.visit_block_end(results, state, block);
     }
 }

compiler/rustc_mir_dataflow/src/framework/graphviz.rs

@@ -660,13 +660,23 @@ where
     A: Analysis<'tcx>,
     A::Domain: DebugWithContext<A>,
 {
-    fn visit_block_start(&mut self, state: &A::Domain) {
+    fn visit_block_start(
+        &mut self,
+        _results: &Results<'tcx, A>,
+        state: &A::Domain,
+        _block: BasicBlock,
+    ) {
         if A::Direction::IS_FORWARD {
             self.prev_state.clone_from(state);
         }
     }
 
-    fn visit_block_end(&mut self, state: &A::Domain) {
+    fn visit_block_end(
+        &mut self,
+        _results: &Results<'tcx, A>,
+        state: &A::Domain,
+        _block: BasicBlock,
+    ) {
         if A::Direction::IS_BACKWARD {
             self.prev_state.clone_from(state);
         }

compiler/rustc_mir_dataflow/src/framework/visitor.rs

@@ -23,7 +23,7 @@ pub fn visit_results<'mir, 'tcx, A>(
 
         let block_data = &body[block];
         state.clone_from(&results.entry_states[block]);
-        A::Direction::visit_results_in_block(&results.analysis, &mut state, block, block_data, vis);
+        A::Direction::visit_results_in_block(results, &mut state, block, block_data, vis);
     }
 }
 
@@ -46,7 +46,13 @@ pub trait ResultsVisitor<'tcx, A>
 where
     A: Analysis<'tcx>,
 {
-    fn visit_block_start(&mut self, _state: &A::Domain) {}
+    fn visit_block_start(
+        &mut self,
+        _results: &Results<'tcx, A>,
+        _state: &A::Domain,
+        _block: BasicBlock,
+    ) {
+    }
 
     /// Called after the "early" effect of the given statement is applied to `state`.
     fn visit_after_early_statement_effect(
@@ -90,5 +96,11 @@ where
     ) {
     }
 
-    fn visit_block_end(&mut self, _state: &A::Domain) {}
+    fn visit_block_end(
+        &mut self,
+        _results: &Results<'tcx, A>,
+        _state: &A::Domain,
+        _block: BasicBlock,
+    ) {
+    }
 }

compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs

@@ -71,6 +71,17 @@ where
         }
     }
 
+    fn visit_operand(&mut self, operand: &Operand<'tcx>, location: Location) {
+        self.super_operand(operand, location);
+
+        // Kill moved operands if the whole local was moved.
+        if let Operand::Move(place) = operand {
+            if let Some(local) = place.as_local() {
+                self.trans.kill(local);
+            }
+        }
+    }
+
     fn visit_rvalue(&mut self, rvalue: &Rvalue<'tcx>, location: Location) {
         self.super_rvalue(rvalue, location);
 

compiler/rustc_mir_dataflow/src/impls/mod.rs

@@ -1,6 +1,7 @@
 mod borrowed_locals;
 mod initialized;
 mod liveness;
+mod precise_liveness;
 mod storage_liveness;
 
 pub use self::borrowed_locals::{MaybeBorrowedLocals, borrowed_locals};
@@ -12,6 +13,7 @@ pub use self::liveness::{
     DefUse, MaybeLiveLocals, MaybeTransitiveLiveLocals,
     TransferFunction as LivenessTransferFunction,
 };
+pub use self::precise_liveness::{SplitPointEffect, SplitPointIndex, liveness_matrix};
 pub use self::storage_liveness::{
     MaybeRequiresStorage, MaybeStorageDead, MaybeStorageLive, always_storage_live_locals,
 };