Auto merge of #156046 - Amanieu:move-elimination, r=<try>

[WIP] MIR move elimination pass

Author: bors

compiler/rustc_index/src/interval.rs

@@ -1,6 +1,7 @@
 use std::iter::Step;
 use std::marker::PhantomData;
-use std::ops::{Bound, Range, RangeBounds};
+use std::ops::{Bound, RangeBounds};
+use std::range::RangeInclusive;
 
 use smallvec::SmallVec;
 
@@ -59,11 +60,14 @@ impl<I: Idx> IntervalSet<I> {
     }
 
     /// Iterates through intervals stored in the set, in order.
-    pub fn iter_intervals(&self) -> impl Iterator<Item = std::ops::Range<I>>
+    pub fn iter_intervals(&self) -> impl Iterator<Item = RangeInclusive<I>>
     where
         I: Step,
     {
-        self.map.iter().map(|&(start, end)| I::new(start as usize)..I::new(end as usize + 1))
+        self.map.iter().map(|&(start, end)| RangeInclusive {
+            start: I::new(start as usize),
+            last: I::new(end as usize),
+        })
     }
 
     /// Returns true if we increased the number of elements present.
@@ -164,6 +168,35 @@ impl<I: Idx> IntervalSet<I> {
         );
     }
 
+    /// Specialized version of `insert` when we know that the inserted range is
+    /// *after* any contained.
+    pub fn append_range(&mut self, range: impl RangeBounds<I> + Clone) {
+        let start = inclusive_start(range.clone());
+        let Some(end) = inclusive_end(self.domain, range) else {
+            // empty range
+            return;
+        };
+        if start > end {
+            return;
+        }
+
+        if let Some((_, last_end)) = self.map.last_mut() {
+            assert!(*last_end < start);
+            if start == *last_end + 1 {
+                *last_end = end;
+            } else {
+                self.map.push((start, end));
+            }
+        } else {
+            self.map.push((start, end));
+        }
+
+        debug_assert!(
+            self.check_invariants(),
+            "wrong intervals after append {start:?}..={end:?} to {self:?}"
+        );
+    }
+
     pub fn contains(&self, needle: I) -> bool {
         let needle = needle.index() as u32;
         let Some(last) = self.map.partition_point(|r| r.0 <= needle).checked_sub(1) else {
@@ -180,11 +213,12 @@ impl<I: Idx> IntervalSet<I> {
     {
         let mut sup_iter = self.iter_intervals();
         let mut current = None;
-        let contains = |sup: Range<I>, sub: Range<I>, current: &mut Option<Range<I>>| {
-            if sup.end < sub.start {
-                // if `sup.end == sub.start`, the next sup doesn't contain `sub.start`
+        let contains = |sup: RangeInclusive<I>,
+                        sub: RangeInclusive<I>,
+                        current: &mut Option<RangeInclusive<I>>| {
+            if sup.last < sub.start {
                 None // continue to the next sup
-            } else if sup.end >= sub.end && sup.start <= sub.start {
+            } else if sup.last >= sub.last && sup.start <= sub.start {
                 *current = Some(sup); // save the current sup
                 Some(true)
             } else {
@@ -194,8 +228,8 @@ impl<I: Idx> IntervalSet<I> {
         other.iter_intervals().all(|sub| {
             current
                 .take()
-                .and_then(|sup| contains(sup, sub.clone(), &mut current))
-                .or_else(|| sup_iter.find_map(|sup| contains(sup, sub.clone(), &mut current)))
+                .and_then(|sup| contains(sup, sub, &mut current))
+                .or_else(|| sup_iter.find_map(|sup| contains(sup, sub, &mut current)))
                 .unwrap_or(false)
         })
     }
@@ -212,11 +246,11 @@ impl<I: Idx> IntervalSet<I> {
             let mut other_current = other_iter.next()?;
 
             loop {
-                if self_current.end <= other_current.start {
+                if self_current.last < other_current.start {
                     self_current = self_iter.next()?;
                     continue;
                 }
-                if other_current.end <= self_current.start {
+                if other_current.last < self_current.start {
                     other_current = other_iter.next()?;
                     continue;
                 }
@@ -340,6 +374,12 @@ impl<R: Idx, C: Step + Idx> SparseIntervalMatrix<R, C> {
         self.rows.get(row)
     }
 
+    pub fn clear_row(&mut self, row: R) {
+        if let Some(row) = self.rows.get_mut(row) {
+            row.clear();
+        }
+    }
+
     fn ensure_row(&mut self, row: R) -> &mut IntervalSet<C> {
         self.rows.ensure_contains_elem(row, || IntervalSet::new(self.column_size))
     }
@@ -363,6 +403,16 @@ impl<R: Idx, C: Step + Idx> SparseIntervalMatrix<R, C> {
         write_row.union(read_row)
     }
 
+    pub fn disjoint_rows(&self, a: R, b: R) -> bool
+    where
+        C: Step,
+    {
+        let (Some(a), Some(b)) = (self.rows.get(a), self.rows.get(b)) else {
+            return true;
+        };
+        a.disjoint(b)
+    }
+
     pub fn insert_all_into_row(&mut self, row: R) {
         self.ensure_row(row).insert_all();
     }
@@ -379,6 +429,10 @@ impl<R: Idx, C: Step + Idx> SparseIntervalMatrix<R, C> {
         self.ensure_row(row).append(point)
     }
 
+    pub fn append_range(&mut self, row: R, range: impl RangeBounds<C> + Clone) {
+        self.ensure_row(row).append_range(range)
+    }
+
     pub fn contains(&self, row: R, point: C) -> bool {
         self.row(row).is_some_and(|r| r.contains(point))
     }

compiler/rustc_index/src/interval/tests.rs

@@ -5,7 +5,7 @@ fn insert_collapses() {
     let mut set = IntervalSet::<u32>::new(10000);
     set.insert_range(9831..=9837);
     set.insert_range(43..=9830);
-    assert_eq!(set.iter_intervals().collect::<Vec<_>>(), [43..9838]);
+    assert_eq!(set.iter_intervals().collect::<Vec<_>>(), [(43..=9837).into()]);
 }
 
 #[test]

compiler/rustc_middle/src/mir/syntax.rs

@@ -124,7 +124,6 @@ pub enum RuntimePhase {
     /// disallowed:
     /// * [`TerminatorKind::Yield`]
     /// * [`TerminatorKind::CoroutineDrop`]
-    /// * [`Rvalue::Aggregate`] for any `AggregateKind` except `Array`
     /// * [`Rvalue::CopyForDeref`]
     /// * [`PlaceElem::OpaqueCast`]
     /// * [`LocalInfo::DerefTemp`](super::LocalInfo::DerefTemp)
@@ -1442,9 +1441,6 @@ pub enum Rvalue<'tcx> {
     /// This is needed because dataflow analysis needs to distinguish
     /// `dest = Foo { x: ..., y: ... }` from `dest.x = ...; dest.y = ...;` in the case that `Foo`
     /// has a destructor.
-    ///
-    /// Disallowed after deaggregation for all aggregate kinds except `Array` and `Coroutine`. After
-    /// coroutine lowering, `Coroutine` aggregate kinds are disallowed too.
     Aggregate(Box<AggregateKind<'tcx>>, IndexVec<FieldIdx, Operand<'tcx>>),
 
     /// A CopyForDeref is equivalent to a read from a place at the

compiler/rustc_mir_dataflow/src/framework/direction.rs

@@ -214,7 +214,7 @@ impl Direction for Backward {
     ) where
         A: Analysis<'tcx>,
     {
-        vis.visit_block_end(state);
+        vis.visit_block_end(state, block);
 
         let loc = Location { block, statement_index: block_data.statements.len() };
         let term = block_data.terminator();
@@ -231,7 +231,7 @@ impl Direction for Backward {
             vis.visit_after_primary_statement_effect(analysis, state, stmt, loc);
         }
 
-        vis.visit_block_start(state);
+        vis.visit_block_start(state, block);
     }
 }
 
@@ -394,7 +394,7 @@ impl Direction for Forward {
     ) where
         A: Analysis<'tcx>,
     {
-        vis.visit_block_start(state);
+        vis.visit_block_start(state, block);
 
         for (statement_index, stmt) in block_data.statements.iter().enumerate() {
             let loc = Location { block, statement_index };
@@ -411,6 +411,6 @@ impl Direction for Forward {
         analysis.apply_primary_terminator_effect(state, term, loc);
         vis.visit_after_primary_terminator_effect(analysis, state, term, loc);
 
-        vis.visit_block_end(state);
+        vis.visit_block_end(state, block);
     }
 }

compiler/rustc_mir_dataflow/src/framework/graphviz.rs

@@ -660,13 +660,13 @@ where
     A: Analysis<'tcx>,
     A::Domain: DebugWithContext<A>,
 {
-    fn visit_block_start(&mut self, state: &A::Domain) {
+    fn visit_block_start(&mut self, state: &A::Domain, _block: BasicBlock) {
         if A::Direction::IS_FORWARD {
             self.prev_state.clone_from(state);
         }
     }
 
-    fn visit_block_end(&mut self, state: &A::Domain) {
+    fn visit_block_end(&mut self, state: &A::Domain, _block: BasicBlock) {
         if A::Direction::IS_BACKWARD {
             self.prev_state.clone_from(state);
         }

compiler/rustc_mir_dataflow/src/framework/visitor.rs

@@ -46,7 +46,7 @@ pub trait ResultsVisitor<'tcx, A>
 where
     A: Analysis<'tcx>,
 {
-    fn visit_block_start(&mut self, _state: &A::Domain) {}
+    fn visit_block_start(&mut self, _state: &A::Domain, _block: BasicBlock) {}
 
     /// Called after the "early" effect of the given statement is applied to `state`.
     fn visit_after_early_statement_effect(
@@ -90,5 +90,5 @@ where
     ) {
     }
 
-    fn visit_block_end(&mut self, _state: &A::Domain) {}
+    fn visit_block_end(&mut self, _state: &A::Domain, _block: BasicBlock) {}
 }

compiler/rustc_mir_dataflow/src/impls/mod.rs

@@ -1,6 +1,7 @@
 mod borrowed_locals;
 mod initialized;
 mod liveness;
+mod precise_liveness;
 mod storage_liveness;
 
 pub use self::borrowed_locals::{MaybeBorrowedLocals, borrowed_locals};
@@ -12,6 +13,7 @@ pub use self::liveness::{
     DefUse, MaybeLiveLocals, MaybeTransitiveLiveLocals,
     TransferFunction as LivenessTransferFunction,
 };
+pub use self::precise_liveness::{SplitPointEffect, SplitPointIndex, liveness_matrix};
 pub use self::storage_liveness::{
     MaybeRequiresStorage, MaybeStorageDead, MaybeStorageLive, always_storage_live_locals,
 };