Reinstate ConstValueKind::Slice.

Author: cjgillot

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -209,6 +209,17 @@ pub(crate) fn codegen_const_value<'tcx>(
             layout,
         ),
         ConstValueKind::ScalarPair(..) => todo!(),
+        ConstValueKind::Slice { data, start, end } => {
+            let alloc_id = fx.tcx.reserve_and_set_memory_alloc(data);
+            let ptr = pointer_for_allocation(fx, alloc_id)
+                .offset_i64(fx, i64::try_from(start).unwrap())
+                .get_addr(fx);
+            let len = fx
+                .bcx
+                .ins()
+                .iconst(fx.pointer_type, i64::try_from(end.checked_sub(start).unwrap()).unwrap());
+            CValue::by_val_pair(ptr, len, layout)
+        }
     }
 }
 

compiler/rustc_codegen_ssa/src/mir/operand.rs

@@ -7,7 +7,7 @@ use crate::traits::*;
 use crate::MemFlags;
 
 use rustc_middle::mir;
-use rustc_middle::mir::interpret::{alloc_range, ConstValue, ConstValueKind};
+use rustc_middle::mir::interpret::{alloc_range, ConstValue, ConstValueKind, Pointer, Scalar};
 use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
 use rustc_middle::ty::Ty;
 use rustc_target::abi::{self, Abi, Align, Size};
@@ -102,7 +102,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
             ConstValueKind::ZeroSized => return OperandRef::zero_sized(layout),
             ConstValueKind::ScalarPair(a, b) => {
                 let Abi::ScalarPair(a_scalar, b_scalar) = layout.abi else {
-                    bug!("from_const: invalid ByVal layout: {:#?}", layout);
+                    bug!("from_const: invalid ScalarPair layout: {:#?}", layout);
                 };
                 let a_llval = bx.scalar_to_backend(
                     a,
@@ -116,6 +116,25 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
                 );
                 OperandValue::Pair(a_llval, b_llval)
             }
+            ConstValueKind::Slice { data, start, end } => {
+                let Abi::ScalarPair(a_scalar, _) = layout.abi else {
+                    bug!("from_const: invalid ScalarPair layout: {:#?}", layout);
+                };
+                let a = Scalar::from_pointer(
+                    Pointer::new(
+                        bx.tcx().reserve_and_set_memory_alloc(data),
+                        Size::from_bytes(start),
+                    ),
+                    &bx.tcx(),
+                );
+                let a_llval = bx.scalar_to_backend(
+                    a,
+                    a_scalar,
+                    bx.scalar_pair_element_backend_type(layout, 0, true),
+                );
+                let b_llval = bx.const_usize((end - start) as u64);
+                OperandValue::Pair(a_llval, b_llval)
+            }
             ConstValueKind::Indirect { alloc_id, offset } => {
                 let alloc = bx.tcx().global_alloc(alloc_id).unwrap_memory();
                 return Self::from_const_alloc(bx, layout, alloc, offset);

compiler/rustc_const_eval/src/interpret/intrinsics.rs

@@ -5,7 +5,9 @@
 use rustc_hir::def_id::DefId;
 use rustc_middle::mir::{
     self,
-    interpret::{AllocId, ConstValue, GlobalId, InterpResult, PointerArithmetic, Scalar},
+    interpret::{
+        Allocation, ConstAllocation, ConstValue, GlobalId, InterpResult, PointerArithmetic, Scalar,
+    },
     BinOp, NonDivergingIntrinsic,
 };
 use rustc_middle::ty;
@@ -42,11 +44,10 @@ fn numeric_intrinsic<Prov>(name: Symbol, bits: u128, kind: Primitive) -> Scalar<
 }
 
 /// Directly returns an `Allocation` containing an absolute path representation of the given type.
-pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> (AllocId, usize) {
+pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ConstAllocation<'tcx> {
     let path = crate::util::type_name(tcx, ty);
-    let bytes = path.into_bytes();
-    let len = bytes.len();
-    (tcx.allocate_bytes(bytes), len)
+    let alloc = Allocation::from_bytes_byte_aligned_immutable(path.into_bytes());
+    tcx.mk_const_alloc(alloc)
 }
 
 /// The logic for all nullary intrinsics is implemented here. These intrinsics don't get evaluated
@@ -62,8 +63,8 @@ pub(crate) fn eval_nullary_intrinsic<'tcx>(
     Ok(match name {
         sym::type_name => {
             ensure_monomorphic_enough(tcx, tp_ty)?;
-            let (alloc, len) = alloc_type_name(tcx, tp_ty);
-            ConstValue::from_slice(tcx, Pointer::from(alloc), len)
+            let alloc = alloc_type_name(tcx, tp_ty);
+            ConstValue::from_raw_slice(tcx, alloc, 0, alloc.inner().len())
         }
         sym::needs_drop => {
             ensure_monomorphic_enough(tcx, tp_ty)?;

compiler/rustc_const_eval/src/interpret/operand.rs

@@ -775,6 +775,19 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 Operand::Immediate(Immediate::ScalarPair(adjust_scalar(a)?, adjust_scalar(b)?))
             }
             ConstValueKind::ZeroSized => Operand::Immediate(Immediate::Uninit),
+            ConstValueKind::Slice { data, start, end } => {
+                // We rely on mutability being set correctly in `data` to prevent writes
+                // where none should happen.
+                let ptr = Pointer::new(
+                    self.tcx.reserve_and_set_memory_alloc(data),
+                    Size::from_bytes(start), // offset: `start`
+                );
+                Operand::Immediate(Immediate::new_slice(
+                    Scalar::from_pointer(self.global_base_pointer(ptr)?, &*self.tcx),
+                    u64::try_from(end.checked_sub(start).unwrap()).unwrap(), // len: `end - start`
+                    self,
+                ))
+            }
         };
         Ok(OpTy { op, layout, align: Some(layout.align.abi) })
     }

compiler/rustc_middle/src/mir/interpret/value.rs

@@ -13,7 +13,8 @@ use rustc_target::abi::{HasDataLayout, Size};
 use crate::ty::{ParamEnv, ScalarInt, Ty, TyCtxt};
 
 use super::{
-    alloc_range, AllocId, InterpResult, Pointer, PointerArithmetic, Provenance, ScalarSizeMismatch,
+    AllocId, ConstAllocation, InterpResult, Pointer, PointerArithmetic, Provenance,
+    ScalarSizeMismatch,
 };
 
 /// Represents the result of const evaluation via the `eval_to_allocation` query.
@@ -27,13 +28,13 @@ pub struct ConstAlloc<'tcx> {
 
 #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)]
 #[derive(HashStable)]
-pub struct ConstValue<'tcx>(pub(crate) Interned<'tcx, ConstValueKind>);
+pub struct ConstValue<'tcx>(pub(crate) Interned<'tcx, ConstValueKind<'tcx>>);
 
 /// Represents a constant value in Rust. `Scalar` and `Slice` are optimizations for
 /// array length computations, enum discriminants and the pattern matching logic.
 #[derive(Copy, Clone, Debug, Eq, PartialEq, TyEncodable, TyDecodable, Hash)]
-#[derive(HashStable)]
-pub enum ConstValueKind {
+#[derive(HashStable, Lift)]
+pub enum ConstValueKind<'tcx> {
     /// Used for types with `layout::abi::Scalar` ABI.
     ///
     /// Not using the enum `Value` to encode that this must not be `Uninit`.
@@ -47,9 +48,16 @@ pub enum ConstValueKind {
     /// Only for ZSTs.
     ZeroSized,
 
+    /// Used for `&[u8]` and `&str`.
+    ///
+    /// This is worth an optimized representation since Rust has literals of these types.
+    /// Not having to indirect those through an `AllocId` (or two, if we used `Indirect`) has shown
+    /// measurable performance improvements on stress tests.
+    Slice { data: ConstAllocation<'tcx>, start: usize, end: usize },
+
     /// A value not representable by the other variants; needs to be stored in-memory.
     ///
-    /// Must *not* be used for scalars or ZST, but having `&str` or other slices in this variant is fine.
+    /// Must *not* be used for scalars, scalar pairs or ZST.
     Indirect {
         /// The backing memory of the value. May contain more memory than needed for just the value
         /// if this points into some other larger ConstValue.
@@ -68,12 +76,12 @@ static_assert_size!(ConstValue<'_>, 8);
 
 impl<'tcx> ConstValue<'tcx> {
     #[inline]
-    pub fn new(tcx: TyCtxt<'tcx>, kind: ConstValueKind) -> ConstValue<'tcx> {
+    pub fn new(tcx: TyCtxt<'tcx>, kind: ConstValueKind<'tcx>) -> ConstValue<'tcx> {
         tcx.intern_const_value(kind)
     }
 
     #[inline]
-    pub fn kind(self) -> &'tcx ConstValueKind {
+    pub fn kind(self) -> &'tcx ConstValueKind<'tcx> {
         self.0.0
     }
 
@@ -82,6 +90,7 @@ impl<'tcx> ConstValue<'tcx> {
         match self.kind() {
             ConstValueKind::Indirect { .. }
             | ConstValueKind::ScalarPair(..)
+            | ConstValueKind::Slice { .. }
             | ConstValueKind::ZeroSized => None,
             ConstValueKind::Scalar(val) => Some(*val),
         }
@@ -162,6 +171,16 @@ impl<'tcx> ConstValue<'tcx> {
         )
     }
 
+    #[inline]
+    pub fn from_raw_slice(
+        tcx: TyCtxt<'tcx>,
+        data: ConstAllocation<'tcx>,
+        start: usize,
+        end: usize,
+    ) -> Self {
+        Self::new(tcx, ConstValueKind::Slice { data, start, end })
+    }
+
     #[inline]
     pub fn from_memory(tcx: TyCtxt<'tcx>, alloc_id: AllocId, offset: Size) -> Self {
         debug_assert!(matches!(tcx.global_alloc(alloc_id), super::GlobalAlloc::Memory(_)));
@@ -178,41 +197,7 @@ impl<'tcx> ConstValue<'tcx> {
                 let length = length.try_to_target_usize(tcx).unwrap() as usize;
                 (alloc, start, start + length)
             }
-            &ConstValueKind::Indirect { alloc_id, offset } => {
-                // The reference itself is stored behind an indirection.
-                // Load the reference, and then load the actual slice contents.
-                let a = tcx.global_alloc(alloc_id).unwrap_memory().inner();
-                let ptr_size = tcx.data_layout.pointer_size;
-                if a.size() < offset + 2 * ptr_size {
-                    // (partially) dangling reference
-                    return None;
-                }
-                // Read the wide pointer components.
-                let ptr = a
-                    .read_scalar(
-                        &tcx,
-                        alloc_range(offset, ptr_size),
-                        /* read_provenance */ true,
-                    )
-                    .ok()?;
-                let ptr = ptr.to_pointer(&tcx).ok()?;
-                let len = a
-                    .read_scalar(
-                        &tcx,
-                        alloc_range(offset + ptr_size, ptr_size),
-                        /* read_provenance */ false,
-                    )
-                    .ok()?;
-                let len = len.to_target_usize(&tcx).ok()?;
-                let len: usize = len.try_into().ok()?;
-                if len == 0 {
-                    return Some(&[]);
-                }
-                // Non-empty slice, must have memory. We know this is a relative pointer.
-                let (inner_alloc_id, offset) = ptr.into_parts();
-                let data = tcx.global_alloc(inner_alloc_id?).unwrap_memory();
-                (data, offset.bytes_usize(), offset.bytes_usize() + len)
-            }
+            &ConstValueKind::Slice { data, start, end } => (data, start, end),
             _ => {
                 bug!("`try_get_slice_bytes` on non-slice constant")
             }

compiler/rustc_middle/src/mir/pretty.rs

@@ -459,6 +459,7 @@ impl<'tcx> Visitor<'tcx> for ExtraComments<'tcx> {
                 ConstValueKind::ZeroSized => "<ZST>".to_string(),
                 ConstValueKind::Scalar(s) => format!("Scalar({s:?})"),
                 ConstValueKind::ScalarPair(a, b) => format!("ScalarPair({a:?}, {b:?})"),
+                ConstValueKind::Slice { .. } => "Slice(..)".to_string(),
                 ConstValueKind::Indirect { .. } => "ByRef(..)".to_string(),
             };
 
@@ -713,6 +714,10 @@ pub fn write_allocations<'tcx>(
             ConstValueKind::ScalarPair(a, b) => {
                 alloc_ids_from_scalar(a).into_iter().chain(alloc_ids_from_scalar(b).into_iter())
             }
+            ConstValueKind::Slice { .. } => {
+                // `u8`/`str` slices, shouldn't contain pointers that we want to print.
+                None.into_iter().chain(None.into_iter())
+            }
             ConstValueKind::Indirect { alloc_id, .. } => {
                 // FIXME: we don't actually want to print all of these, since some are printed nicely directly as values inline in MIR.
                 // Really we'd want `pretty_print_const_value` to decide which allocations to print, instead of having a separate visitor.

compiler/rustc_middle/src/ty/context.rs

@@ -156,7 +156,7 @@ pub struct CtxtInterners<'tcx> {
     external_constraints: InternedSet<'tcx, ExternalConstraintsData<'tcx>>,
     predefined_opaques_in_body: InternedSet<'tcx, PredefinedOpaquesData<'tcx>>,
     fields: InternedSet<'tcx, List<FieldIdx>>,
-    const_value: InternedSet<'tcx, ConstValueKind>,
+    const_value: InternedSet<'tcx, ConstValueKind<'tcx>>,
 }
 
 impl<'tcx> CtxtInterners<'tcx> {
@@ -1499,7 +1499,7 @@ direct_interners! {
     region: pub(crate) intern_region(RegionKind<'tcx>): Region -> Region<'tcx>,
     const_: intern_const(ConstData<'tcx>): Const -> Const<'tcx>,
     const_allocation: pub mk_const_alloc(Allocation): ConstAllocation -> ConstAllocation<'tcx>,
-    const_value: pub(crate) intern_const_value(ConstValueKind): ConstValue -> ConstValue<'tcx>,
+    const_value: pub(crate) intern_const_value(ConstValueKind<'tcx>): ConstValue -> ConstValue<'tcx>,
     layout: pub mk_layout(LayoutS): Layout -> Layout<'tcx>,
     adt_def: pub mk_adt_def_from_data(AdtDefData): AdtDef -> AdtDef<'tcx>,
     external_constraints: pub mk_external_constraints(ExternalConstraintsData<'tcx>):

compiler/rustc_middle/src/ty/structural_impls.rs

@@ -627,7 +627,7 @@ impl<'a, 'tcx> Lift<'tcx> for ty::ParamEnv<'a> {
 impl<'a, 'tcx> Lift<'tcx> for interpret::ConstValue<'a> {
     type Lifted = interpret::ConstValue<'tcx>;
     fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        Some(interpret::ConstValue::new(tcx, *self.kind()))
+        tcx.lift(*self.kind()).map(|kind| interpret::ConstValue::new(tcx, kind))
     }
 }
 

compiler/rustc_mir_build/src/build/expr/as_constant.rs

@@ -3,7 +3,9 @@
 use crate::build::{parse_float_into_constval, Builder};
 use rustc_ast as ast;
 use rustc_middle::mir;
-use rustc_middle::mir::interpret::{ConstValue, LitToConstError, LitToConstInput, Scalar};
+use rustc_middle::mir::interpret::{
+    Allocation, ConstValue, LitToConstError, LitToConstInput, Scalar,
+};
 use rustc_middle::mir::*;
 use rustc_middle::thir::*;
 use rustc_middle::ty::{
@@ -128,24 +130,27 @@ fn lit_to_mir_constant<'tcx>(
 
     let value = match (lit, &ty.kind()) {
         (ast::LitKind::Str(s, _), ty::Ref(_, inner_ty, _)) if inner_ty.is_str() => {
-            let s = s.as_str().as_bytes();
-            let allocation = tcx.allocate_bytes(s);
-            ConstValue::from_slice(tcx, allocation.into(), s.len())
+            let s = s.as_str();
+            let allocation = Allocation::from_bytes_byte_aligned_immutable(s.as_bytes());
+            let allocation = tcx.mk_const_alloc(allocation);
+            ConstValue::from_raw_slice(tcx, allocation, 0, s.len())
         }
         (ast::LitKind::ByteStr(data, _), ty::Ref(_, inner_ty, _))
             if matches!(inner_ty.kind(), ty::Slice(_)) =>
         {
-            let allocation = tcx.allocate_bytes(data as &[u8]);
-            ConstValue::from_slice(tcx, allocation.into(), data.len())
+            let allocation = Allocation::from_bytes_byte_aligned_immutable(data as &[u8]);
+            let allocation = tcx.mk_const_alloc(allocation);
+            ConstValue::from_raw_slice(tcx, allocation, 0, data.len())
         }
         (ast::LitKind::ByteStr(data, _), ty::Ref(_, inner_ty, _)) if inner_ty.is_array() => {
             let id = tcx.allocate_bytes(&**data);
             ConstValue::from_pointer(tcx, id.into())
         }
         (ast::LitKind::CStr(data, _), ty::Ref(_, inner_ty, _)) if matches!(inner_ty.kind(), ty::Adt(def, _) if Some(def.did()) == tcx.lang_items().c_str()) =>
         {
-            let allocation = tcx.allocate_bytes(data as &[u8]);
-            ConstValue::from_slice(tcx, allocation.into(), data.len())
+            let allocation = Allocation::from_bytes_byte_aligned_immutable(data as &[u8]);
+            let allocation = tcx.mk_const_alloc(allocation);
+            ConstValue::from_raw_slice(tcx, allocation, 0, data.len())
         }
         (ast::LitKind::Byte(n), ty::Uint(ty::UintTy::U8)) => {
             ConstValue::from_scalar(tcx, Scalar::from_uint(*n, Size::from_bytes(1)))

compiler/rustc_monomorphize/src/collector.rs

@@ -1461,5 +1461,10 @@ fn collect_const_value<'tcx>(
         }
         ConstValueKind::ZeroSized => {}
         ConstValueKind::Indirect { alloc_id, .. } => collect_alloc(tcx, alloc_id, output),
+        ConstValueKind::Slice { data, start: _, end: _ } => {
+            for &id in data.inner().provenance().ptrs().values() {
+                collect_alloc(tcx, id, output);
+            }
+        }
     }
 }

compiler/rustc_smir/src/rustc_smir/alloc.rs

@@ -1,4 +1,4 @@
-use rustc_middle::mir::interpret::{alloc_range, AllocRange, ConstValue, ConstValueKind};
+use rustc_middle::mir::interpret::{alloc_range, AllocRange, ConstValue, ConstValueKind, Pointer};
 
 use crate::{
     rustc_smir::{Stable, Tables},
@@ -61,6 +61,34 @@ pub fn new_allocation<'tcx>(
                 .unwrap();
             allocation.stable(tables)
         }
+        ConstValueKind::Slice { data, start, end } => {
+            let alloc_id = tables.tcx.reserve_and_set_memory_alloc(data);
+            let ptr = Pointer::new(alloc_id, rustc_target::abi::Size::from_bytes(start));
+            let scalar_ptr = rustc_middle::mir::interpret::Scalar::from_pointer(ptr, &tables.tcx);
+            let scalar_len = rustc_middle::mir::interpret::Scalar::from_target_usize(
+                (end - start) as u64,
+                &tables.tcx,
+            );
+            let layout =
+                tables.tcx.layout_of(rustc_middle::ty::ParamEnv::reveal_all().and(ty)).unwrap();
+            let mut allocation =
+                rustc_middle::mir::interpret::Allocation::uninit(layout.size, layout.align.abi);
+            allocation
+                .write_scalar(
+                    &tables.tcx,
+                    alloc_range(rustc_target::abi::Size::ZERO, tables.tcx.data_layout.pointer_size),
+                    scalar_ptr,
+                )
+                .unwrap();
+            allocation
+                .write_scalar(
+                    &tables.tcx,
+                    alloc_range(tables.tcx.data_layout.pointer_size, scalar_len.size()),
+                    scalar_len,
+                )
+                .unwrap();
+            allocation.stable(tables)
+        }
         ConstValueKind::Indirect { alloc_id, offset } => {
             let alloc = tables.tcx.global_alloc(alloc_id).unwrap_memory();
             let ty_size = tables

tests/mir-opt/building/issue_101867.main.built.after.mir

@@ -67,7 +67,3 @@ fn main() -> () {
         resume;
     }
 }
-
-alloc1 (size: 14, align: 1) {
-    65 78 70 6c 69 63 69 74 20 70 61 6e 69 63       │ explicit panic
-}