From 9a1a75321ed1c62bffef83e7965d7450638608be Mon Sep 17 00:00:00 2001
From: Lukas Markeffsky <@>
Date: Fri, 7 Oct 2022 20:57:34 +0200
Subject: [PATCH 1/7] constify `exact_div` intrinsic
---
compiler/rustc_const_eval/src/interpret/intrinsics.rs | 5 +++++
library/core/src/intrinsics.rs | 1 +
library/core/src/lib.rs | 1 +
src/tools/miri/src/shims/intrinsics/mod.rs | 5 -----
4 files changed, 7 insertions(+), 5 deletions(-)
diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
index 8637d6a7767e4..7534a27f0770c 100644
--- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs
+++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
@@ -234,6 +234,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
let discr_val = self.read_discriminant(&place.into())?.0;
self.write_scalar(discr_val, dest)?;
}
+ sym::exact_div => {
+ let l = self.read_immediate(&args[0])?;
+ let r = self.read_immediate(&args[1])?;
+ self.exact_div(&l, &r, dest)?;
+ }
sym::unchecked_shl
| sym::unchecked_shr
| sym::unchecked_add
diff --git a/library/core/src/intrinsics.rs b/library/core/src/intrinsics.rs
index 2399262c05b46..be9acae8c986b 100644
--- a/library/core/src/intrinsics.rs
+++ b/library/core/src/intrinsics.rs
@@ -1846,6 +1846,7 @@ extern "rust-intrinsic" {
/// `x % y != 0` or `y == 0` or `x == T::MIN && y == -1`
///
/// This intrinsic does not have a stable counterpart.
+ #[rustc_const_unstable(feature = "const_exact_div", issue = "none")]
pub fn exact_div<T: Copy>(x: T, y: T) -> T;
/// Performs an unchecked division, resulting in undefined behavior
diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs
index 2fd8180f8b2a2..6294ef0df3dec 100644
--- a/library/core/src/lib.rs
+++ b/library/core/src/lib.rs
@@ -109,6 +109,7 @@
#![feature(const_cmp)]
#![feature(const_discriminant)]
#![feature(const_eval_select)]
+#![feature(const_exact_div)]
#![feature(const_float_bits_conv)]
#![feature(const_float_classify)]
#![feature(const_fmt_arguments_new)]
diff --git a/src/tools/miri/src/shims/intrinsics/mod.rs b/src/tools/miri/src/shims/intrinsics/mod.rs
index e0985ace5be7d..31e9f7a7796db 100644
--- a/src/tools/miri/src/shims/intrinsics/mod.rs
+++ b/src/tools/miri/src/shims/intrinsics/mod.rs
@@ -357,11 +357,6 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
}
// Other
- "exact_div" => {
- let [num, denom] = check_arg_count(args)?;
- this.exact_div(&this.read_immediate(num)?, &this.read_immediate(denom)?, dest)?;
- }
-
"breakpoint" => {
let [] = check_arg_count(args)?;
// normally this would raise a SIGTRAP, which aborts if no debugger is connected
From f24fc19b51acadaf82b7c81849788c8ff5f6ed62 Mon Sep 17 00:00:00 2001
From: Lukas Markeffsky <@>
Date: Thu, 20 Oct 2022 20:15:37 +0200
Subject: [PATCH 2/7] unify inherent impls of `CompileTimeEvalContext`
---
.../src/const_eval/machine.rs | 90 +++++++++----------
1 file changed, 44 insertions(+), 46 deletions(-)
diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs
index e5acacd918888..de601c574562e 100644
--- a/compiler/rustc_const_eval/src/const_eval/machine.rs
+++ b/compiler/rustc_const_eval/src/const_eval/machine.rs
@@ -23,52 +23,6 @@ use crate::interpret::{
use super::error::*;
-impl<'mir, 'tcx> InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>> {
- /// "Intercept" a function call to a panic-related function
- /// because we have something special to do for it.
- /// If this returns successfully (`Ok`), the function should just be evaluated normally.
- fn hook_special_const_fn(
- &mut self,
- instance: ty::Instance<'tcx>,
- args: &[OpTy<'tcx>],
- ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
- // All `#[rustc_do_not_const_check]` functions should be hooked here.
- let def_id = instance.def_id();
-
- if Some(def_id) == self.tcx.lang_items().panic_display()
- || Some(def_id) == self.tcx.lang_items().begin_panic_fn()
- {
- // &str or &&str
- assert!(args.len() == 1);
-
- let mut msg_place = self.deref_operand(&args[0])?;
- while msg_place.layout.ty.is_ref() {
- msg_place = self.deref_operand(&msg_place.into())?;
- }
-
- let msg = Symbol::intern(self.read_str(&msg_place)?);
- let span = self.find_closest_untracked_caller_location();
- let (file, line, col) = self.location_triple_for_span(span);
- return Err(ConstEvalErrKind::Panic { msg, file, line, col }.into());
- } else if Some(def_id) == self.tcx.lang_items().panic_fmt() {
- // For panic_fmt, call const_panic_fmt instead.
- if let Some(const_panic_fmt) = self.tcx.lang_items().const_panic_fmt() {
- return Ok(Some(
- ty::Instance::resolve(
- *self.tcx,
- ty::ParamEnv::reveal_all(),
- const_panic_fmt,
- self.tcx.intern_substs(&[]),
- )
- .unwrap()
- .unwrap(),
- ));
- }
- }
- Ok(None)
- }
-}
-
/// Extra machine state for CTFE, and the Machine instance
pub struct CompileTimeInterpreter<'mir, 'tcx> {
/// For now, the number of terminators that can be evaluated before we throw a resource
@@ -191,6 +145,50 @@ impl interpret::MayLeak for ! {
}
impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> {
+ /// "Intercept" a function call to a panic-related function
+ /// because we have something special to do for it.
+ /// If this returns successfully (`Ok`), the function should just be evaluated normally.
+ fn hook_special_const_fn(
+ &mut self,
+ instance: ty::Instance<'tcx>,
+ args: &[OpTy<'tcx>],
+ ) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
+ // All `#[rustc_do_not_const_check]` functions should be hooked here.
+ let def_id = instance.def_id();
+
+ if Some(def_id) == self.tcx.lang_items().panic_display()
+ || Some(def_id) == self.tcx.lang_items().begin_panic_fn()
+ {
+ // &str or &&str
+ assert!(args.len() == 1);
+
+ let mut msg_place = self.deref_operand(&args[0])?;
+ while msg_place.layout.ty.is_ref() {
+ msg_place = self.deref_operand(&msg_place.into())?;
+ }
+
+ let msg = Symbol::intern(self.read_str(&msg_place)?);
+ let span = self.find_closest_untracked_caller_location();
+ let (file, line, col) = self.location_triple_for_span(span);
+ return Err(ConstEvalErrKind::Panic { msg, file, line, col }.into());
+ } else if Some(def_id) == self.tcx.lang_items().panic_fmt() {
+ // For panic_fmt, call const_panic_fmt instead.
+ if let Some(const_panic_fmt) = self.tcx.lang_items().const_panic_fmt() {
+ return Ok(Some(
+ ty::Instance::resolve(
+ *self.tcx,
+ ty::ParamEnv::reveal_all(),
+ const_panic_fmt,
+ self.tcx.intern_substs(&[]),
+ )
+ .unwrap()
+ .unwrap(),
+ ));
+ }
+ }
+ Ok(None)
+ }
+
/// See documentation on the `ptr_guaranteed_cmp` intrinsic.
fn guaranteed_cmp(&mut self, a: Scalar, b: Scalar) -> InterpResult<'tcx, u8> {
Ok(match (a, b) {
From d4e692d465e87081c3e0ba3e01c76117c54f8e33 Mon Sep 17 00:00:00 2001
From: Lukas Markeffsky <@>
Date: Fri, 7 Oct 2022 22:23:34 +0200
Subject: [PATCH 3/7] make const `align_offset` useful
---
.../src/const_eval/machine.rs | 129 ++++++++++++++----
library/core/src/ptr/const_ptr.rs | 15 ++
library/core/src/ptr/mod.rs | 21 ++-
library/core/src/ptr/mut_ptr.rs | 15 ++
4 files changed, 152 insertions(+), 28 deletions(-)
diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs
index de601c574562e..dffd4c74e86bd 100644
--- a/compiler/rustc_const_eval/src/const_eval/machine.rs
+++ b/compiler/rustc_const_eval/src/const_eval/machine.rs
@@ -1,9 +1,12 @@
use rustc_hir::def::DefKind;
use rustc_middle::mir;
+use rustc_middle::mir::interpret::PointerArithmetic;
+use rustc_middle::ty::layout::FnAbiOf;
use rustc_middle::ty::{self, Ty, TyCtxt};
use std::borrow::Borrow;
use std::collections::hash_map::Entry;
use std::hash::Hash;
+use std::ops::ControlFlow;
use rustc_data_structures::fx::FxHashMap;
use std::fmt;
@@ -17,8 +20,8 @@ use rustc_target::abi::{Align, Size};
use rustc_target::spec::abi::Abi as CallAbi;
use crate::interpret::{
- self, compile_time_machine, AllocId, ConstAllocation, Frame, ImmTy, InterpCx, InterpResult,
- OpTy, PlaceTy, Pointer, Scalar, StackPopUnwind,
+ self, compile_time_machine, AllocId, ConstAllocation, FnVal, Frame, ImmTy, InterpCx,
+ InterpResult, OpTy, PlaceTy, Pointer, Scalar, StackPopUnwind,
};
use super::error::*;
@@ -145,15 +148,19 @@ impl interpret::MayLeak for ! {
}
impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> {
- /// "Intercept" a function call to a panic-related function
- /// because we have something special to do for it.
- /// If this returns successfully (`Ok`), the function should just be evaluated normally.
+ /// "Intercept" a function call, because we have something special to do for it.
+ /// All `#[rustc_do_not_const_check]` functions should be hooked here.
+ /// If this returns `Some` function, which may be `instance` or a different function with
+ /// compatible arguments, then evaluation should continue with that function.
+ /// If this returns `None`, the function call has been handled and the function has returned.
fn hook_special_const_fn(
&mut self,
instance: ty::Instance<'tcx>,
+ _abi: CallAbi,
args: &[OpTy<'tcx>],
+ dest: &PlaceTy<'tcx>,
+ ret: Option<mir::BasicBlock>,
) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
- // All `#[rustc_do_not_const_check]` functions should be hooked here.
let def_id = instance.def_id();
if Some(def_id) == self.tcx.lang_items().panic_display()
@@ -173,20 +180,91 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> {
return Err(ConstEvalErrKind::Panic { msg, file, line, col }.into());
} else if Some(def_id) == self.tcx.lang_items().panic_fmt() {
// For panic_fmt, call const_panic_fmt instead.
- if let Some(const_panic_fmt) = self.tcx.lang_items().const_panic_fmt() {
- return Ok(Some(
- ty::Instance::resolve(
- *self.tcx,
- ty::ParamEnv::reveal_all(),
- const_panic_fmt,
- self.tcx.intern_substs(&[]),
- )
- .unwrap()
- .unwrap(),
- ));
+ let Some(const_def_id) = self.tcx.lang_items().const_panic_fmt() else {
+ bug!("`const_panic_fmt` must be defined to call `panic_fmt` in const eval")
+ };
+ let new_instance = ty::Instance::resolve(
+ *self.tcx,
+ ty::ParamEnv::reveal_all(),
+ const_def_id,
+ instance.substs,
+ )
+ .unwrap()
+ .unwrap();
+
+ return Ok(Some(new_instance));
+ } else if Some(def_id) == self.tcx.lang_items().align_offset_fn() {
+ // For align_offset, we replace the function call if the pointer has no address.
+ match self.align_offset(instance, args, dest, ret)? {
+ ControlFlow::Continue(()) => return Ok(Some(instance)),
+ ControlFlow::Break(()) => return Ok(None),
+ }
+ }
+ Ok(Some(instance))
+ }
+
+ /// `align_offset(ptr, target_align)` needs special handling in const eval, because the pointer
+ /// may not have an address.
+ ///
+ /// If `ptr` does have a known address, then we return `CONTINUE` and the function call should
+ /// proceed as normal.
+ ///
+ /// If `ptr` doesn't have an address, but its underlying allocation's alignment is at most
+ /// `target_align`, then we call the function again with an dummy address relative to the
+ /// allocation.
+ ///
+ /// If `ptr` doesn't have an address and `target_align` is stricter than the underlying
+ /// allocation's alignment, then we return `usize::MAX` immediately.
+ fn align_offset(
+ &mut self,
+ instance: ty::Instance<'tcx>,
+ args: &[OpTy<'tcx>],
+ dest: &PlaceTy<'tcx>,
+ ret: Option<mir::BasicBlock>,
+ ) -> InterpResult<'tcx, ControlFlow<()>> {
+ assert_eq!(args.len(), 2);
+
+ let ptr = self.read_pointer(&args[0])?;
+ let target_align = self.read_scalar(&args[1])?.to_machine_usize(self)?;
+
+ if !target_align.is_power_of_two() {
+ throw_ub_format!("`align_offset` called with non-power-of-two align: {}", target_align);
+ }
+
+ match self.ptr_try_get_alloc_id(ptr) {
+ Ok((alloc_id, offset, _extra)) => {
+ let (_size, alloc_align, _kind) = self.get_alloc_info(alloc_id);
+
+ if target_align <= alloc_align.bytes() {
+ // Extract the address relative to the allocation base that is definitely
+ // sufficiently aligned and call `align_offset` again.
+ let addr = ImmTy::from_uint(offset.bytes(), args[0].layout).into();
+ let align = ImmTy::from_uint(target_align, args[1].layout).into();
+
+ let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty())?;
+ self.eval_fn_call(
+ FnVal::Instance(instance),
+ (CallAbi::Rust, fn_abi),
+ &[addr, align],
+ false,
+ dest,
+ ret,
+ StackPopUnwind::NotAllowed,
+ )?;
+ Ok(ControlFlow::BREAK)
+ } else {
+ // Not alignable in const, return `usize::MAX`.
+ let usize_max = Scalar::from_machine_usize(self.machine_usize_max(), self);
+ self.write_scalar(usize_max, dest)?;
+ self.return_to_block(ret)?;
+ Ok(ControlFlow::BREAK)
+ }
+ }
+ Err(_addr) => {
+ // The pointer has an address, continue with function call.
+ Ok(ControlFlow::CONTINUE)
}
}
- Ok(None)
}
/// See documentation on the `ptr_guaranteed_cmp` intrinsic.
@@ -269,8 +347,8 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
instance: ty::Instance<'tcx>,
_abi: CallAbi,
args: &[OpTy<'tcx>],
- _dest: &PlaceTy<'tcx>,
- _ret: Option<mir::BasicBlock>,
+ dest: &PlaceTy<'tcx>,
+ ret: Option<mir::BasicBlock>,
_unwind: StackPopUnwind, // unwinding is not supported in consts
) -> InterpResult<'tcx, Option<(&'mir mir::Body<'tcx>, ty::Instance<'tcx>)>> {
debug!("find_mir_or_eval_fn: {:?}", instance);
@@ -289,7 +367,11 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
}
}
- if let Some(new_instance) = ecx.hook_special_const_fn(instance, args)? {
+ let Some(new_instance) = ecx.hook_special_const_fn(instance, _abi, args, dest, ret)? else {
+ return Ok(None);
+ };
+
+ if new_instance != instance {
// We call another const fn instead.
// However, we return the *original* instance to make backtraces work out
// (and we hope this does not confuse the FnAbi checks too much).
@@ -298,13 +380,14 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
new_instance,
_abi,
args,
- _dest,
- _ret,
+ dest,
+ ret,
_unwind,
)?
.map(|(body, _instance)| (body, instance)));
}
}
+
// This is a const fn. Call it.
Ok(Some((ecx.load_mir(instance.def, None)?, instance)))
}
diff --git a/library/core/src/ptr/const_ptr.rs b/library/core/src/ptr/const_ptr.rs
index 67e59460d74b0..29ba36c501637 100644
--- a/library/core/src/ptr/const_ptr.rs
+++ b/library/core/src/ptr/const_ptr.rs
@@ -1303,6 +1303,21 @@ impl<T: ?Sized> *const T {
/// ```
#[stable(feature = "align_offset", since = "1.36.0")]
#[rustc_const_unstable(feature = "const_align_offset", issue = "90962")]
+ #[cfg(not(bootstrap))]
+ pub const fn align_offset(self, align: usize) -> usize
+ where
+ T: Sized,
+ {
+ assert!(align.is_power_of_two(), "align_offset: align is not a power-of-two");
+
+ // SAFETY: `align` has been checked to be a power of 2 above
+ unsafe { align_offset(self, align) }
+ }
+
+ #[stable(feature = "align_offset", since = "1.36.0")]
+ #[rustc_const_unstable(feature = "const_align_offset", issue = "90962")]
+ #[allow(missing_docs)]
+ #[cfg(bootstrap)]
pub const fn align_offset(self, align: usize) -> usize
where
T: Sized,
diff --git a/library/core/src/ptr/mod.rs b/library/core/src/ptr/mod.rs
index 1f7cf6e5d052c..1e9add0067f8b 100644
--- a/library/core/src/ptr/mod.rs
+++ b/library/core/src/ptr/mod.rs
@@ -1554,10 +1554,14 @@ pub unsafe fn write_volatile<T>(dst: *mut T, src: T) {
/// Align pointer `p`.
///
-/// Calculate offset (in terms of elements of `stride` stride) that has to be applied
+/// Calculate offset (in terms of elements of `size_of::<T>()` stride) that has to be applied
/// to pointer `p` so that pointer `p` would get aligned to `a`.
///
-/// Note: This implementation has been carefully tailored to not panic. It is UB for this to panic.
+/// # Safety
+/// `a` must be a power of two.
+///
+/// # Notes
+/// This implementation has been carefully tailored to not panic. It is UB for this to panic.
/// The only real change that can be made here is change of `INV_TABLE_MOD_16` and associated
/// constants.
///
@@ -1566,8 +1570,10 @@ pub unsafe fn write_volatile<T>(dst: *mut T, src: T) {
/// than trying to adapt this to accommodate that change.
///
/// Any questions go to @nagisa.
+// #[cfg(not(bootstrap))] -- Calling this function in a const context from the bootstrap
+// compiler will always cause an error.
#[lang = "align_offset"]
-pub(crate) unsafe fn align_offset<T: Sized>(p: *const T, a: usize) -> usize {
+pub(crate) const unsafe fn align_offset<T: Sized>(p: *const T, a: usize) -> usize {
// FIXME(#75598): Direct use of these intrinsics improves codegen significantly at opt-level <=
// 1, where the method versions of these operations are not inlined.
use intrinsics::{
@@ -1584,7 +1590,7 @@ pub(crate) unsafe fn align_offset<T: Sized>(p: *const T, a: usize) -> usize {
///
/// Implementation of this function shall not panic. Ever.
#[inline]
- unsafe fn mod_inv(x: usize, m: usize) -> usize {
+ const unsafe fn mod_inv(x: usize, m: usize) -> usize {
/// Multiplicative modular inverse table modulo 2⁴ = 16.
///
/// Note, that this table does not contain values where inverse does not exist (i.e., for
@@ -1624,8 +1630,13 @@ pub(crate) unsafe fn align_offset<T: Sized>(p: *const T, a: usize) -> usize {
}
}
- let addr = p.addr();
let stride = mem::size_of::<T>();
+
+ // SAFETY: At runtime transmuting a pointer to `usize` is always safe, because they have the
+ // same layout. During const eval we hook this function to ensure that the pointer always has
+ // an address (only the standard library can do this).
+ let addr = unsafe { mem::transmute(p) };
+
// SAFETY: `a` is a power-of-two, therefore non-zero.
let a_minus_one = unsafe { unchecked_sub(a, 1) };
diff --git a/library/core/src/ptr/mut_ptr.rs b/library/core/src/ptr/mut_ptr.rs
index bbcc7c699e036..9990f738688a3 100644
--- a/library/core/src/ptr/mut_ptr.rs
+++ b/library/core/src/ptr/mut_ptr.rs
@@ -1574,6 +1574,21 @@ impl<T: ?Sized> *mut T {
/// ```
#[stable(feature = "align_offset", since = "1.36.0")]
#[rustc_const_unstable(feature = "const_align_offset", issue = "90962")]
+ #[cfg(not(bootstrap))]
+ pub const fn align_offset(self, align: usize) -> usize
+ where
+ T: Sized,
+ {
+ assert!(align.is_power_of_two(), "align_offset: align is not a power-of-two");
+
+ // SAFETY: `align` has been checked to be a power of 2 above
+ unsafe { align_offset(self, align) }
+ }
+
+ #[stable(feature = "align_offset", since = "1.36.0")]
+ #[rustc_const_unstable(feature = "const_align_offset", issue = "90962")]
+ #[allow(missing_docs)]
+ #[cfg(bootstrap)]
pub const fn align_offset(self, align: usize) -> usize
where
T: Sized,
From de33f9ac9f588665203de1cabc4b9d361a473ff7 Mon Sep 17 00:00:00 2001
From: Lukas Markeffsky <@>
Date: Fri, 7 Oct 2022 20:59:23 +0200
Subject: [PATCH 4/7] add coretests for const `align_offset`
---
library/core/tests/lib.rs | 1 +
library/core/tests/ptr.rs | 165 ++++++++++++++++++++++++++++++++++++++
2 files changed, 166 insertions(+)
diff --git a/library/core/tests/lib.rs b/library/core/tests/lib.rs
index b1f492381b136..f4b3ab8f89c7f 100644
--- a/library/core/tests/lib.rs
+++ b/library/core/tests/lib.rs
@@ -4,6 +4,7 @@
#![feature(array_windows)]
#![feature(bigint_helper_methods)]
#![feature(cell_update)]
+#![feature(const_align_offset)]
#![feature(const_assume)]
#![feature(const_black_box)]
#![feature(const_bool_to_option)]
diff --git a/library/core/tests/ptr.rs b/library/core/tests/ptr.rs
index 97a369810056d..81a66f5f71231 100644
--- a/library/core/tests/ptr.rs
+++ b/library/core/tests/ptr.rs
@@ -358,6 +358,23 @@ fn align_offset_zst() {
}
}
+#[test]
+#[cfg(not(bootstrap))]
+fn align_offset_zst_const() {
+ const {
+ // For pointers of stride = 0, the pointer is already aligned or it cannot be aligned at
+ // all, because no amount of elements will align the pointer.
+ let mut p = 1;
+ while p < 1024 {
+ assert!(ptr::invalid::<()>(p).align_offset(p) == 0);
+ if p != 1 {
+ assert!(ptr::invalid::<()>(p + 1).align_offset(p) == !0);
+ }
+ p = (p + 1).next_power_of_two();
+ }
+ }
+}
+
#[test]
fn align_offset_stride_one() {
// For pointers of stride = 1, the pointer can always be aligned. The offset is equal to
@@ -379,6 +396,26 @@ fn align_offset_stride_one() {
}
}
+#[test]
+#[cfg(not(bootstrap))]
+fn align_offset_stride_one_const() {
+ const {
+ // For pointers of stride = 1, the pointer can always be aligned. The offset is equal to
+ // number of bytes.
+ let mut align = 1;
+ while align < 1024 {
+ let mut ptr = 1;
+ while ptr < 2 * align {
+ let expected = ptr % align;
+ let offset = if expected == 0 { 0 } else { align - expected };
+ assert!(ptr::invalid::<u8>(ptr).align_offset(align) == offset);
+ ptr += 1;
+ }
+ align = (align + 1).next_power_of_two();
+ }
+ }
+}
+
#[test]
fn align_offset_various_strides() {
unsafe fn test_stride<T>(ptr: *const T, align: usize) -> bool {
@@ -455,6 +492,134 @@ fn align_offset_various_strides() {
assert!(!x);
}
+#[test]
+#[cfg(not(bootstrap))]
+fn align_offset_various_strides_const() {
+ const unsafe fn test_stride<T>(ptr: *const T, numptr: usize, align: usize) {
+ let mut expected = usize::MAX;
+ // Naive but definitely correct way to find the *first* aligned element of stride::<T>.
+ let mut el = 0;
+ while el < align {
+ if (numptr + el * ::std::mem::size_of::<T>()) % align == 0 {
+ expected = el;
+ break;
+ }
+ el += 1;
+ }
+ let got = ptr.align_offset(align);
+ assert!(got == expected);
+ }
+
+ // For pointers of stride != 1, we verify the algorithm against the naivest possible
+ // implementation
+ let mut align = 1;
+ let limit = 1024;
+ while align < limit {
+ for ptr in 1usize..4 * align {
+ unsafe {
+ #[repr(packed)]
+ struct A3(u16, u8);
+ test_stride::<A3>(ptr::invalid::<A3>(ptr), ptr, align);
+
+ struct A4(u32);
+ test_stride::<A4>(ptr::invalid::<A4>(ptr), ptr, align);
+
+ #[repr(packed)]
+ struct A5(u32, u8);
+ test_stride::<A5>(ptr::invalid::<A5>(ptr), ptr, align);
+
+ #[repr(packed)]
+ struct A6(u32, u16);
+ test_stride::<A6>(ptr::invalid::<A6>(ptr), ptr, align);
+
+ #[repr(packed)]
+ struct A7(u32, u16, u8);
+ test_stride::<A7>(ptr::invalid::<A7>(ptr), ptr, align);
+
+ #[repr(packed)]
+ struct A8(u32, u32);
+ test_stride::<A8>(ptr::invalid::<A8>(ptr), ptr, align);
+
+ #[repr(packed)]
+ struct A9(u32, u32, u8);
+ test_stride::<A9>(ptr::invalid::<A9>(ptr), ptr, align);
+
+ #[repr(packed)]
+ struct A10(u32, u32, u16);
+ test_stride::<A10>(ptr::invalid::<A10>(ptr), ptr, align);
+
+ test_stride::<u32>(ptr::invalid::<u32>(ptr), ptr, align);
+ test_stride::<u128>(ptr::invalid::<u128>(ptr), ptr, align);
+ }
+ }
+ align = (align + 1).next_power_of_two();
+ }
+}
+
+#[test]
+#[cfg(not(bootstrap))]
+fn align_offset_with_provenance_const() {
+ const {
+ let data = 42;
+
+ let ptr: *const i32 = &data;
+ assert!(ptr.align_offset(1) == 0);
+ assert!(ptr.align_offset(2) == 0);
+ assert!(ptr.align_offset(4) == 0);
+ assert!(ptr.align_offset(8) == usize::MAX);
+ assert!(ptr.wrapping_byte_add(1).align_offset(1) == 0);
+ assert!(ptr.wrapping_byte_add(1).align_offset(2) == usize::MAX);
+ assert!(ptr.wrapping_byte_add(2).align_offset(1) == 0);
+ assert!(ptr.wrapping_byte_add(2).align_offset(2) == 0);
+ assert!(ptr.wrapping_byte_add(2).align_offset(4) == usize::MAX);
+ assert!(ptr.wrapping_byte_add(3).align_offset(1) == 0);
+ assert!(ptr.wrapping_byte_add(3).align_offset(2) == usize::MAX);
+
+ assert!(ptr.wrapping_add(42).align_offset(4) == 0);
+ assert!(ptr.wrapping_add(42).align_offset(8) == usize::MAX);
+
+ let ptr1: *const i8 = ptr.cast();
+ assert!(ptr1.align_offset(1) == 0);
+ assert!(ptr1.align_offset(2) == 0);
+ assert!(ptr1.align_offset(4) == 0);
+ assert!(ptr1.align_offset(8) == usize::MAX);
+ assert!(ptr1.wrapping_byte_add(1).align_offset(1) == 0);
+ assert!(ptr1.wrapping_byte_add(1).align_offset(2) == 1);
+ assert!(ptr1.wrapping_byte_add(1).align_offset(4) == 3);
+ assert!(ptr1.wrapping_byte_add(1).align_offset(8) == usize::MAX);
+ assert!(ptr1.wrapping_byte_add(2).align_offset(1) == 0);
+ assert!(ptr1.wrapping_byte_add(2).align_offset(2) == 0);
+ assert!(ptr1.wrapping_byte_add(2).align_offset(4) == 2);
+ assert!(ptr1.wrapping_byte_add(2).align_offset(8) == usize::MAX);
+ assert!(ptr1.wrapping_byte_add(3).align_offset(1) == 0);
+ assert!(ptr1.wrapping_byte_add(3).align_offset(2) == 1);
+ assert!(ptr1.wrapping_byte_add(3).align_offset(4) == 1);
+ assert!(ptr1.wrapping_byte_add(3).align_offset(8) == usize::MAX);
+
+ let ptr2: *const i16 = ptr.cast();
+ assert!(ptr2.align_offset(1) == 0);
+ assert!(ptr2.align_offset(2) == 0);
+ assert!(ptr2.align_offset(4) == 0);
+ assert!(ptr2.align_offset(8) == usize::MAX);
+ assert!(ptr2.wrapping_byte_add(1).align_offset(1) == 0);
+ assert!(ptr2.wrapping_byte_add(1).align_offset(2) == usize::MAX);
+ assert!(ptr2.wrapping_byte_add(2).align_offset(1) == 0);
+ assert!(ptr2.wrapping_byte_add(2).align_offset(2) == 0);
+ assert!(ptr2.wrapping_byte_add(2).align_offset(4) == 1);
+ assert!(ptr2.wrapping_byte_add(2).align_offset(8) == usize::MAX);
+ assert!(ptr2.wrapping_byte_add(3).align_offset(1) == 0);
+ assert!(ptr2.wrapping_byte_add(3).align_offset(2) == usize::MAX);
+
+ let ptr3: *const i64 = ptr.cast();
+ assert!(ptr3.align_offset(1) == 0);
+ assert!(ptr3.align_offset(2) == 0);
+ assert!(ptr3.align_offset(4) == 0);
+ assert!(ptr3.align_offset(8) == usize::MAX);
+ assert!(ptr3.wrapping_byte_add(1).align_offset(1) == 0);
+ assert!(ptr3.wrapping_byte_add(1).align_offset(2) == usize::MAX);
+ }
+}
+
#[test]
fn offset_from() {
let mut a = [0; 5];
From 5d1c7a2524727291e30c4f8926f36074d0c6f3a1 Mon Sep 17 00:00:00 2001
From: Lukas Markeffsky <@>
Date: Fri, 7 Oct 2022 22:24:31 +0200
Subject: [PATCH 5/7] constify `pointer::is_aligned{,_to}`
---
library/core/src/lib.rs | 1 +
library/core/src/ptr/const_ptr.rs | 25 +++++++++++++++++++------
library/core/src/ptr/mut_ptr.rs | 25 +++++++++++++++++++------
3 files changed, 39 insertions(+), 12 deletions(-)
diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs
index 6294ef0df3dec..b6c1dd85d5792 100644
--- a/library/core/src/lib.rs
+++ b/library/core/src/lib.rs
@@ -129,6 +129,7 @@
#![feature(const_option)]
#![feature(const_option_ext)]
#![feature(const_pin)]
+#![feature(const_pointer_is_aligned)]
#![feature(const_ptr_sub_ptr)]
#![feature(const_replace)]
#![feature(const_ptr_as_ref)]
diff --git a/library/core/src/ptr/const_ptr.rs b/library/core/src/ptr/const_ptr.rs
index 29ba36c501637..2baa4e19ae937 100644
--- a/library/core/src/ptr/const_ptr.rs
+++ b/library/core/src/ptr/const_ptr.rs
@@ -1344,10 +1344,13 @@ impl<T: ?Sized> *const T {
}
/// Returns whether the pointer is properly aligned for `T`.
+ // #[cfg(not(bootstrap))] -- Calling this function in a const context from the bootstrap
+ // compiler will always return false.
#[must_use]
#[inline]
#[unstable(feature = "pointer_is_aligned", issue = "96284")]
- pub fn is_aligned(self) -> bool
+ #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "none")]
+ pub const fn is_aligned(self) -> bool
where
T: Sized,
{
@@ -1362,16 +1365,26 @@ impl<T: ?Sized> *const T {
/// # Panics
///
/// The function panics if `align` is not a power-of-two (this includes 0).
+ // #[cfg(not(bootstrap))] -- Calling this function in a const context from the bootstrap
+ // compiler will always return false.
#[must_use]
#[inline]
#[unstable(feature = "pointer_is_aligned", issue = "96284")]
- pub fn is_aligned_to(self, align: usize) -> bool {
- if !align.is_power_of_two() {
- panic!("is_aligned_to: align is not a power-of-two");
+ #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "none")]
+ pub const fn is_aligned_to(self, align: usize) -> bool {
+ assert!(align.is_power_of_two(), "is_aligned_to: align is not a power-of-two");
+
+ #[inline]
+ fn runtime(ptr: *const u8, align: usize) -> bool {
+ ptr.addr() & (align - 1) == 0
+ }
+
+ const fn comptime(ptr: *const u8, align: usize) -> bool {
+ ptr.align_offset(align) == 0
}
- // Cast is needed for `T: !Sized`
- self.cast::<u8>().addr() & align - 1 == 0
+ // SAFETY: `ptr.align_offset(align)` returns 0 if and only if the pointer is already aligned.
+ unsafe { intrinsics::const_eval_select((self.cast::<u8>(), align), comptime, runtime) }
}
}
diff --git a/library/core/src/ptr/mut_ptr.rs b/library/core/src/ptr/mut_ptr.rs
index 9990f738688a3..25437dd06ae75 100644
--- a/library/core/src/ptr/mut_ptr.rs
+++ b/library/core/src/ptr/mut_ptr.rs
@@ -1615,10 +1615,13 @@ impl<T: ?Sized> *mut T {
}
/// Returns whether the pointer is properly aligned for `T`.
+ // #[cfg(not(bootstrap))] -- Calling this function in a const context from the bootstrap
+ // compiler will always return false.
#[must_use]
#[inline]
#[unstable(feature = "pointer_is_aligned", issue = "96284")]
- pub fn is_aligned(self) -> bool
+ #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "none")]
+ pub const fn is_aligned(self) -> bool
where
T: Sized,
{
@@ -1633,16 +1636,26 @@ impl<T: ?Sized> *mut T {
/// # Panics
///
/// The function panics if `align` is not a power-of-two (this includes 0).
+ // #[cfg(not(bootstrap))] -- Calling this function in a const context from the bootstrap
+ // compiler will always return false.
#[must_use]
#[inline]
#[unstable(feature = "pointer_is_aligned", issue = "96284")]
- pub fn is_aligned_to(self, align: usize) -> bool {
- if !align.is_power_of_two() {
- panic!("is_aligned_to: align is not a power-of-two");
+ #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "none")]
+ pub const fn is_aligned_to(self, align: usize) -> bool {
+ assert!(align.is_power_of_two(), "is_aligned_to: align is not a power-of-two");
+
+ #[inline]
+ fn runtime(ptr: *mut u8, align: usize) -> bool {
+ ptr.addr() & (align - 1) == 0
+ }
+
+ const fn comptime(ptr: *mut u8, align: usize) -> bool {
+ ptr.align_offset(align) == 0
}
- // Cast is needed for `T: !Sized`
- self.cast::<u8>().addr() & align - 1 == 0
+ // SAFETY: `ptr.align_offset(align)` returns 0 if and only if the pointer is already aligned.
+ unsafe { intrinsics::const_eval_select((self.cast::<u8>(), align), comptime, runtime) }
}
}
From 5371cc4d4c657b783a87c9b47231c425a6ff8845 Mon Sep 17 00:00:00 2001
From: Lukas Markeffsky <@>
Date: Fri, 7 Oct 2022 21:46:28 +0200
Subject: [PATCH 6/7] add coretests for `is_aligned`
---
library/core/tests/lib.rs | 2 ++
library/core/tests/ptr.rs | 48 +++++++++++++++++++++++++++++++++++++++
2 files changed, 50 insertions(+)
diff --git a/library/core/tests/lib.rs b/library/core/tests/lib.rs
index f4b3ab8f89c7f..9704650edf0f5 100644
--- a/library/core/tests/lib.rs
+++ b/library/core/tests/lib.rs
@@ -17,6 +17,7 @@
#![feature(const_nonnull_new)]
#![feature(const_num_from_num)]
#![feature(const_pointer_byte_offsets)]
+#![feature(const_pointer_is_aligned)]
#![feature(const_ptr_as_ref)]
#![feature(const_ptr_read)]
#![feature(const_ptr_write)]
@@ -80,6 +81,7 @@
#![feature(never_type)]
#![feature(unwrap_infallible)]
#![feature(pointer_byte_offsets)]
+#![feature(pointer_is_aligned)]
#![feature(portable_simd)]
#![feature(ptr_metadata)]
#![feature(once_cell)]
diff --git a/library/core/tests/ptr.rs b/library/core/tests/ptr.rs
index 81a66f5f71231..a1c23db2932a4 100644
--- a/library/core/tests/ptr.rs
+++ b/library/core/tests/ptr.rs
@@ -620,6 +620,54 @@ fn align_offset_with_provenance_const() {
}
}
+#[test]
+fn is_aligned() {
+ let data = 42;
+ let ptr: *const i32 = &data;
+ assert!(ptr.is_aligned());
+ assert!(ptr.is_aligned_to(1));
+ assert!(ptr.is_aligned_to(2));
+ assert!(ptr.is_aligned_to(4));
+ assert!(ptr.wrapping_byte_add(2).is_aligned_to(1));
+ assert!(ptr.wrapping_byte_add(2).is_aligned_to(2));
+ assert!(!ptr.wrapping_byte_add(2).is_aligned_to(4));
+
+ // At runtime either `ptr` or `ptr+1` is aligned to 8.
+ assert_ne!(ptr.is_aligned_to(8), ptr.wrapping_add(1).is_aligned_to(8));
+}
+
+#[test]
+#[cfg(not(bootstrap))]
+fn is_aligned_const() {
+ const {
+ let data = 42;
+ let ptr: *const i32 = &data;
+ assert!(ptr.is_aligned());
+ assert!(ptr.is_aligned_to(1));
+ assert!(ptr.is_aligned_to(2));
+ assert!(ptr.is_aligned_to(4));
+ assert!(ptr.wrapping_byte_add(2).is_aligned_to(1));
+ assert!(ptr.wrapping_byte_add(2).is_aligned_to(2));
+ assert!(!ptr.wrapping_byte_add(2).is_aligned_to(4));
+
+ // At comptime neither `ptr` nor `ptr+1` is aligned to 8.
+ assert!(!ptr.is_aligned_to(8));
+ assert!(!ptr.wrapping_add(1).is_aligned_to(8));
+ }
+}
+
+#[test]
+#[cfg(bootstrap)]
+fn is_aligned_const() {
+ const {
+ let data = 42;
+ let ptr: *const i32 = &data;
+ // The bootstrap compiler always returns false for is_aligned.
+ assert!(!ptr.is_aligned());
+ assert!(!ptr.is_aligned_to(1));
+ }
+}
+
#[test]
fn offset_from() {
let mut a = [0; 5];
From c2605f6711796bd2101bfa4f138cf973d5336136 Mon Sep 17 00:00:00 2001
From: Lukas Markeffsky <@>
Date: Thu, 20 Oct 2022 19:46:31 +0200
Subject: [PATCH 7/7] mark `align_offset` as `#[must_use]`
---
library/core/src/ptr/const_ptr.rs | 1 +
library/core/src/ptr/mut_ptr.rs | 1 +
2 files changed, 2 insertions(+)
diff --git a/library/core/src/ptr/const_ptr.rs b/library/core/src/ptr/const_ptr.rs
index 2baa4e19ae937..41e62cc769e3b 100644
--- a/library/core/src/ptr/const_ptr.rs
+++ b/library/core/src/ptr/const_ptr.rs
@@ -1301,6 +1301,7 @@ impl<T: ?Sized> *const T {
/// }
/// # }
/// ```
+ #[must_use]
#[stable(feature = "align_offset", since = "1.36.0")]
#[rustc_const_unstable(feature = "const_align_offset", issue = "90962")]
#[cfg(not(bootstrap))]
diff --git a/library/core/src/ptr/mut_ptr.rs b/library/core/src/ptr/mut_ptr.rs
index 25437dd06ae75..381652b9a3635 100644
--- a/library/core/src/ptr/mut_ptr.rs
+++ b/library/core/src/ptr/mut_ptr.rs
@@ -1572,6 +1572,7 @@ impl<T: ?Sized> *mut T {
/// }
/// # }
/// ```
+ #[must_use]
#[stable(feature = "align_offset", since = "1.36.0")]
#[rustc_const_unstable(feature = "const_align_offset", issue = "90962")]
#[cfg(not(bootstrap))]