Skip to content

Commit 90d9374

Browse files
scottmcmscottmcm
committed
Move {widening, carrying}_mul to an intrinsic with fallback MIR
Including implementing it for `u128`, so it can be defined in `uint_impl!`. This way it works for all backends, including CTFE.
1 parent ca13e91 commit 90d9374

File tree

9 files changed

+322
-135
lines changed

9 files changed

+322
-135
lines changed

‎compiler/rustc_hir_analysis/src/check/intrinsic.rs

+5
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -
9595
| sym::add_with_overflow
9696
| sym::sub_with_overflow
9797
| sym::mul_with_overflow
98+
| sym::carrying_mul_add
9899
| sym::wrapping_add
99100
| sym::wrapping_sub
100101
| sym::wrapping_mul
@@ -437,6 +438,10 @@ pub fn check_intrinsic_type(
437438
(1, 0, vec![param(0), param(0)], Ty::new_tup(tcx, &[param(0), tcx.types.bool]))
438439
}
439440

441+
sym::carrying_mul_add => {
442+
(2, 0, vec![param(0); 4], Ty::new_tup(tcx, &[param(1), param(0)]))
443+
}
444+
440445
sym::ptr_guaranteed_cmp => (
441446
1,
442447
0,

‎compiler/rustc_span/src/symbol.rs

+1
Original file line numberDiff line numberDiff line change
@@ -538,6 +538,7 @@ symbols! {
538538
call_ref_future,
539539
caller_location,
540540
capture_disjoint_fields,
541+
carrying_mul_add,
541542
catch_unwind,
542543
cause,
543544
cdylib,

‎library/core/src/intrinsics/fallback.rs

+111
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
#![unstable(
2+
feature = "core_intrinsics_fallbacks",
3+
reason = "The fallbacks will never be stable, as they exist only to be called \
4+
by the fallback MIR, but they're exported so they can be tested on \
5+
platforms where the fallback MIR isn't actually used",
6+
issue = "none"
7+
)]
8+
#![allow(missing_docs)]
9+
10+
#[const_trait]
11+
pub trait CarryingMulAdd: Copy + 'static {
12+
type Unsigned: Copy + 'static;
13+
fn carrying_mul_add(
14+
self,
15+
multiplicand: Self,
16+
addend: Self,
17+
carry: Self,
18+
) -> (Self::Unsigned, Self);
19+
}
20+
21+
macro_rules! impl_carrying_mul_add_by_widening {
22+
($($t:ident $u:ident $w:ident,)+) => {$(
23+
#[rustc_const_unstable(feature = "core_intrinsics_fallbacks", issue = "none")]
24+
impl const CarryingMulAdd for $t {
25+
type Unsigned = $u;
26+
#[inline]
27+
fn carrying_mul_add(self, a: Self, b: Self, c: Self) -> ($u, $t) {
28+
let wide = (self as $w) * (a as $w) + (b as $w) + (c as $w);
29+
(wide as _, (wide >> Self::BITS) as _)
30+
}
31+
}
32+
)+};
33+
}
34+
impl_carrying_mul_add_by_widening! {
35+
u8 u8 u16,
36+
u16 u16 u32,
37+
u32 u32 u64,
38+
u64 u64 u128,
39+
usize usize UDoubleSize,
40+
i8 u8 i16,
41+
i16 u16 i32,
42+
i32 u32 i64,
43+
i64 u64 i128,
44+
isize usize UDoubleSize,
45+
}
46+
47+
#[cfg(target_pointer_width = "16")]
48+
type UDoubleSize = u32;
49+
#[cfg(target_pointer_width = "32")]
50+
type UDoubleSize = u64;
51+
#[cfg(target_pointer_width = "64")]
52+
type UDoubleSize = u128;
53+
54+
#[inline]
55+
const fn wide_mul_u128(a: u128, b: u128) -> (u128, u128) {
56+
#[inline]
57+
const fn to_low_high(x: u128) -> [u128; 2] {
58+
const MASK: u128 = u64::MAX as _;
59+
[x & MASK, x >> 64]
60+
}
61+
#[inline]
62+
const fn from_low_high(x: [u128; 2]) -> u128 {
63+
x[0] | (x[1] << 64)
64+
}
65+
#[inline]
66+
const fn scalar_mul(low_high: [u128; 2], k: u128) -> [u128; 3] {
67+
let [x, c] = to_low_high(k * low_high[0]);
68+
let [y, z] = to_low_high(k * low_high[1] + c);
69+
[x, y, z]
70+
}
71+
let a = to_low_high(a);
72+
let b = to_low_high(b);
73+
let low = scalar_mul(a, b[0]);
74+
let high = scalar_mul(a, b[1]);
75+
let r0 = low[0];
76+
let [r1, c] = to_low_high(low[1] + high[0]);
77+
let [r2, c] = to_low_high(low[2] + high[1] + c);
78+
let r3 = high[2] + c;
79+
(from_low_high([r0, r1]), from_low_high([r2, r3]))
80+
}
81+
82+
#[rustc_const_unstable(feature = "core_intrinsics_fallbacks", issue = "none")]
83+
impl const CarryingMulAdd for u128 {
84+
type Unsigned = u128;
85+
#[inline]
86+
fn carrying_mul_add(self, b: u128, c: u128, d: u128) -> (u128, u128) {
87+
let (low, mut high) = wide_mul_u128(self, b);
88+
let (low, carry) = u128::overflowing_add(low, c);
89+
high += carry as u128;
90+
let (low, carry) = u128::overflowing_add(low, d);
91+
high += carry as u128;
92+
(low, high)
93+
}
94+
}
95+
96+
#[rustc_const_unstable(feature = "core_intrinsics_fallbacks", issue = "none")]
97+
impl const CarryingMulAdd for i128 {
98+
type Unsigned = u128;
99+
#[inline]
100+
fn carrying_mul_add(self, b: i128, c: i128, d: i128) -> (u128, i128) {
101+
let (low, high) = wide_mul_u128(self as u128, b as u128);
102+
let mut high = high as i128;
103+
high = high.wrapping_add((self >> 127) * b);
104+
high = high.wrapping_add(self * (b >> 127));
105+
let (low, carry) = u128::overflowing_add(low, c as u128);
106+
high = high.wrapping_add((carry as i128) + (c >> 127));
107+
let (low, carry) = u128::overflowing_add(low, d as u128);
108+
high = high.wrapping_add((carry as i128) + (d >> 127));
109+
(low, high)
110+
}
111+
}

‎library/core/src/intrinsics/mod.rs

+29
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ use crate::marker::{DiscriminantKind, Tuple};
6868
use crate::mem::SizedTypeProperties;
6969
use crate::{ptr, ub_checks};
7070

71+
pub mod fallback;
7172
pub mod mir;
7273
pub mod simd;
7374

@@ -3305,6 +3306,34 @@ pub const fn mul_with_overflow<T: Copy>(_x: T, _y: T) -> (T, bool) {
33053306
unimplemented!()
33063307
}
33073308

3309+
/// Performs full-width multiplication and addition with a carry:
3310+
/// `multiplier * multiplicand + addend + carry`.
3311+
///
3312+
/// This is possible without any overflow. For `uN`:
3313+
/// MAX * MAX + MAX + MAX
3314+
/// => (2ⁿ-1) × (2ⁿ-1) + (2ⁿ-1) + (2ⁿ-1)
3315+
/// => (2²ⁿ - 2ⁿ⁺¹ + 1) + (2ⁿ⁺¹ - 2)
3316+
/// => 2²ⁿ - 1
3317+
///
3318+
/// For `iN`, the upper bound is MIN * MIN + MAX + MAX => 2²ⁿ⁻² + 2ⁿ - 2,
3319+
/// and the lower bound is MAX * MIN + MIN + MIN => -2²ⁿ⁻² - 2ⁿ + 2ⁿ⁺¹.
3320+
///
3321+
/// This currently supports unsigned integers *only*, no signed ones.
3322+
/// The stabilized versions of this intrinsic are available on integers.
3323+
#[unstable(feature = "core_intrinsics", issue = "none")]
3324+
#[rustc_const_unstable(feature = "const_carrying_mul_add", issue = "85532")]
3325+
#[rustc_nounwind]
3326+
#[cfg_attr(not(bootstrap), rustc_intrinsic)]
3327+
#[cfg_attr(not(bootstrap), miri::intrinsic_fallback_is_spec)]
3328+
pub const fn carrying_mul_add<T: ~const fallback::CarryingMulAdd<Unsigned = U>, U>(
3329+
multiplier: T,
3330+
multiplicand: T,
3331+
addend: T,
3332+
carry: T,
3333+
) -> (U, T) {
3334+
multiplier.carrying_mul_add(multiplicand, addend, carry)
3335+
}
3336+
33083337
/// Performs an exact division, resulting in undefined behavior where
33093338
/// `x % y != 0` or `y == 0` or `x == T::MIN && y == -1`
33103339
///

‎library/core/src/lib.rs

+1
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@
109109
// tidy-alphabetical-start
110110
#![feature(array_ptr_get)]
111111
#![feature(asm_experimental_arch)]
112+
#![feature(const_carrying_mul_add)]
112113
#![feature(const_eval_select)]
113114
#![feature(const_typed_swap)]
114115
#![feature(core_intrinsics)]

‎library/core/src/num/mod.rs

-135
Original file line numberDiff line numberDiff line change
@@ -203,134 +203,6 @@ macro_rules! midpoint_impl {
203203
};
204204
}
205205

206-
macro_rules! widening_impl {
207-
($SelfT:ty, $WideT:ty, $BITS:literal, unsigned) => {
208-
/// Calculates the complete product `self * rhs` without the possibility to overflow.
209-
///
210-
/// This returns the low-order (wrapping) bits and the high-order (overflow) bits
211-
/// of the result as two separate values, in that order.
212-
///
213-
/// If you also need to add a carry to the wide result, then you want
214-
/// [`Self::carrying_mul`] instead.
215-
///
216-
/// # Examples
217-
///
218-
/// Basic usage:
219-
///
220-
/// Please note that this example is shared between integer types.
221-
/// Which explains why `u32` is used here.
222-
///
223-
/// ```
224-
/// #![feature(bigint_helper_methods)]
225-
/// assert_eq!(5u32.widening_mul(2), (10, 0));
226-
/// assert_eq!(1_000_000_000u32.widening_mul(10), (1410065408, 2));
227-
/// ```
228-
#[unstable(feature = "bigint_helper_methods", issue = "85532")]
229-
#[must_use = "this returns the result of the operation, \
230-
without modifying the original"]
231-
#[inline]
232-
pub const fn widening_mul(self, rhs: Self) -> (Self, Self) {
233-
// note: longer-term this should be done via an intrinsic,
234-
// but for now we can deal without an impl for u128/i128
235-
// SAFETY: overflow will be contained within the wider types
236-
let wide = unsafe { (self as $WideT).unchecked_mul(rhs as $WideT) };
237-
(wide as $SelfT, (wide >> $BITS) as $SelfT)
238-
}
239-
240-
/// Calculates the "full multiplication" `self * rhs + carry`
241-
/// without the possibility to overflow.
242-
///
243-
/// This returns the low-order (wrapping) bits and the high-order (overflow) bits
244-
/// of the result as two separate values, in that order.
245-
///
246-
/// Performs "long multiplication" which takes in an extra amount to add, and may return an
247-
/// additional amount of overflow. This allows for chaining together multiple
248-
/// multiplications to create "big integers" which represent larger values.
249-
///
250-
/// If you don't need the `carry`, then you can use [`Self::widening_mul`] instead.
251-
///
252-
/// # Examples
253-
///
254-
/// Basic usage:
255-
///
256-
/// Please note that this example is shared between integer types.
257-
/// Which explains why `u32` is used here.
258-
///
259-
/// ```
260-
/// #![feature(bigint_helper_methods)]
261-
/// assert_eq!(5u32.carrying_mul(2, 0), (10, 0));
262-
/// assert_eq!(5u32.carrying_mul(2, 10), (20, 0));
263-
/// assert_eq!(1_000_000_000u32.carrying_mul(10, 0), (1410065408, 2));
264-
/// assert_eq!(1_000_000_000u32.carrying_mul(10, 10), (1410065418, 2));
265-
#[doc = concat!("assert_eq!(",
266-
stringify!($SelfT), "::MAX.carrying_mul(", stringify!($SelfT), "::MAX, ", stringify!($SelfT), "::MAX), ",
267-
"(0, ", stringify!($SelfT), "::MAX));"
268-
)]
269-
/// ```
270-
///
271-
/// This is the core operation needed for scalar multiplication when
272-
/// implementing it for wider-than-native types.
273-
///
274-
/// ```
275-
/// #![feature(bigint_helper_methods)]
276-
/// fn scalar_mul_eq(little_endian_digits: &mut Vec<u16>, multiplicand: u16) {
277-
/// let mut carry = 0;
278-
/// for d in little_endian_digits.iter_mut() {
279-
/// (*d, carry) = d.carrying_mul(multiplicand, carry);
280-
/// }
281-
/// if carry != 0 {
282-
/// little_endian_digits.push(carry);
283-
/// }
284-
/// }
285-
///
286-
/// let mut v = vec![10, 20];
287-
/// scalar_mul_eq(&mut v, 3);
288-
/// assert_eq!(v, [30, 60]);
289-
///
290-
/// assert_eq!(0x87654321_u64 * 0xFEED, 0x86D3D159E38D);
291-
/// let mut v = vec![0x4321, 0x8765];
292-
/// scalar_mul_eq(&mut v, 0xFEED);
293-
/// assert_eq!(v, [0xE38D, 0xD159, 0x86D3]);
294-
/// ```
295-
///
296-
/// If `carry` is zero, this is similar to [`overflowing_mul`](Self::overflowing_mul),
297-
/// except that it gives the value of the overflow instead of just whether one happened:
298-
///
299-
/// ```
300-
/// #![feature(bigint_helper_methods)]
301-
/// let r = u8::carrying_mul(7, 13, 0);
302-
/// assert_eq!((r.0, r.1 != 0), u8::overflowing_mul(7, 13));
303-
/// let r = u8::carrying_mul(13, 42, 0);
304-
/// assert_eq!((r.0, r.1 != 0), u8::overflowing_mul(13, 42));
305-
/// ```
306-
///
307-
/// The value of the first field in the returned tuple matches what you'd get
308-
/// by combining the [`wrapping_mul`](Self::wrapping_mul) and
309-
/// [`wrapping_add`](Self::wrapping_add) methods:
310-
///
311-
/// ```
312-
/// #![feature(bigint_helper_methods)]
313-
/// assert_eq!(
314-
/// 789_u16.carrying_mul(456, 123).0,
315-
/// 789_u16.wrapping_mul(456).wrapping_add(123),
316-
/// );
317-
/// ```
318-
#[unstable(feature = "bigint_helper_methods", issue = "85532")]
319-
#[must_use = "this returns the result of the operation, \
320-
without modifying the original"]
321-
#[inline]
322-
pub const fn carrying_mul(self, rhs: Self, carry: Self) -> (Self, Self) {
323-
// note: longer-term this should be done via an intrinsic,
324-
// but for now we can deal without an impl for u128/i128
325-
// SAFETY: overflow will be contained within the wider types
326-
let wide = unsafe {
327-
(self as $WideT).unchecked_mul(rhs as $WideT).unchecked_add(carry as $WideT)
328-
};
329-
(wide as $SelfT, (wide >> $BITS) as $SelfT)
330-
}
331-
};
332-
}
333-
334206
impl i8 {
335207
int_impl! {
336208
Self = i8,
@@ -551,7 +423,6 @@ impl u8 {
551423
from_xe_bytes_doc = "",
552424
bound_condition = "",
553425
}
554-
widening_impl! { u8, u16, 8, unsigned }
555426
midpoint_impl! { u8, u16, unsigned }
556427

557428
/// Checks if the value is within the ASCII range.
@@ -1167,7 +1038,6 @@ impl u16 {
11671038
from_xe_bytes_doc = "",
11681039
bound_condition = "",
11691040
}
1170-
widening_impl! { u16, u32, 16, unsigned }
11711041
midpoint_impl! { u16, u32, unsigned }
11721042

11731043
/// Checks if the value is a Unicode surrogate code point, which are disallowed values for [`char`].
@@ -1215,7 +1085,6 @@ impl u32 {
12151085
from_xe_bytes_doc = "",
12161086
bound_condition = "",
12171087
}
1218-
widening_impl! { u32, u64, 32, unsigned }
12191088
midpoint_impl! { u32, u64, unsigned }
12201089
}
12211090

@@ -1239,7 +1108,6 @@ impl u64 {
12391108
from_xe_bytes_doc = "",
12401109
bound_condition = "",
12411110
}
1242-
widening_impl! { u64, u128, 64, unsigned }
12431111
midpoint_impl! { u64, u128, unsigned }
12441112
}
12451113

@@ -1289,7 +1157,6 @@ impl usize {
12891157
from_xe_bytes_doc = usize_isize_from_xe_bytes_doc!(),
12901158
bound_condition = " on 16-bit targets",
12911159
}
1292-
widening_impl! { usize, u32, 16, unsigned }
12931160
midpoint_impl! { usize, u32, unsigned }
12941161
}
12951162

@@ -1314,7 +1181,6 @@ impl usize {
13141181
from_xe_bytes_doc = usize_isize_from_xe_bytes_doc!(),
13151182
bound_condition = " on 32-bit targets",
13161183
}
1317-
widening_impl! { usize, u64, 32, unsigned }
13181184
midpoint_impl! { usize, u64, unsigned }
13191185
}
13201186

@@ -1339,7 +1205,6 @@ impl usize {
13391205
from_xe_bytes_doc = usize_isize_from_xe_bytes_doc!(),
13401206
bound_condition = " on 64-bit targets",
13411207
}
1342-
widening_impl! { usize, u128, 64, unsigned }
13431208
midpoint_impl! { usize, u128, unsigned }
13441209
}
13451210

0 commit comments

Comments
 (0)