From 744c664ba2e6440024457d5ec0d3600b3e0c0144 Mon Sep 17 00:00:00 2001
From: Scott McMurray <scottmcm@users.noreply.github.com>
Date: Sun, 26 Nov 2023 00:36:56 -0800
Subject: [PATCH 1/4] Add a MIR pre-codegen test for derived PartialOrd
---
tests/mir-opt/pre-codegen/derived_ord.rs | 9 +
....{impl#0}-partial_cmp.PreCodegen.after.mir | 159 ++++++++++++++++++
2 files changed, 168 insertions(+)
create mode 100644 tests/mir-opt/pre-codegen/derived_ord.rs
create mode 100644 tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir
diff --git a/tests/mir-opt/pre-codegen/derived_ord.rs b/tests/mir-opt/pre-codegen/derived_ord.rs
new file mode 100644
index 0000000000000..bad751edf8419
--- /dev/null
+++ b/tests/mir-opt/pre-codegen/derived_ord.rs
@@ -0,0 +1,9 @@
+// skip-filecheck
+//@ compile-flags: -O -Zmir-opt-level=2 -Cdebuginfo=0
+
+#![crate_type = "lib"]
+
+#[derive(PartialOrd, PartialEq)]
+pub struct MultiField(char, i16);
+
+// EMIT_MIR derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir
diff --git a/tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir
new file mode 100644
index 0000000000000..dd7fce3ed0b46
--- /dev/null
+++ b/tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir
@@ -0,0 +1,159 @@
+// MIR for `<impl at $DIR/derived_ord.rs:6:10: 6:20>::partial_cmp` after PreCodegen
+
+fn <impl at $DIR/derived_ord.rs:6:10: 6:20>::partial_cmp(_1: &MultiField, _2: &MultiField) -> Option<std::cmp::Ordering> {
+ debug self => _1;
+ debug other => _2;
+ let mut _0: std::option::Option<std::cmp::Ordering>;
+ let mut _3: &char;
+ let mut _4: &char;
+ let mut _10: std::option::Option<std::cmp::Ordering>;
+ let mut _11: &i16;
+ let mut _12: &i16;
+ let _18: std::option::Option<std::cmp::Ordering>;
+ scope 1 {
+ debug cmp => _18;
+ }
+ scope 2 (inlined std::cmp::impls::<impl PartialOrd for char>::partial_cmp) {
+ debug self => _3;
+ debug other => _4;
+ let mut _9: std::cmp::Ordering;
+ scope 3 (inlined std::cmp::impls::<impl Ord for char>::cmp) {
+ debug self => _3;
+ debug other => _4;
+ let mut _5: char;
+ let mut _6: char;
+ let mut _7: bool;
+ let mut _8: bool;
+ }
+ }
+ scope 4 (inlined std::cmp::impls::<impl PartialOrd for i16>::partial_cmp) {
+ debug self => _11;
+ debug other => _12;
+ let mut _17: std::cmp::Ordering;
+ scope 5 (inlined std::cmp::impls::<impl Ord for i16>::cmp) {
+ debug self => _11;
+ debug other => _12;
+ let mut _13: i16;
+ let mut _14: i16;
+ let mut _15: bool;
+ let mut _16: bool;
+ }
+ }
+
+ bb0: {
+ StorageLive(_3);
+ _3 = &((*_1).0: char);
+ StorageLive(_4);
+ _4 = &((*_2).0: char);
+ StorageLive(_9);
+ StorageLive(_5);
+ StorageLive(_6);
+ StorageLive(_7);
+ _5 = ((*_1).0: char);
+ _6 = ((*_2).0: char);
+ _7 = Lt(_5, _6);
+ switchInt(move _7) -> [0: bb1, otherwise: bb10];
+ }
+
+ bb1: {
+ StorageLive(_8);
+ _8 = Eq(_5, _6);
+ switchInt(move _8) -> [0: bb2, otherwise: bb3];
+ }
+
+ bb2: {
+ _9 = const Greater;
+ StorageDead(_8);
+ StorageDead(_7);
+ StorageDead(_6);
+ StorageDead(_5);
+ _10 = Option::<std::cmp::Ordering>::Some(move _9);
+ StorageDead(_9);
+ StorageDead(_4);
+ StorageDead(_3);
+ goto -> bb11;
+ }
+
+ bb3: {
+ StorageDead(_8);
+ StorageDead(_7);
+ StorageDead(_6);
+ StorageDead(_5);
+ StorageDead(_9);
+ StorageDead(_4);
+ StorageDead(_3);
+ StorageLive(_11);
+ _11 = &((*_1).1: i16);
+ StorageLive(_12);
+ _12 = &((*_2).1: i16);
+ StorageLive(_17);
+ StorageLive(_13);
+ StorageLive(_14);
+ StorageLive(_15);
+ _13 = ((*_1).1: i16);
+ _14 = ((*_2).1: i16);
+ _15 = Lt(_13, _14);
+ switchInt(move _15) -> [0: bb4, otherwise: bb8];
+ }
+
+ bb4: {
+ StorageLive(_16);
+ _16 = Eq(_13, _14);
+ switchInt(move _16) -> [0: bb5, otherwise: bb6];
+ }
+
+ bb5: {
+ _17 = const Greater;
+ goto -> bb7;
+ }
+
+ bb6: {
+ _17 = const Equal;
+ goto -> bb7;
+ }
+
+ bb7: {
+ StorageDead(_16);
+ goto -> bb9;
+ }
+
+ bb8: {
+ _17 = const Less;
+ goto -> bb9;
+ }
+
+ bb9: {
+ StorageDead(_15);
+ StorageDead(_14);
+ StorageDead(_13);
+ _0 = Option::<std::cmp::Ordering>::Some(move _17);
+ StorageDead(_17);
+ StorageDead(_12);
+ StorageDead(_11);
+ goto -> bb12;
+ }
+
+ bb10: {
+ _9 = const Less;
+ StorageDead(_7);
+ StorageDead(_6);
+ StorageDead(_5);
+ _10 = Option::<std::cmp::Ordering>::Some(move _9);
+ StorageDead(_9);
+ StorageDead(_4);
+ StorageDead(_3);
+ goto -> bb11;
+ }
+
+ bb11: {
+ StorageLive(_18);
+ _18 = _10;
+ _0 = _10;
+ StorageDead(_18);
+ goto -> bb12;
+ }
+
+ bb12: {
+ return;
+ }
+}
From 3da115a93b782796e3d267266b4241e5258f1fef Mon Sep 17 00:00:00 2001
From: Scott McMurray <scottmcm@users.noreply.github.com>
Date: Sun, 5 Mar 2023 20:19:41 -0800
Subject: [PATCH 2/4] Add+Use `mir::BinOp::Cmp`
---
.../src/codegen_i128.rs | 3 +-
compiler/rustc_codegen_cranelift/src/num.rs | 24 ++-
compiler/rustc_codegen_gcc/src/common.rs | 4 +
compiler/rustc_codegen_llvm/src/common.rs | 4 +
compiler/rustc_codegen_ssa/src/mir/rvalue.rs | 30 ++++
.../rustc_codegen_ssa/src/traits/consts.rs | 1 +
.../rustc_const_eval/src/interpret/operand.rs | 7 +
.../src/interpret/operator.rs | 18 +++
.../src/transform/validate.rs | 9 ++
compiler/rustc_const_eval/src/util/mod.rs | 4 +-
compiler/rustc_hir/src/lang_items.rs | 1 +
.../rustc_hir_analysis/src/check/intrinsic.rs | 5 +
compiler/rustc_middle/src/mir/syntax.rs | 2 +
compiler/rustc_middle/src/mir/tcx.rs | 8 +-
compiler/rustc_middle/src/ty/context.rs | 7 +
.../src/lower_intrinsics.rs | 2 +
.../rustc_mir_transform/src/promote_consts.rs | 1 +
.../rustc_smir/src/rustc_smir/convert/mir.rs | 1 +
compiler/rustc_span/src/symbol.rs | 1 +
compiler/rustc_ty_utils/src/consts.rs | 2 +-
compiler/stable_mir/src/mir/body.rs | 4 +
library/core/src/cmp.rs | 8 +
library/core/src/intrinsics.rs | 12 ++
library/core/tests/intrinsics.rs | 19 +++
library/core/tests/lib.rs | 1 +
tests/assembly/x86_64-cmp.rs | 51 +++++++
tests/codegen/intrinsics/three_way_compare.rs | 47 ++++++
tests/mir-opt/lower_intrinsics.rs | 15 ++
...pare_char.LowerIntrinsics.panic-abort.diff | 34 +++++
...are_char.LowerIntrinsics.panic-unwind.diff | 34 +++++
...re_signed.LowerIntrinsics.panic-abort.diff | 31 ++++
...e_signed.LowerIntrinsics.panic-unwind.diff | 31 ++++
..._unsigned.LowerIntrinsics.panic-abort.diff | 34 +++++
...unsigned.LowerIntrinsics.panic-unwind.diff | 34 +++++
....{impl#0}-partial_cmp.PreCodegen.after.mir | 137 ++++--------------
35 files changed, 514 insertions(+), 112 deletions(-)
create mode 100644 tests/assembly/x86_64-cmp.rs
create mode 100644 tests/codegen/intrinsics/three_way_compare.rs
create mode 100644 tests/mir-opt/lower_intrinsics.three_way_compare_char.LowerIntrinsics.panic-abort.diff
create mode 100644 tests/mir-opt/lower_intrinsics.three_way_compare_char.LowerIntrinsics.panic-unwind.diff
create mode 100644 tests/mir-opt/lower_intrinsics.three_way_compare_signed.LowerIntrinsics.panic-abort.diff
create mode 100644 tests/mir-opt/lower_intrinsics.three_way_compare_signed.LowerIntrinsics.panic-unwind.diff
create mode 100644 tests/mir-opt/lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.panic-abort.diff
create mode 100644 tests/mir-opt/lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.panic-unwind.diff
diff --git a/compiler/rustc_codegen_cranelift/src/codegen_i128.rs b/compiler/rustc_codegen_cranelift/src/codegen_i128.rs
index b2bc289a5b6ba..4a5ef352151f3 100644
--- a/compiler/rustc_codegen_cranelift/src/codegen_i128.rs
+++ b/compiler/rustc_codegen_cranelift/src/codegen_i128.rs
@@ -68,7 +68,7 @@ pub(crate) fn maybe_codegen<'tcx>(
Some(CValue::by_val(ret_val, lhs.layout()))
}
}
- BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => None,
+ BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne | BinOp::Cmp => None,
BinOp::Shl | BinOp::ShlUnchecked | BinOp::Shr | BinOp::ShrUnchecked => None,
}
}
@@ -134,6 +134,7 @@ pub(crate) fn maybe_codegen_checked<'tcx>(
BinOp::AddUnchecked | BinOp::SubUnchecked | BinOp::MulUnchecked => unreachable!(),
BinOp::Offset => unreachable!("offset should only be used on pointers, not 128bit ints"),
BinOp::Div | BinOp::Rem => unreachable!(),
+ BinOp::Cmp => unreachable!(),
BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => unreachable!(),
BinOp::Shl | BinOp::ShlUnchecked | BinOp::Shr | BinOp::ShrUnchecked => unreachable!(),
}
diff --git a/compiler/rustc_codegen_cranelift/src/num.rs b/compiler/rustc_codegen_cranelift/src/num.rs
index 8992f40fb903a..796182418ad6c 100644
--- a/compiler/rustc_codegen_cranelift/src/num.rs
+++ b/compiler/rustc_codegen_cranelift/src/num.rs
@@ -40,6 +40,22 @@ pub(crate) fn bin_op_to_intcc(bin_op: BinOp, signed: bool) -> Option<IntCC> {
})
}
+fn codegen_three_way_compare<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ signed: bool,
+ lhs: Value,
+ rhs: Value,
+) -> CValue<'tcx> {
+ // This emits `(lhs > rhs) - (lhs < rhs)`, which is cranelift's preferred form per
+ // <https://github.com/bytecodealliance/wasmtime/blob/8052bb9e3b792503b225f2a5b2ba3bc023bff462/cranelift/codegen/src/prelude_opt.isle#L41-L47>
+ let gt_cc = crate::num::bin_op_to_intcc(BinOp::Gt, signed).unwrap();
+ let lt_cc = crate::num::bin_op_to_intcc(BinOp::Lt, signed).unwrap();
+ let gt = fx.bcx.ins().icmp(gt_cc, lhs, rhs);
+ let lt = fx.bcx.ins().icmp(lt_cc, lhs, rhs);
+ let val = fx.bcx.ins().isub(gt, lt);
+ CValue::by_val(val, fx.layout_of(fx.tcx.ty_ordering_enum(Some(fx.mir.span))))
+}
+
fn codegen_compare_bin_op<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
bin_op: BinOp,
@@ -47,6 +63,10 @@ fn codegen_compare_bin_op<'tcx>(
lhs: Value,
rhs: Value,
) -> CValue<'tcx> {
+ if bin_op == BinOp::Cmp {
+ return codegen_three_way_compare(fx, signed, lhs, rhs);
+ }
+
let intcc = crate::num::bin_op_to_intcc(bin_op, signed).unwrap();
let val = fx.bcx.ins().icmp(intcc, lhs, rhs);
CValue::by_val(val, fx.layout_of(fx.tcx.types.bool))
@@ -59,7 +79,7 @@ pub(crate) fn codegen_binop<'tcx>(
in_rhs: CValue<'tcx>,
) -> CValue<'tcx> {
match bin_op {
- BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => {
+ BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt | BinOp::Cmp => {
match in_lhs.layout().ty.kind() {
ty::Bool | ty::Uint(_) | ty::Int(_) | ty::Char => {
let signed = type_sign(in_lhs.layout().ty);
@@ -160,7 +180,7 @@ pub(crate) fn codegen_int_binop<'tcx>(
}
BinOp::Offset => unreachable!("Offset is not an integer operation"),
// Compare binops handles by `codegen_binop`.
- BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge => {
+ BinOp::Eq | BinOp::Ne | BinOp::Lt | BinOp::Le | BinOp::Gt | BinOp::Ge | BinOp::Cmp => {
unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs.layout().ty, in_rhs.layout().ty);
}
};
diff --git a/compiler/rustc_codegen_gcc/src/common.rs b/compiler/rustc_codegen_gcc/src/common.rs
index d243d7088ada9..78d943192db07 100644
--- a/compiler/rustc_codegen_gcc/src/common.rs
+++ b/compiler/rustc_codegen_gcc/src/common.rs
@@ -94,6 +94,10 @@ impl<'gcc, 'tcx> ConstMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
self.const_int(self.type_i32(), i as i64)
}
+ fn const_i8(&self, i: i8) -> RValue<'gcc> {
+ self.const_int(self.type_i8(), i as i64)
+ }
+
fn const_u32(&self, i: u32) -> RValue<'gcc> {
self.const_uint(self.type_u32(), i as u64)
}
diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs
index 25cbd90460f27..568fcc3f3cf49 100644
--- a/compiler/rustc_codegen_llvm/src/common.rs
+++ b/compiler/rustc_codegen_llvm/src/common.rs
@@ -160,6 +160,10 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
self.const_int(self.type_i32(), i as i64)
}
+ fn const_i8(&self, i: i8) -> &'ll Value {
+ self.const_int(self.type_i8(), i as i64)
+ }
+
fn const_u32(&self, i: u32) -> &'ll Value {
self.const_uint(self.type_i32(), i as u64)
}
diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
index 0e8c4abf21264..f0933fe33d747 100644
--- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
@@ -7,6 +7,7 @@ use crate::common::{self, IntPredicate};
use crate::traits::*;
use crate::MemFlags;
+use rustc_hir as hir;
use rustc_middle::mir;
use rustc_middle::mir::Operand;
use rustc_middle::ty::cast::{CastTy, IntTy};
@@ -882,6 +883,35 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
bx.icmp(base::bin_op_to_icmp_predicate(op.to_hir_binop(), is_signed), lhs, rhs)
}
}
+ mir::BinOp::Cmp => {
+ use std::cmp::Ordering;
+ debug_assert!(!is_float);
+ let pred = |op| base::bin_op_to_icmp_predicate(op, is_signed);
+ if bx.cx().tcx().sess.opts.optimize == OptLevel::No {
+ // FIXME: This actually generates tighter assembly, and is a classic trick
+ // <https://graphics.stanford.edu/~seander/bithacks.html#CopyIntegerSign>
+ // However, as of 2023-11 it optimizes worse in things like derived
+ // `PartialOrd`, so only use it in debug for now. Once LLVM can handle it
+ // better (see <https://github.com/llvm/llvm-project/issues/73417>), it'll
+ // be worth trying it in optimized builds as well.
+ let is_gt = bx.icmp(pred(hir::BinOpKind::Gt), lhs, rhs);
+ let gtext = bx.zext(is_gt, bx.type_i8());
+ let is_lt = bx.icmp(pred(hir::BinOpKind::Lt), lhs, rhs);
+ let ltext = bx.zext(is_lt, bx.type_i8());
+ bx.unchecked_ssub(gtext, ltext)
+ } else {
+ // These operations are those expected by `tests/codegen/integer-cmp.rs`,
+ // from <https://github.com/rust-lang/rust/pull/63767>.
+ let is_lt = bx.icmp(pred(hir::BinOpKind::Lt), lhs, rhs);
+ let is_ne = bx.icmp(pred(hir::BinOpKind::Ne), lhs, rhs);
+ let ge = bx.select(
+ is_ne,
+ bx.cx().const_i8(Ordering::Greater as i8),
+ bx.cx().const_i8(Ordering::Equal as i8),
+ );
+ bx.select(is_lt, bx.cx().const_i8(Ordering::Less as i8), ge)
+ }
+ }
}
}
diff --git a/compiler/rustc_codegen_ssa/src/traits/consts.rs b/compiler/rustc_codegen_ssa/src/traits/consts.rs
index 4dff9c7684f18..8cb17a5b37a89 100644
--- a/compiler/rustc_codegen_ssa/src/traits/consts.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/consts.rs
@@ -19,6 +19,7 @@ pub trait ConstMethods<'tcx>: BackendTypes {
fn const_bool(&self, val: bool) -> Self::Value;
fn const_i16(&self, i: i16) -> Self::Value;
fn const_i32(&self, i: i32) -> Self::Value;
+ fn const_i8(&self, i: i8) -> Self::Value;
fn const_u32(&self, i: u32) -> Self::Value;
fn const_u64(&self, i: u64) -> Self::Value;
fn const_u128(&self, i: u128) -> Self::Value;
diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs
index dbc6a317640c1..831787a92c869 100644
--- a/compiler/rustc_const_eval/src/interpret/operand.rs
+++ b/compiler/rustc_const_eval/src/interpret/operand.rs
@@ -235,6 +235,13 @@ impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> {
Self::from_scalar(Scalar::from_bool(b), layout)
}
+ #[inline]
+ pub fn from_ordering(c: std::cmp::Ordering, tcx: TyCtxt<'tcx>) -> Self {
+ let ty = tcx.ty_ordering_enum(None);
+ let layout = tcx.layout_of(ty::ParamEnv::reveal_all().and(ty)).unwrap();
+ Self::from_scalar(Scalar::from_i8(c as i8), layout)
+ }
+
#[inline]
pub fn to_const_int(self) -> ConstInt {
assert!(self.layout.ty.is_integral());
diff --git a/compiler/rustc_const_eval/src/interpret/operator.rs b/compiler/rustc_const_eval/src/interpret/operator.rs
index 475c533077af5..5665bb4999f59 100644
--- a/compiler/rustc_const_eval/src/interpret/operator.rs
+++ b/compiler/rustc_const_eval/src/interpret/operator.rs
@@ -61,6 +61,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
}
impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
+ fn three_way_compare<T: Ord>(&self, lhs: T, rhs: T) -> (ImmTy<'tcx, M::Provenance>, bool) {
+ let res = Ord::cmp(&lhs, &rhs);
+ return (ImmTy::from_ordering(res, *self.tcx), false);
+ }
+
fn binary_char_op(
&self,
bin_op: mir::BinOp,
@@ -69,6 +74,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
) -> (ImmTy<'tcx, M::Provenance>, bool) {
use rustc_middle::mir::BinOp::*;
+ if bin_op == Cmp {
+ return self.three_way_compare(l, r);
+ }
+
let res = match bin_op {
Eq => l == r,
Ne => l != r,
@@ -231,6 +240,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
let r = self.sign_extend(r, right_layout) as i128;
return Ok((ImmTy::from_bool(op(&l, &r), *self.tcx), false));
}
+ if bin_op == Cmp {
+ let l = self.sign_extend(l, left_layout) as i128;
+ let r = self.sign_extend(r, right_layout) as i128;
+ return Ok(self.three_way_compare(l, r));
+ }
let op: Option<fn(i128, i128) -> (i128, bool)> = match bin_op {
Div if r == 0 => throw_ub!(DivisionByZero),
Rem if r == 0 => throw_ub!(RemainderByZero),
@@ -270,6 +284,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
}
}
+ if bin_op == Cmp {
+ return Ok(self.three_way_compare(l, r));
+ }
+
let val = match bin_op {
Eq => ImmTy::from_bool(l == r, *self.tcx),
Ne => ImmTy::from_bool(l != r, *self.tcx),
diff --git a/compiler/rustc_const_eval/src/transform/validate.rs b/compiler/rustc_const_eval/src/transform/validate.rs
index 1664475f52a13..6cde2296459e5 100644
--- a/compiler/rustc_const_eval/src/transform/validate.rs
+++ b/compiler/rustc_const_eval/src/transform/validate.rs
@@ -987,6 +987,15 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
)
}
}
+ Cmp => {
+ for x in [a, b] {
+ check_kinds!(
+ x,
+ "Cannot three-way compare non-integer type {:?}",
+ ty::Char | ty::Uint(..) | ty::Int(..)
+ )
+ }
+ }
AddUnchecked | SubUnchecked | MulUnchecked | Shl | ShlUnchecked | Shr
| ShrUnchecked => {
for x in [a, b] {
diff --git a/compiler/rustc_const_eval/src/util/mod.rs b/compiler/rustc_const_eval/src/util/mod.rs
index a8060463b69ed..0c3b59a0e78ec 100644
--- a/compiler/rustc_const_eval/src/util/mod.rs
+++ b/compiler/rustc_const_eval/src/util/mod.rs
@@ -19,7 +19,7 @@ pub fn binop_left_homogeneous(op: mir::BinOp) -> bool {
match op {
Add | AddUnchecked | Sub | SubUnchecked | Mul | MulUnchecked | Div | Rem | BitXor
| BitAnd | BitOr | Offset | Shl | ShlUnchecked | Shr | ShrUnchecked => true,
- Eq | Ne | Lt | Le | Gt | Ge => false,
+ Eq | Ne | Lt | Le | Gt | Ge | Cmp => false,
}
}
@@ -30,7 +30,7 @@ pub fn binop_right_homogeneous(op: mir::BinOp) -> bool {
use rustc_middle::mir::BinOp::*;
match op {
Add | AddUnchecked | Sub | SubUnchecked | Mul | MulUnchecked | Div | Rem | BitXor
- | BitAnd | BitOr | Eq | Ne | Lt | Le | Gt | Ge => true,
+ | BitAnd | BitOr | Eq | Ne | Lt | Le | Gt | Ge | Cmp => true,
Offset | Shl | ShlUnchecked | Shr | ShrUnchecked => false,
}
}
diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs
index dbf86f5cf747d..ba971958aa92b 100644
--- a/compiler/rustc_hir/src/lang_items.rs
+++ b/compiler/rustc_hir/src/lang_items.rs
@@ -225,6 +225,7 @@ language_item_table! {
Unpin, sym::unpin, unpin_trait, Target::Trait, GenericRequirement::None;
Pin, sym::pin, pin_type, Target::Struct, GenericRequirement::None;
+ OrderingEnum, sym::Ordering, ordering_enum, Target::Enum, GenericRequirement::Exact(0);
PartialEq, sym::eq, eq_trait, Target::Trait, GenericRequirement::Exact(1);
PartialOrd, sym::partial_ord, partial_ord_trait, Target::Trait, GenericRequirement::Exact(1);
CVoid, sym::c_void, c_void, Target::Enum, GenericRequirement::None;
diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs
index f482ae4f5fa07..93d71e4bfb933 100644
--- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs
+++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs
@@ -107,6 +107,7 @@ pub fn intrinsic_operation_unsafety(tcx: TyCtxt<'_>, intrinsic_id: LocalDefId) -
| sym::cttz
| sym::bswap
| sym::bitreverse
+ | sym::three_way_compare
| sym::discriminant_value
| sym::type_id
| sym::likely
@@ -418,6 +419,10 @@ pub fn check_intrinsic_type(
| sym::bswap
| sym::bitreverse => (1, 0, vec![param(0)], param(0)),
+ sym::three_way_compare => {
+ (1, 0, vec![param(0), param(0)], tcx.ty_ordering_enum(Some(span)))
+ }
+
sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
(1, 0, vec![param(0), param(0)], Ty::new_tup(tcx, &[param(0), tcx.types.bool]))
}
diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs
index 36b7a48b2a2b8..a33de2efe2fad 100644
--- a/compiler/rustc_middle/src/mir/syntax.rs
+++ b/compiler/rustc_middle/src/mir/syntax.rs
@@ -1444,6 +1444,8 @@ pub enum BinOp {
Ge,
/// The `>` operator (greater than)
Gt,
+ /// The `<=>` operator (three-way comparison, like `Ord::cmp`)
+ Cmp,
/// The `ptr.offset` operator
Offset,
}
diff --git a/compiler/rustc_middle/src/mir/tcx.rs b/compiler/rustc_middle/src/mir/tcx.rs
index 56a0a62339700..74c2aef8fcc20 100644
--- a/compiler/rustc_middle/src/mir/tcx.rs
+++ b/compiler/rustc_middle/src/mir/tcx.rs
@@ -276,6 +276,11 @@ impl<'tcx> BinOp {
&BinOp::Eq | &BinOp::Lt | &BinOp::Le | &BinOp::Ne | &BinOp::Ge | &BinOp::Gt => {
tcx.types.bool
}
+ &BinOp::Cmp => {
+ // these should be integer-like types of the same size.
+ assert_eq!(lhs_ty, rhs_ty);
+ tcx.ty_ordering_enum(None)
+ }
}
}
}
@@ -312,7 +317,8 @@ impl BinOp {
BinOp::Gt => hir::BinOpKind::Gt,
BinOp::Le => hir::BinOpKind::Le,
BinOp::Ge => hir::BinOpKind::Ge,
- BinOp::AddUnchecked
+ BinOp::Cmp
+ | BinOp::AddUnchecked
| BinOp::SubUnchecked
| BinOp::MulUnchecked
| BinOp::ShlUnchecked
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index 3393f44484388..01fd815be6d67 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -905,6 +905,13 @@ impl<'tcx> TyCtxt<'tcx> {
self.get_lang_items(())
}
+ /// Gets a `Ty` representing the [`LangItem::OrderingEnum`]
+ #[track_caller]
+ pub fn ty_ordering_enum(self, span: Option<Span>) -> Ty<'tcx> {
+ let ordering_enum = self.require_lang_item(hir::LangItem::OrderingEnum, span);
+ self.type_of(ordering_enum).no_bound_vars().unwrap()
+ }
+
/// Obtain the given diagnostic item's `DefId`. Use `is_diagnostic_item` if you just want to
/// compare against another `DefId`, since `is_diagnostic_item` is cheaper.
pub fn get_diagnostic_item(self, name: Symbol) -> Option<DefId> {
diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs
index 7d4c1b9c21a62..7e8920604c176 100644
--- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs
+++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs
@@ -90,6 +90,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
sym::wrapping_add
| sym::wrapping_sub
| sym::wrapping_mul
+ | sym::three_way_compare
| sym::unchecked_add
| sym::unchecked_sub
| sym::unchecked_mul
@@ -109,6 +110,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
sym::wrapping_add => BinOp::Add,
sym::wrapping_sub => BinOp::Sub,
sym::wrapping_mul => BinOp::Mul,
+ sym::three_way_compare => BinOp::Cmp,
sym::unchecked_add => BinOp::AddUnchecked,
sym::unchecked_sub => BinOp::SubUnchecked,
sym::unchecked_mul => BinOp::MulUnchecked,
diff --git a/compiler/rustc_mir_transform/src/promote_consts.rs b/compiler/rustc_mir_transform/src/promote_consts.rs
index 2951897ebd697..d64cb9b12b2bd 100644
--- a/compiler/rustc_mir_transform/src/promote_consts.rs
+++ b/compiler/rustc_mir_transform/src/promote_consts.rs
@@ -525,6 +525,7 @@ impl<'tcx> Validator<'_, 'tcx> {
| BinOp::Lt
| BinOp::Ge
| BinOp::Gt
+ | BinOp::Cmp
| BinOp::Offset
| BinOp::Add
| BinOp::AddUnchecked
diff --git a/compiler/rustc_smir/src/rustc_smir/convert/mir.rs b/compiler/rustc_smir/src/rustc_smir/convert/mir.rs
index b6a722da602e0..22e7e5323c776 100644
--- a/compiler/rustc_smir/src/rustc_smir/convert/mir.rs
+++ b/compiler/rustc_smir/src/rustc_smir/convert/mir.rs
@@ -493,6 +493,7 @@ impl<'tcx> Stable<'tcx> for mir::BinOp {
BinOp::Ne => stable_mir::mir::BinOp::Ne,
BinOp::Ge => stable_mir::mir::BinOp::Ge,
BinOp::Gt => stable_mir::mir::BinOp::Gt,
+ BinOp::Cmp => stable_mir::mir::BinOp::Cmp,
BinOp::Offset => stable_mir::mir::BinOp::Offset,
}
}
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 73fcd2a76dfc0..7dffeab992284 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -1785,6 +1785,7 @@ symbols! {
thread,
thread_local,
thread_local_macro,
+ three_way_compare,
thumb2,
thumb_mode: "thumb-mode",
tmm_reg,
diff --git a/compiler/rustc_ty_utils/src/consts.rs b/compiler/rustc_ty_utils/src/consts.rs
index 2a2e53a81ed81..acbcc3918b2fb 100644
--- a/compiler/rustc_ty_utils/src/consts.rs
+++ b/compiler/rustc_ty_utils/src/consts.rs
@@ -82,7 +82,7 @@ fn check_binop(op: mir::BinOp) -> bool {
match op {
Add | AddUnchecked | Sub | SubUnchecked | Mul | MulUnchecked | Div | Rem | BitXor
| BitAnd | BitOr | Shl | ShlUnchecked | Shr | ShrUnchecked | Eq | Lt | Le | Ne | Ge
- | Gt => true,
+ | Gt | Cmp => true,
Offset => false,
}
}
diff --git a/compiler/stable_mir/src/mir/body.rs b/compiler/stable_mir/src/mir/body.rs
index 7c536a3e91402..636bf919f5c63 100644
--- a/compiler/stable_mir/src/mir/body.rs
+++ b/compiler/stable_mir/src/mir/body.rs
@@ -329,6 +329,7 @@ pub enum BinOp {
Ne,
Ge,
Gt,
+ Cmp,
Offset,
}
@@ -368,6 +369,9 @@ impl BinOp {
assert!(lhs_kind.is_primitive() || lhs_kind.is_raw_ptr() || lhs_kind.is_fn_ptr());
Ty::bool_ty()
}
+ BinOp::Cmp => {
+ unimplemented!("Should cmp::Ordering be a RigidTy?");
+ }
}
}
}
diff --git a/library/core/src/cmp.rs b/library/core/src/cmp.rs
index a2f07814726ac..7581c6dd759c3 100644
--- a/library/core/src/cmp.rs
+++ b/library/core/src/cmp.rs
@@ -376,6 +376,7 @@ pub struct AssertParamIsEq<T: Eq + ?Sized> {
/// ```
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
#[stable(feature = "rust1", since = "1.0.0")]
+#[cfg_attr(not(bootstrap), lang = "Ordering")]
#[repr(i8)]
pub enum Ordering {
/// An ordering where a compared value is less than another.
@@ -1563,12 +1564,19 @@ mod impls {
impl Ord for $t {
#[inline]
fn cmp(&self, other: &$t) -> Ordering {
+ #[cfg(bootstrap)]
+ {
// The order here is important to generate more optimal assembly.
// See <https://github.com/rust-lang/rust/issues/63758> for more info.
if *self < *other { Less }
else if *self == *other { Equal }
else { Greater }
}
+ #[cfg(not(bootstrap))]
+ {
+ crate::intrinsics::three_way_compare(*self, *other)
+ }
+ }
}
)*)
}
diff --git a/library/core/src/intrinsics.rs b/library/core/src/intrinsics.rs
index dec31548fc8c3..af771b88dc30e 100644
--- a/library/core/src/intrinsics.rs
+++ b/library/core/src/intrinsics.rs
@@ -2146,6 +2146,18 @@ extern "rust-intrinsic" {
#[rustc_nounwind]
pub fn bitreverse<T: Copy>(x: T) -> T;
+ /// Does a three-way comparison between the two integer arguments.
+ ///
+ /// This is included as an intrinsic as it's useful to let it be one thing
+ /// in MIR, rather than the multiple checks and switches that make its IR
+ /// large and difficult to optimize.
+ ///
+ /// The stabilized version of this intrinsic is [`Ord::cmp`].
+ #[cfg(not(bootstrap))]
+ #[rustc_const_unstable(feature = "const_three_way_compare", issue = "none")]
+ #[rustc_safe_intrinsic]
+ pub fn three_way_compare<T: Copy>(lhs: T, rhs: T) -> crate::cmp::Ordering;
+
/// Performs checked integer addition.
///
/// Note that, unlike most intrinsics, this is safe to call;
diff --git a/library/core/tests/intrinsics.rs b/library/core/tests/intrinsics.rs
index 740565d0df6b4..a9ef5415ada6f 100644
--- a/library/core/tests/intrinsics.rs
+++ b/library/core/tests/intrinsics.rs
@@ -99,3 +99,22 @@ fn test_const_deallocate_at_runtime() {
const_deallocate(core::ptr::null_mut(), 1, 1); // nop
}
}
+
+#[cfg(not(bootstrap))]
+#[test]
+fn test_three_way_compare_in_const_contexts() {
+ use core::cmp::Ordering::*;
+ use core::intrinsics::three_way_compare;
+
+ const {
+ assert!(Less as i8 == three_way_compare(123_u16, 456) as _);
+ assert!(Equal as i8 == three_way_compare(456_u16, 456) as _);
+ assert!(Greater as i8 == three_way_compare(789_u16, 456) as _);
+ assert!(Less as i8 == three_way_compare('A', 'B') as _);
+ assert!(Equal as i8 == three_way_compare('B', 'B') as _);
+ assert!(Greater as i8 == three_way_compare('C', 'B') as _);
+ assert!(Less as i8 == three_way_compare(-123_i16, 456) as _);
+ assert!(Equal as i8 == three_way_compare(456_i16, 456) as _);
+ assert!(Greater as i8 == three_way_compare(123_i16, -456) as _);
+ }
+}
diff --git a/library/core/tests/lib.rs b/library/core/tests/lib.rs
index 421062f5873cd..6a2ff3a2285a8 100644
--- a/library/core/tests/lib.rs
+++ b/library/core/tests/lib.rs
@@ -21,6 +21,7 @@
#![feature(const_pointer_is_aligned)]
#![feature(const_ptr_as_ref)]
#![feature(const_ptr_write)]
+#![cfg_attr(not(bootstrap), feature(const_three_way_compare))]
#![feature(const_trait_impl)]
#![feature(const_likely)]
#![feature(const_location_fields)]
diff --git a/tests/assembly/x86_64-cmp.rs b/tests/assembly/x86_64-cmp.rs
new file mode 100644
index 0000000000000..31efdda1bfafa
--- /dev/null
+++ b/tests/assembly/x86_64-cmp.rs
@@ -0,0 +1,51 @@
+//@ revisions: DEBUG OPTIM
+//@ [DEBUG] compile-flags: -C opt-level=0
+//@ [OPTIM] compile-flags: -C opt-level=3
+//@ assembly-output: emit-asm
+//@ compile-flags: --crate-type=lib -C llvm-args=-x86-asm-syntax=intel
+//@ only-x86_64
+//@ ignore-sgx
+
+#![feature(core_intrinsics)]
+
+use std::intrinsics::three_way_compare;
+
+#[no_mangle]
+// CHECK-LABEL: signed_cmp:
+pub fn signed_cmp(a: i16, b: i16) -> std::cmp::Ordering {
+ // DEBUG: cmp
+ // DEBUG: setg
+ // DEBUG: and
+ // DEBUG: cmp
+ // DEBUG: setl
+ // DEBUG: and
+ // DEBUG: sub
+
+ // OPTIM: xor
+ // OPTIM: cmp
+ // OPTIM: setne
+ // OPTIM: mov
+ // OPTIM: cmovge
+ // OPTIM: ret
+ three_way_compare(a, b)
+}
+
+#[no_mangle]
+// CHECK-LABEL: unsigned_cmp:
+pub fn unsigned_cmp(a: u16, b: u16) -> std::cmp::Ordering {
+ // DEBUG: cmp
+ // DEBUG: seta
+ // DEBUG: and
+ // DEBUG: cmp
+ // DEBUG: setb
+ // DEBUG: and
+ // DEBUG: sub
+
+ // OPTIM: xor
+ // OPTIM: cmp
+ // OPTIM: setne
+ // OPTIM: mov
+ // OPTIM: cmovae
+ // OPTIM: ret
+ three_way_compare(a, b)
+}
diff --git a/tests/codegen/intrinsics/three_way_compare.rs b/tests/codegen/intrinsics/three_way_compare.rs
new file mode 100644
index 0000000000000..f3b631abc227e
--- /dev/null
+++ b/tests/codegen/intrinsics/three_way_compare.rs
@@ -0,0 +1,47 @@
+//@ revisions: DEBUG OPTIM
+//@ [DEBUG] compile-flags: -C opt-level=0
+//@ [OPTIM] compile-flags: -C opt-level=3
+//@ compile-flags: -C no-prepopulate-passes
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::three_way_compare;
+
+#[no_mangle]
+// CHECK-LABEL: @signed_cmp
+// DEBUG-SAME: (i16 %a, i16 %b)
+// OPTIM-SAME: (i16 noundef %a, i16 noundef %b)
+pub fn signed_cmp(a: i16, b: i16) -> std::cmp::Ordering {
+ // DEBUG: %[[GT:.+]] = icmp sgt i16 %a, %b
+ // DEBUG: %[[ZGT:.+]] = zext i1 %[[GT]] to i8
+ // DEBUG: %[[LT:.+]] = icmp slt i16 %a, %b
+ // DEBUG: %[[ZLT:.+]] = zext i1 %[[LT]] to i8
+ // DEBUG: %[[R:.+]] = sub nsw i8 %[[ZGT]], %[[ZLT]]
+
+ // OPTIM: %[[LT:.+]] = icmp slt i16 %a, %b
+ // OPTIM: %[[NE:.+]] = icmp ne i16 %a, %b
+ // OPTIM: %[[CGE:.+]] = select i1 %[[NE]], i8 1, i8 0
+ // OPTIM: %[[CGEL:.+]] = select i1 %[[LT]], i8 -1, i8 %[[CGE]]
+ // OPTIM: ret i8 %[[CGEL]]
+ three_way_compare(a, b)
+}
+
+#[no_mangle]
+// CHECK-LABEL: @unsigned_cmp
+// DEBUG-SAME: (i16 %a, i16 %b)
+// OPTIM-SAME: (i16 noundef %a, i16 noundef %b)
+pub fn unsigned_cmp(a: u16, b: u16) -> std::cmp::Ordering {
+ // DEBUG: %[[GT:.+]] = icmp ugt i16 %a, %b
+ // DEBUG: %[[ZGT:.+]] = zext i1 %[[GT]] to i8
+ // DEBUG: %[[LT:.+]] = icmp ult i16 %a, %b
+ // DEBUG: %[[ZLT:.+]] = zext i1 %[[LT]] to i8
+ // DEBUG: %[[R:.+]] = sub nsw i8 %[[ZGT]], %[[ZLT]]
+
+ // OPTIM: %[[LT:.+]] = icmp ult i16 %a, %b
+ // OPTIM: %[[NE:.+]] = icmp ne i16 %a, %b
+ // OPTIM: %[[CGE:.+]] = select i1 %[[NE]], i8 1, i8 0
+ // OPTIM: %[[CGEL:.+]] = select i1 %[[LT]], i8 -1, i8 %[[CGE]]
+ // OPTIM: ret i8 %[[CGEL]]
+ three_way_compare(a, b)
+}
diff --git a/tests/mir-opt/lower_intrinsics.rs b/tests/mir-opt/lower_intrinsics.rs
index 278ddfce1c331..e9eb392e79528 100644
--- a/tests/mir-opt/lower_intrinsics.rs
+++ b/tests/mir-opt/lower_intrinsics.rs
@@ -229,3 +229,18 @@ pub unsafe fn ptr_offset(p: *const i32, d: isize) -> *const i32 {
core::intrinsics::offset(p, d)
}
+
+// EMIT_MIR lower_intrinsics.three_way_compare_char.LowerIntrinsics.diff
+pub fn three_way_compare_char(a: char, b: char) {
+ let _x = core::intrinsics::three_way_compare(a, b);
+}
+
+// EMIT_MIR lower_intrinsics.three_way_compare_signed.LowerIntrinsics.diff
+pub fn three_way_compare_signed(a: i16, b: i16) {
+ core::intrinsics::three_way_compare(a, b);
+}
+
+// EMIT_MIR lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.diff
+pub fn three_way_compare_unsigned(a: u32, b: u32) {
+ let _x = core::intrinsics::three_way_compare(a, b);
+}
diff --git a/tests/mir-opt/lower_intrinsics.three_way_compare_char.LowerIntrinsics.panic-abort.diff b/tests/mir-opt/lower_intrinsics.three_way_compare_char.LowerIntrinsics.panic-abort.diff
new file mode 100644
index 0000000000000..816d620971588
--- /dev/null
+++ b/tests/mir-opt/lower_intrinsics.three_way_compare_char.LowerIntrinsics.panic-abort.diff
@@ -0,0 +1,34 @@
+- // MIR for `three_way_compare_char` before LowerIntrinsics
++ // MIR for `three_way_compare_char` after LowerIntrinsics
+
+ fn three_way_compare_char(_1: char, _2: char) -> () {
+ debug a => _1;
+ debug b => _2;
+ let mut _0: ();
+ let _3: std::cmp::Ordering;
+ let mut _4: char;
+ let mut _5: char;
+ scope 1 {
+ debug _x => _3;
+ }
+
+ bb0: {
+ StorageLive(_3);
+ StorageLive(_4);
+ _4 = _1;
+ StorageLive(_5);
+ _5 = _2;
+- _3 = three_way_compare::<char>(move _4, move _5) -> [return: bb1, unwind unreachable];
++ _3 = Cmp(move _4, move _5);
++ goto -> bb1;
+ }
+
+ bb1: {
+ StorageDead(_5);
+ StorageDead(_4);
+ _0 = const ();
+ StorageDead(_3);
+ return;
+ }
+ }
+
diff --git a/tests/mir-opt/lower_intrinsics.three_way_compare_char.LowerIntrinsics.panic-unwind.diff b/tests/mir-opt/lower_intrinsics.three_way_compare_char.LowerIntrinsics.panic-unwind.diff
new file mode 100644
index 0000000000000..80b4bd7a2be75
--- /dev/null
+++ b/tests/mir-opt/lower_intrinsics.three_way_compare_char.LowerIntrinsics.panic-unwind.diff
@@ -0,0 +1,34 @@
+- // MIR for `three_way_compare_char` before LowerIntrinsics
++ // MIR for `three_way_compare_char` after LowerIntrinsics
+
+ fn three_way_compare_char(_1: char, _2: char) -> () {
+ debug a => _1;
+ debug b => _2;
+ let mut _0: ();
+ let _3: std::cmp::Ordering;
+ let mut _4: char;
+ let mut _5: char;
+ scope 1 {
+ debug _x => _3;
+ }
+
+ bb0: {
+ StorageLive(_3);
+ StorageLive(_4);
+ _4 = _1;
+ StorageLive(_5);
+ _5 = _2;
+- _3 = three_way_compare::<char>(move _4, move _5) -> [return: bb1, unwind continue];
++ _3 = Cmp(move _4, move _5);
++ goto -> bb1;
+ }
+
+ bb1: {
+ StorageDead(_5);
+ StorageDead(_4);
+ _0 = const ();
+ StorageDead(_3);
+ return;
+ }
+ }
+
diff --git a/tests/mir-opt/lower_intrinsics.three_way_compare_signed.LowerIntrinsics.panic-abort.diff b/tests/mir-opt/lower_intrinsics.three_way_compare_signed.LowerIntrinsics.panic-abort.diff
new file mode 100644
index 0000000000000..05c20aaa09a27
--- /dev/null
+++ b/tests/mir-opt/lower_intrinsics.three_way_compare_signed.LowerIntrinsics.panic-abort.diff
@@ -0,0 +1,31 @@
+- // MIR for `three_way_compare_signed` before LowerIntrinsics
++ // MIR for `three_way_compare_signed` after LowerIntrinsics
+
+ fn three_way_compare_signed(_1: i16, _2: i16) -> () {
+ debug a => _1;
+ debug b => _2;
+ let mut _0: ();
+ let _3: std::cmp::Ordering;
+ let mut _4: i16;
+ let mut _5: i16;
+
+ bb0: {
+ StorageLive(_3);
+ StorageLive(_4);
+ _4 = _1;
+ StorageLive(_5);
+ _5 = _2;
+- _3 = three_way_compare::<i16>(move _4, move _5) -> [return: bb1, unwind unreachable];
++ _3 = Cmp(move _4, move _5);
++ goto -> bb1;
+ }
+
+ bb1: {
+ StorageDead(_5);
+ StorageDead(_4);
+ StorageDead(_3);
+ _0 = const ();
+ return;
+ }
+ }
+
diff --git a/tests/mir-opt/lower_intrinsics.three_way_compare_signed.LowerIntrinsics.panic-unwind.diff b/tests/mir-opt/lower_intrinsics.three_way_compare_signed.LowerIntrinsics.panic-unwind.diff
new file mode 100644
index 0000000000000..8a254d02a47b2
--- /dev/null
+++ b/tests/mir-opt/lower_intrinsics.three_way_compare_signed.LowerIntrinsics.panic-unwind.diff
@@ -0,0 +1,31 @@
+- // MIR for `three_way_compare_signed` before LowerIntrinsics
++ // MIR for `three_way_compare_signed` after LowerIntrinsics
+
+ fn three_way_compare_signed(_1: i16, _2: i16) -> () {
+ debug a => _1;
+ debug b => _2;
+ let mut _0: ();
+ let _3: std::cmp::Ordering;
+ let mut _4: i16;
+ let mut _5: i16;
+
+ bb0: {
+ StorageLive(_3);
+ StorageLive(_4);
+ _4 = _1;
+ StorageLive(_5);
+ _5 = _2;
+- _3 = three_way_compare::<i16>(move _4, move _5) -> [return: bb1, unwind continue];
++ _3 = Cmp(move _4, move _5);
++ goto -> bb1;
+ }
+
+ bb1: {
+ StorageDead(_5);
+ StorageDead(_4);
+ StorageDead(_3);
+ _0 = const ();
+ return;
+ }
+ }
+
diff --git a/tests/mir-opt/lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.panic-abort.diff b/tests/mir-opt/lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.panic-abort.diff
new file mode 100644
index 0000000000000..437614ec6738d
--- /dev/null
+++ b/tests/mir-opt/lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.panic-abort.diff
@@ -0,0 +1,34 @@
+- // MIR for `three_way_compare_unsigned` before LowerIntrinsics
++ // MIR for `three_way_compare_unsigned` after LowerIntrinsics
+
+ fn three_way_compare_unsigned(_1: u32, _2: u32) -> () {
+ debug a => _1;
+ debug b => _2;
+ let mut _0: ();
+ let _3: std::cmp::Ordering;
+ let mut _4: u32;
+ let mut _5: u32;
+ scope 1 {
+ debug _x => _3;
+ }
+
+ bb0: {
+ StorageLive(_3);
+ StorageLive(_4);
+ _4 = _1;
+ StorageLive(_5);
+ _5 = _2;
+- _3 = three_way_compare::<u32>(move _4, move _5) -> [return: bb1, unwind unreachable];
++ _3 = Cmp(move _4, move _5);
++ goto -> bb1;
+ }
+
+ bb1: {
+ StorageDead(_5);
+ StorageDead(_4);
+ _0 = const ();
+ StorageDead(_3);
+ return;
+ }
+ }
+
diff --git a/tests/mir-opt/lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.panic-unwind.diff b/tests/mir-opt/lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.panic-unwind.diff
new file mode 100644
index 0000000000000..7d6137979c81b
--- /dev/null
+++ b/tests/mir-opt/lower_intrinsics.three_way_compare_unsigned.LowerIntrinsics.panic-unwind.diff
@@ -0,0 +1,34 @@
+- // MIR for `three_way_compare_unsigned` before LowerIntrinsics
++ // MIR for `three_way_compare_unsigned` after LowerIntrinsics
+
+ fn three_way_compare_unsigned(_1: u32, _2: u32) -> () {
+ debug a => _1;
+ debug b => _2;
+ let mut _0: ();
+ let _3: std::cmp::Ordering;
+ let mut _4: u32;
+ let mut _5: u32;
+ scope 1 {
+ debug _x => _3;
+ }
+
+ bb0: {
+ StorageLive(_3);
+ StorageLive(_4);
+ _4 = _1;
+ StorageLive(_5);
+ _5 = _2;
+- _3 = three_way_compare::<u32>(move _4, move _5) -> [return: bb1, unwind continue];
++ _3 = Cmp(move _4, move _5);
++ goto -> bb1;
+ }
+
+ bb1: {
+ StorageDead(_5);
+ StorageDead(_4);
+ _0 = const ();
+ StorageDead(_3);
+ return;
+ }
+ }
+
diff --git a/tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir
index dd7fce3ed0b46..7ce2d229d5474 100644
--- a/tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir
+++ b/tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir
@@ -6,37 +6,33 @@ fn <impl at $DIR/derived_ord.rs:6:10: 6:20>::partial_cmp(_1: &MultiField, _2: &M
let mut _0: std::option::Option<std::cmp::Ordering>;
let mut _3: &char;
let mut _4: &char;
- let mut _10: std::option::Option<std::cmp::Ordering>;
+ let mut _8: std::option::Option<std::cmp::Ordering>;
+ let mut _9: i8;
+ let mut _10: &i16;
let mut _11: &i16;
- let mut _12: &i16;
- let _18: std::option::Option<std::cmp::Ordering>;
scope 1 {
- debug cmp => _18;
+ debug cmp => _8;
}
scope 2 (inlined std::cmp::impls::<impl PartialOrd for char>::partial_cmp) {
debug self => _3;
debug other => _4;
- let mut _9: std::cmp::Ordering;
+ let mut _7: std::cmp::Ordering;
scope 3 (inlined std::cmp::impls::<impl Ord for char>::cmp) {
debug self => _3;
debug other => _4;
let mut _5: char;
let mut _6: char;
- let mut _7: bool;
- let mut _8: bool;
}
}
scope 4 (inlined std::cmp::impls::<impl PartialOrd for i16>::partial_cmp) {
- debug self => _11;
- debug other => _12;
- let mut _17: std::cmp::Ordering;
+ debug self => _10;
+ debug other => _11;
+ let mut _14: std::cmp::Ordering;
scope 5 (inlined std::cmp::impls::<impl Ord for i16>::cmp) {
- debug self => _11;
- debug other => _12;
+ debug self => _10;
+ debug other => _11;
+ let mut _12: i16;
let mut _13: i16;
- let mut _14: i16;
- let mut _15: bool;
- let mut _16: bool;
}
}
@@ -45,115 +41,46 @@ fn <impl at $DIR/derived_ord.rs:6:10: 6:20>::partial_cmp(_1: &MultiField, _2: &M
_3 = &((*_1).0: char);
StorageLive(_4);
_4 = &((*_2).0: char);
- StorageLive(_9);
StorageLive(_5);
- StorageLive(_6);
- StorageLive(_7);
_5 = ((*_1).0: char);
+ StorageLive(_6);
_6 = ((*_2).0: char);
- _7 = Lt(_5, _6);
- switchInt(move _7) -> [0: bb1, otherwise: bb10];
- }
-
- bb1: {
- StorageLive(_8);
- _8 = Eq(_5, _6);
- switchInt(move _8) -> [0: bb2, otherwise: bb3];
- }
-
- bb2: {
- _9 = const Greater;
- StorageDead(_8);
- StorageDead(_7);
+ _7 = Cmp(move _5, move _6);
StorageDead(_6);
StorageDead(_5);
- _10 = Option::<std::cmp::Ordering>::Some(move _9);
- StorageDead(_9);
+ _8 = Option::<std::cmp::Ordering>::Some(_7);
StorageDead(_4);
StorageDead(_3);
- goto -> bb11;
+ _9 = discriminant(_7);
+ switchInt(move _9) -> [0: bb1, otherwise: bb2];
}
- bb3: {
- StorageDead(_8);
- StorageDead(_7);
- StorageDead(_6);
- StorageDead(_5);
- StorageDead(_9);
- StorageDead(_4);
- StorageDead(_3);
+ bb1: {
+ StorageLive(_10);
+ _10 = &((*_1).1: i16);
StorageLive(_11);
- _11 = &((*_1).1: i16);
+ _11 = &((*_2).1: i16);
+ StorageLive(_14);
StorageLive(_12);
- _12 = &((*_2).1: i16);
- StorageLive(_17);
+ _12 = ((*_1).1: i16);
StorageLive(_13);
- StorageLive(_14);
- StorageLive(_15);
- _13 = ((*_1).1: i16);
- _14 = ((*_2).1: i16);
- _15 = Lt(_13, _14);
- switchInt(move _15) -> [0: bb4, otherwise: bb8];
- }
-
- bb4: {
- StorageLive(_16);
- _16 = Eq(_13, _14);
- switchInt(move _16) -> [0: bb5, otherwise: bb6];
- }
-
- bb5: {
- _17 = const Greater;
- goto -> bb7;
- }
-
- bb6: {
- _17 = const Equal;
- goto -> bb7;
- }
-
- bb7: {
- StorageDead(_16);
- goto -> bb9;
- }
-
- bb8: {
- _17 = const Less;
- goto -> bb9;
- }
-
- bb9: {
- StorageDead(_15);
- StorageDead(_14);
+ _13 = ((*_2).1: i16);
+ _14 = Cmp(move _12, move _13);
StorageDead(_13);
- _0 = Option::<std::cmp::Ordering>::Some(move _17);
- StorageDead(_17);
StorageDead(_12);
+ _0 = Option::<std::cmp::Ordering>::Some(move _14);
+ StorageDead(_14);
StorageDead(_11);
- goto -> bb12;
+ StorageDead(_10);
+ goto -> bb3;
}
- bb10: {
- _9 = const Less;
- StorageDead(_7);
- StorageDead(_6);
- StorageDead(_5);
- _10 = Option::<std::cmp::Ordering>::Some(move _9);
- StorageDead(_9);
- StorageDead(_4);
- StorageDead(_3);
- goto -> bb11;
- }
-
- bb11: {
- StorageLive(_18);
- _18 = _10;
- _0 = _10;
- StorageDead(_18);
- goto -> bb12;
+ bb2: {
+ _0 = _8;
+ goto -> bb3;
}
- bb12: {
+ bb3: {
return;
}
}
From 8d5977d6af03af18c7851025b9ca70d0f2d12c86 Mon Sep 17 00:00:00 2001
From: Scott McMurray <scottmcm@users.noreply.github.com>
Date: Sun, 24 Mar 2024 10:01:37 -0700
Subject: [PATCH 3/4] Slightly simplify the `iN::partial_cmp` MIR
This saves some debug and scope metadata in every single function that calls it.
Normally wouldn't be worth it, but with the derives there's *so* many of these.
---
library/core/src/cmp.rs | 21 ++++++++++++-------
....{impl#0}-partial_cmp.PreCodegen.after.mir | 18 +++++-----------
2 files changed, 19 insertions(+), 20 deletions(-)
diff --git a/library/core/src/cmp.rs b/library/core/src/cmp.rs
index 7581c6dd759c3..e3b7cf84920ea 100644
--- a/library/core/src/cmp.rs
+++ b/library/core/src/cmp.rs
@@ -1548,7 +1548,14 @@ mod impls {
impl PartialOrd for $t {
#[inline]
fn partial_cmp(&self, other: &$t) -> Option<Ordering> {
- Some(self.cmp(other))
+ #[cfg(bootstrap)]
+ {
+ Some(self.cmp(other))
+ }
+ #[cfg(not(bootstrap))]
+ {
+ Some(crate::intrinsics::three_way_compare(*self, *other))
+ }
}
#[inline(always)]
fn lt(&self, other: &$t) -> bool { (*self) < (*other) }
@@ -1566,12 +1573,12 @@ mod impls {
fn cmp(&self, other: &$t) -> Ordering {
#[cfg(bootstrap)]
{
- // The order here is important to generate more optimal assembly.
- // See <https://github.com/rust-lang/rust/issues/63758> for more info.
- if *self < *other { Less }
- else if *self == *other { Equal }
- else { Greater }
- }
+ // The order here is important to generate more optimal assembly.
+ // See <https://github.com/rust-lang/rust/issues/63758> for more info.
+ if *self < *other { Less }
+ else if *self == *other { Equal }
+ else { Greater }
+ }
#[cfg(not(bootstrap))]
{
crate::intrinsics::three_way_compare(*self, *other)
diff --git a/tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir
index 7ce2d229d5474..a6c644259124a 100644
--- a/tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir
+++ b/tests/mir-opt/pre-codegen/derived_ord.{impl#0}-partial_cmp.PreCodegen.after.mir
@@ -16,24 +16,16 @@ fn <impl at $DIR/derived_ord.rs:6:10: 6:20>::partial_cmp(_1: &MultiField, _2: &M
scope 2 (inlined std::cmp::impls::<impl PartialOrd for char>::partial_cmp) {
debug self => _3;
debug other => _4;
+ let mut _5: char;
+ let mut _6: char;
let mut _7: std::cmp::Ordering;
- scope 3 (inlined std::cmp::impls::<impl Ord for char>::cmp) {
- debug self => _3;
- debug other => _4;
- let mut _5: char;
- let mut _6: char;
- }
}
- scope 4 (inlined std::cmp::impls::<impl PartialOrd for i16>::partial_cmp) {
+ scope 3 (inlined std::cmp::impls::<impl PartialOrd for i16>::partial_cmp) {
debug self => _10;
debug other => _11;
+ let mut _12: i16;
+ let mut _13: i16;
let mut _14: std::cmp::Ordering;
- scope 5 (inlined std::cmp::impls::<impl Ord for i16>::cmp) {
- debug self => _10;
- debug other => _11;
- let mut _12: i16;
- let mut _13: i16;
- }
}
bb0: {
From c59e93c753ba7e5945fe95f4d1785b09300aa0e0 Mon Sep 17 00:00:00 2001
From: Scott McMurray <scottmcm@users.noreply.github.com>
Date: Sun, 24 Mar 2024 17:13:26 -0700
Subject: [PATCH 4/4] Address PR feedback
---
compiler/rustc_middle/src/mir/syntax.rs | 8 +++++++
library/core/src/cmp.rs | 3 +++
library/core/tests/intrinsics.rs | 32 +++++++++++++++----------
3 files changed, 31 insertions(+), 12 deletions(-)
diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs
index a33de2efe2fad..67f881e862a16 100644
--- a/compiler/rustc_middle/src/mir/syntax.rs
+++ b/compiler/rustc_middle/src/mir/syntax.rs
@@ -1445,6 +1445,14 @@ pub enum BinOp {
/// The `>` operator (greater than)
Gt,
/// The `<=>` operator (three-way comparison, like `Ord::cmp`)
+ ///
+ /// This is supported only on the integer types and `char`, always returning
+ /// [`rustc_hir::LangItem::OrderingEnum`] (aka [`std::cmp::Ordering`]).
+ ///
+ /// [`Rvalue::BinaryOp`]`(BinOp::Cmp, A, B)` returns
+ /// - `Ordering::Less` (`-1_i8`, as a Scalar) if `A < B`
+ /// - `Ordering::Equal` (`0_i8`, as a Scalar) if `A == B`
+ /// - `Ordering::Greater` (`+1_i8`, as a Scalar) if `A > B`
Cmp,
/// The `ptr.offset` operator
Offset,
diff --git a/library/core/src/cmp.rs b/library/core/src/cmp.rs
index e3b7cf84920ea..7f9c041f6918e 100644
--- a/library/core/src/cmp.rs
+++ b/library/core/src/cmp.rs
@@ -376,6 +376,9 @@ pub struct AssertParamIsEq<T: Eq + ?Sized> {
/// ```
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
#[stable(feature = "rust1", since = "1.0.0")]
+// This is a lang item only so that `BinOp::Cmp` in MIR can return it.
+// It has no special behaviour, but does require that the three variants
+// `Less`/`Equal`/`Greater` remain `-1_i8`/`0_i8`/`+1_i8` respectively.
#[cfg_attr(not(bootstrap), lang = "Ordering")]
#[repr(i8)]
pub enum Ordering {
diff --git a/library/core/tests/intrinsics.rs b/library/core/tests/intrinsics.rs
index a9ef5415ada6f..eb1e1a0b9b1dd 100644
--- a/library/core/tests/intrinsics.rs
+++ b/library/core/tests/intrinsics.rs
@@ -103,18 +103,26 @@ fn test_const_deallocate_at_runtime() {
#[cfg(not(bootstrap))]
#[test]
fn test_three_way_compare_in_const_contexts() {
- use core::cmp::Ordering::*;
+ use core::cmp::Ordering::{self, *};
use core::intrinsics::three_way_compare;
- const {
- assert!(Less as i8 == three_way_compare(123_u16, 456) as _);
- assert!(Equal as i8 == three_way_compare(456_u16, 456) as _);
- assert!(Greater as i8 == three_way_compare(789_u16, 456) as _);
- assert!(Less as i8 == three_way_compare('A', 'B') as _);
- assert!(Equal as i8 == three_way_compare('B', 'B') as _);
- assert!(Greater as i8 == three_way_compare('C', 'B') as _);
- assert!(Less as i8 == three_way_compare(-123_i16, 456) as _);
- assert!(Equal as i8 == three_way_compare(456_i16, 456) as _);
- assert!(Greater as i8 == three_way_compare(123_i16, -456) as _);
- }
+ const UNSIGNED_LESS: Ordering = three_way_compare(123_u16, 456);
+ const UNSIGNED_EQUAL: Ordering = three_way_compare(456_u16, 456);
+ const UNSIGNED_GREATER: Ordering = three_way_compare(789_u16, 456);
+ const CHAR_LESS: Ordering = three_way_compare('A', 'B');
+ const CHAR_EQUAL: Ordering = three_way_compare('B', 'B');
+ const CHAR_GREATER: Ordering = three_way_compare('C', 'B');
+ const SIGNED_LESS: Ordering = three_way_compare(123_i64, 456);
+ const SIGNED_EQUAL: Ordering = three_way_compare(456_i64, 456);
+ const SIGNED_GREATER: Ordering = three_way_compare(789_i64, 456);
+
+ assert_eq!(UNSIGNED_LESS, Less);
+ assert_eq!(UNSIGNED_EQUAL, Equal);
+ assert_eq!(UNSIGNED_GREATER, Greater);
+ assert_eq!(CHAR_LESS, Less);
+ assert_eq!(CHAR_EQUAL, Equal);
+ assert_eq!(CHAR_GREATER, Greater);
+ assert_eq!(SIGNED_LESS, Less);
+ assert_eq!(SIGNED_EQUAL, Equal);
+ assert_eq!(SIGNED_GREATER, Greater);
}