expose expect intrinsics + mark push slow path

src/libcollections/vec.rs

@@ -1687,7 +1687,7 @@ impl<T> MutableSeq<T> for Vec<T> {
             unsafe { mem::forget(value); }
             return
         }
-        if self.len == self.cap {
+        if unlikely!(self.len == self.cap) {
             let old_size = self.cap * mem::size_of::<T>();
             let size = max(old_size, 2 * mem::size_of::<T>()) * 2;
             if old_size > size { fail!("capacity overflow") }

src/libcore/intrinsics.rs

@@ -616,8 +616,37 @@ extern "rust-intrinsic" {
     pub fn u32_mul_with_overflow(x: u32, y: u32) -> (u32, bool);
     /// Performs checked `u64` multiplication.
     pub fn u64_mul_with_overflow(x: u64, y: u64) -> (u64, bool);
+
+    /// Inform the optimizer that `expected_val` is the expected (most probable) value of `val`.
+    #[cfg(not(stage0))]
+    pub fn expect8(val: u8, expected_val: u8) -> u8;
+    /// Inform the optimizer that `expected_val` is the expected (most probable) value of `val`.
+    #[cfg(not(stage0))]
+    pub fn expect16(val: u16, expected_val: u16) -> u16;
+    /// Inform the optimizer that `expected_val` is the expected (most probable) value of `val`.
+    #[cfg(not(stage0))]
+    pub fn expect32(val: u32, expected_val: u32) -> u32;
+    /// Inform the optimizer that `expected_val` is the expected (most probable) value of `val`.
+    #[cfg(not(stage0))]
+    pub fn expect64(val: u64, expected_val: u64) -> u64;
 }
 
+/// Inform the optimizer that `expected_val` is the expected (most probable) value of `val`.
+#[cfg(stage0)]
+#[inline(always)]
+pub unsafe fn expect8(val: u8, _expected_val: u8) -> u8 { val }
+/// Inform the optimizer that `expected_val` is the expected (most probable) value of `val`.
+#[cfg(stage0)]
+#[inline(always)]
+pub unsafe fn expect16(val: u16, _expected_val: u16) -> u16 { val }
+/// Inform the optimizer that `expected_val` is the expected (most probable) value of `val`.
+#[cfg(stage0)]
+#[inline(always)]
+pub unsafe fn expect32(val: u32, _expected_val: u32) -> u32 { val }
+/// Inform the optimizer that `expected_val` is the expected (most probable) value of `val`.
+#[cfg(stage0)]
+#[inline(always)]
+pub unsafe fn expect64(val: u64, _expected_val: u64) -> u64 { val }
 
 /// `TypeId` represents a globally unique identifier for a type
 #[lang="type_id"] // This needs to be kept in lockstep with the code in trans/intrinsic.rs and

src/libcore/macros.rs

@@ -131,3 +131,27 @@ macro_rules! write(
 
 #[macro_export]
 macro_rules! unreachable( () => (fail!("unreachable code")) )
+
+// FIXME: #17316: should be marked experimental
+/// Inform the optimizer that a boolean condition is likely.
+#[macro_export]
+macro_rules! likely(
+    ($val:expr) => {
+        {
+            let x: bool = $val;
+            unsafe { ::core::intrinsics::expect8(x as u8, 1) != 0 }
+        }
+    }
+)
+
+// FIXME: #17316: should be marked experimental
+/// Inform the optimizer that a boolean condition is unlikely.
+#[macro_export]
+macro_rules! unlikely(
+    ($val:expr) => {
+        {
+            let x: bool = $val;
+            unsafe { ::core::intrinsics::expect8(x as u8, 0) != 0 }
+        }
+    }
+)

src/librustc/middle/trans/context.rs

@@ -860,6 +860,10 @@ fn declare_intrinsic(ccx: &CrateContext, key: & &'static str) -> Option<ValueRef
     ifn!("llvm.lifetime.end" fn(t_i64, i8p) -> void);
 
     ifn!("llvm.expect.i1" fn(i1, i1) -> i1);
+    ifn!("llvm.expect.i8" fn(t_i8, t_i8) -> t_i8);
+    ifn!("llvm.expect.i16" fn(t_i16, t_i16) -> t_i16);
+    ifn!("llvm.expect.i32" fn(t_i32, t_i32) -> t_i32);
+    ifn!("llvm.expect.i64" fn(t_i64, t_i64) -> t_i64);
 
     // Some intrinsics were introduced in later versions of LLVM, but they have
     // fallbacks in libc or libm and such. Currently, all of these intrinsics

src/librustc/middle/trans/intrinsic.rs

@@ -81,6 +81,10 @@ pub fn get_simple_intrinsic(ccx: &CrateContext, item: &ast::ForeignItem) -> Opti
         "bswap16" => "llvm.bswap.i16",
         "bswap32" => "llvm.bswap.i32",
         "bswap64" => "llvm.bswap.i64",
+        "expect8" => "llvm.expect.i8",
+        "expect16" => "llvm.expect.i16",
+        "expect32" => "llvm.expect.i32",
+        "expect64" => "llvm.expect.i64",
         _ => return None
     };
     Some(ccx.get_intrinsic(&name))

src/librustc/middle/typeck/check/mod.rs

@@ -5811,6 +5811,11 @@ pub fn check_intrinsic_type(ccx: &CrateCtxt, it: &ast::ForeignItem) {
 
             "return_address" => (0, vec![], ty::mk_imm_ptr(tcx, ty::mk_u8())),
 
+            "expect8" => (0, vec![ty::mk_u8(), ty::mk_u8()], ty::mk_u8()),
+            "expect16" => (0, vec![ty::mk_u16(), ty::mk_u16()], ty::mk_u16()),
+            "expect32" => (0, vec![ty::mk_u32(), ty::mk_u32()], ty::mk_u32()),
+            "expect64" => (0, vec![ty::mk_u64(), ty::mk_u64()], ty::mk_u64()),
+
             ref other => {
                 span_err!(tcx.sess, it.span, E0093,
                     "unrecognized intrinsic function: `{}`", *other);

src/libstd/macros.rs

@@ -214,6 +214,30 @@ macro_rules! unreachable(
     () => (fail!("internal error: entered unreachable code"))
 )
 
+// FIXME: #17316: should be marked experimental
+/// Inform the optimizer that a boolean condition is likely.
+#[macro_export]
+macro_rules! likely(
+    ($val:expr) => {
+        {
+            let x: bool = $val;
+            unsafe { ::std::intrinsics::expect8(x as u8, 1) != 0 }
+        }
+    }
+)
+
+// FIXME: #17316: should be marked experimental
+/// Inform the optimizer that a boolean condition is unlikely.
+#[macro_export]
+macro_rules! unlikely(
+    ($val:expr) => {
+        {
+            let x: bool = $val;
+            unsafe { ::std::intrinsics::expect8(x as u8, 0) != 0 }
+        }
+    }
+)
+
 /// A standardised placeholder for marking unfinished code. It fails with the
 /// message `"not yet implemented"` when executed.
 #[macro_export]