Add ADX-related intrinsics (#631)

Closes #322

Author: alexcrichton

coresimd/x86/adx.rs

@@ -0,0 +1,46 @@
+#[cfg(test)]
+use stdsimd_test::assert_instr;
+
+#[allow(improper_ctypes)]
+extern "unadjusted" {
+    #[link_name = "llvm.x86.addcarry.u32"]
+    fn llvm_addcarry_u32(a: u8, b: u32, c: u32) -> (u8, u32);
+    #[link_name = "llvm.x86.subborrow.u32"]
+    fn llvm_subborrow_u32(a: u8, b: u32, c: u32) -> (u8, u32);
+}
+
+/// Add unsigned 32-bit integers a and b with unsigned 8-bit carry-in c_in
+/// (carry flag), and store the unsigned 32-bit result in out, and the carry-out
+/// is returned (carry or overflow flag).
+#[inline]
+#[cfg_attr(test, assert_instr(adc))]
+#[stable(feature = "simd_x86_adx", since = "1.33.0")]
+pub unsafe fn _addcarry_u32(c_in: u8, a: u32, b: u32, out: &mut u32) -> u8 {
+    let (a, b) = llvm_addcarry_u32(c_in, a, b);
+    *out = b;
+    a
+}
+
+/// Add unsigned 32-bit integers a and b with unsigned 8-bit carry-in c_in
+/// (carry or overflow flag), and store the unsigned 32-bit result in out, and
+/// the carry-out is returned (carry or overflow flag).
+#[inline]
+#[target_feature(enable = "adx")]
+#[cfg_attr(test, assert_instr(adc))]
+#[stable(feature = "simd_x86_adx", since = "1.33.0")]
+#[cfg(not(stage0))]
+pub unsafe fn _addcarryx_u32(c_in: u8, a: u32, b: u32, out: &mut u32) -> u8 {
+    _addcarry_u32(c_in, a, b, out)
+}
+
+/// Add unsigned 32-bit integers a and b with unsigned 8-bit carry-in c_in
+/// (carry or overflow flag), and store the unsigned 32-bit result in out, and
+/// the carry-out is returned (carry or overflow flag).
+#[inline]
+#[cfg_attr(test, assert_instr(sbb))]
+#[stable(feature = "simd_x86_adx", since = "1.33.0")]
+pub unsafe fn _subborrow_u32(c_in: u8, a: u32, b: u32, out: &mut u32) -> u8 {
+    let (a, b) = llvm_subborrow_u32(c_in, a, b);
+    *out = b;
+    a
+}

coresimd/x86/mod.rs

@@ -593,6 +593,9 @@ pub use self::rdrand::*;
 mod sha;
 pub use self::sha::*;
 
+mod adx;
+pub use self::adx::*;
+
 #[cfg(test)]
 use stdsimd_test::assert_instr;
 

coresimd/x86_64/adx.rs

@@ -0,0 +1,46 @@
+#[cfg(test)]
+use stdsimd_test::assert_instr;
+
+#[allow(improper_ctypes)]
+extern "unadjusted" {
+    #[link_name = "llvm.x86.addcarry.u64"]
+    fn llvm_addcarry_u64(a: u8, b: u64, c: u64) -> (u8, u64);
+    #[link_name = "llvm.x86.subborrow.u64"]
+    fn llvm_subborrow_u64(a: u8, b: u64, c: u64) -> (u8, u64);
+}
+
+/// Add unsigned 64-bit integers a and b with unsigned 8-bit carry-in c_in
+/// (carry flag), and store the unsigned 64-bit result in out, and the carry-out
+/// is returned (carry or overflow flag).
+#[inline]
+#[cfg_attr(test, assert_instr(adc))]
+#[stable(feature = "simd_x86_adx", since = "1.33.0")]
+pub unsafe fn _addcarry_u64(c_in: u8, a: u64, b: u64, out: &mut u64) -> u8 {
+    let (a, b) = llvm_addcarry_u64(c_in, a, b);
+    *out = b;
+    a
+}
+
+/// Add unsigned 64-bit integers a and b with unsigned 8-bit carry-in c_in
+/// (carry or overflow flag), and store the unsigned 64-bit result in out, and
+/// the carry-out is returned (carry or overflow flag).
+#[inline]
+#[target_feature(enable = "adx")]
+#[cfg_attr(test, assert_instr(adc))]
+#[stable(feature = "simd_x86_adx", since = "1.33.0")]
+#[cfg(not(stage0))]
+pub unsafe fn _addcarryx_u64(c_in: u8, a: u64, b: u64, out: &mut u64) -> u8 {
+    _addcarry_u64(c_in, a, b, out)
+}
+
+/// Add unsigned 64-bit integers a and b with unsigned 8-bit carry-in c_in
+/// (carry or overflow flag), and store the unsigned 64-bit result in out, and
+/// the carry-out is returned (carry or overflow flag).
+#[inline]
+#[cfg_attr(test, assert_instr(sbb))]
+#[stable(feature = "simd_x86_adx", since = "1.33.0")]
+pub unsafe fn _subborrow_u64(c_in: u8, a: u64, b: u64, out: &mut u64) -> u8 {
+    let (a, b) = llvm_subborrow_u64(c_in, a, b);
+    *out = b;
+    a
+}

coresimd/x86_64/mod.rs

@@ -41,3 +41,6 @@ pub use self::rdrand::*;
 
 mod cmpxchg16b;
 pub use self::cmpxchg16b::*;
+
+mod adx;
+pub use self::adx::*;

crates/coresimd/src/lib.rs

@@ -37,7 +37,8 @@
     mips_target_feature,
     powerpc_target_feature,
     wasm_target_feature,
-    abi_unadjusted
+    abi_unadjusted,
+    adx_target_feature
 )]
 // NB: When running nvptx/nvptx64 cross tests, enabling "integer_atomics" yields
 // a compile-time error: 'unknown feature `integer_atomics`'. This ought to be

crates/stdsimd-verify/tests/x86-intel.rs

@@ -240,8 +240,12 @@ fn matches(rust: &Function, intel: &Intrinsic) -> Result<(), String> {
     // ensuring that we've actually enabled the right instruction
     // set for this intrinsic.
     match rust.name {
-        "_bswap" => {}
-        "_bswap64" => {}
+        "_bswap" | "_bswap64" => {}
+
+        // These don't actually have a target feature unlike their brethren with
+        // the `x` inside the name which requires adx
+        "_addcarry_u32" | "_addcarry_u64" | "_subborrow_u32" | "_subborrow_u64" => {}
+
         _ => {
             if intel.cpuid.is_empty() {
                 bail!("missing cpuid for {}", rust.name);

stdsimd/arch/detect/arch/x86.rs

@@ -230,6 +230,10 @@ macro_rules! is_x86_feature_detected {
         cfg!(target_feature = "cmpxchg16b") || $crate::arch::detect::check_for(
             $crate::arch::detect::Feature::cmpxchg16b)
     };
+    ("adx") => {
+        cfg!(target_feature = "adx") || $crate::arch::detect::check_for(
+            $crate::arch::detect::Feature::adx)
+    };
     ($t:tt) => {
         compile_error!(concat!("unknown target feature: ", $t))
     };
@@ -322,4 +326,6 @@ pub enum Feature {
     xsavec,
     /// CMPXCH16B, a 16-byte compare-and-swap instruction
     cmpxchg16b,
+    /// ADX, Intel ADX (Multi-Precision Add-Carry Instruction Extensions)
+    adx,
 }

stdsimd/arch/detect/os/x86.rs

@@ -124,6 +124,7 @@ fn detect_features() -> cache::Initializer {
         enable(proc_info_ecx, 1, Feature::pclmulqdq);
         enable(proc_info_ecx, 30, Feature::rdrand);
         enable(extended_features_ebx, 18, Feature::rdseed);
+        enable(extended_features_ebx, 19, Feature::adx);
         enable(proc_info_edx, 4, Feature::tsc);
         enable(proc_info_edx, 23, Feature::mmx);
         enable(proc_info_edx, 24, Feature::fxsr);
@@ -290,6 +291,7 @@ mod tests {
         println!("xsaves: {:?}", is_x86_feature_detected!("xsaves"));
         println!("xsavec: {:?}", is_x86_feature_detected!("xsavec"));
         println!("cmpxchg16b: {:?}", is_x86_feature_detected!("cmpxchg16b"));
+        println!("adx: {:?}", is_x86_feature_detected!("adx"));
     }
 
     #[test]
@@ -350,5 +352,9 @@ mod tests {
             is_x86_feature_detected!("cmpxchg16b"),
             information.cmpxchg16b(),
         );
+        assert_eq!(
+            is_x86_feature_detected!("adx"),
+            information.adx(),
+        );
     }
 }