diff --git a/ci/run.sh b/ci/run.sh index 28d53c5375..da64602ce8 100755 --- a/ci/run.sh +++ b/ci/run.sh @@ -124,6 +124,10 @@ case ${TARGET} in export RUSTFLAGS="${RUSTFLAGS} -C target-feature=+msa" cargo_test "${PROFILE}" ;; + s390x*) + export RUSTFLAGS="${RUSTFLAGS} -C target-feature=+vector-enhancements-1" + cargo_test "${PROFILE}" + ;; powerpc64*) # We don't build the ppc 32-bit targets with these - these targets # are mostly unsupported for now. diff --git a/crates/core_arch/src/s390x/macros.rs b/crates/core_arch/src/s390x/macros.rs index 92faf387bd..fd260bbc74 100644 --- a/crates/core_arch/src/s390x/macros.rs +++ b/crates/core_arch/src/s390x/macros.rs @@ -2,6 +2,13 @@ #![allow(unused_imports)] // FIXME remove when more tests are added macro_rules! test_impl { + ($fun:ident ($($v:ident : $ty:ty),*) -> $r:ty [$call:ident, _]) => { + #[inline] + #[target_feature(enable = "vector")] + pub unsafe fn $fun ($($v : $ty),*) -> $r { + $call ($($v),*) + } + }; ($fun:ident +($($v:ident : $ty:ty),*) -> $r:ty [$call:ident, $instr:ident]) => { #[inline] #[target_feature(enable = "vector")] @@ -10,6 +17,22 @@ macro_rules! test_impl { transmute($call ($($v),*)) } }; + ($fun:ident +($($v:ident : $ty:ty),*) -> $r:ty [$call:ident, $tf:literal $instr:ident]) => { + #[inline] + #[target_feature(enable = "vector")] + #[cfg_attr(all(test, target_feature = $tf), assert_instr($instr))] + pub unsafe fn $fun ($($v : $ty),*) -> $r { + transmute($call ($($v),*)) + } + }; + ($fun:ident ($($v:ident : $ty:ty),*) -> $r:ty [$call:ident, $tf:literal $instr:ident]) => { + #[inline] + #[target_feature(enable = "vector")] + #[cfg_attr(all(test, target_feature = $tf), assert_instr($instr))] + pub unsafe fn $fun ($($v : $ty),*) -> $r { + $call ($($v),*) + } + }; ($fun:ident ($($v:ident : $ty:ty),*) -> $r:ty [$call:ident, $instr:ident]) => { #[inline] #[target_feature(enable = "vector")] diff --git a/crates/core_arch/src/s390x/vector.rs b/crates/core_arch/src/s390x/vector.rs index da9ea0157d..1b59a4c474 100644 --- a/crates/core_arch/src/s390x/vector.rs +++ b/crates/core_arch/src/s390x/vector.rs @@ -73,6 +73,15 @@ unsafe extern "unadjusted" { #[link_name = "llvm.umin.v8i16"] fn vmnlh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short; #[link_name = "llvm.umin.v4i32"] fn vmnlf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int; #[link_name = "llvm.umin.v2i64"] fn vmnlg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long; + + #[link_name = "llvm.nearbyint.v4f32"] fn nearbyint_v4f32(a: vector_float) -> vector_float; + #[link_name = "llvm.nearbyint.v2f64"] fn nearbyint_v2f64(a: vector_double) -> vector_double; + + #[link_name = "llvm.rint.v4f32"] fn rint_v4f32(a: vector_float) -> vector_float; + #[link_name = "llvm.rint.v2f64"] fn rint_v2f64(a: vector_double) -> vector_double; + + #[link_name = "llvm.roundeven.v4f32"] fn roundeven_v4f32(a: vector_float) -> vector_float; + #[link_name = "llvm.roundeven.v2f64"] fn roundeven_v2f64(a: vector_double) -> vector_double; } impl_from! { i8x16, u8x16, i16x8, u16x8, i32x4, u32x4, i64x2, u64x2, f32x4, f64x2 } @@ -154,8 +163,7 @@ mod sealed { #[inline] #[target_feature(enable = "vector")] - // FIXME: "vfasb" is part of vector enhancements 1, add a test for it when possible - // #[cfg_attr(test, assert_instr(vfasb))] + #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vfasb))] pub unsafe fn va_float(a: vector_float, b: vector_float) -> vector_float { transmute(simd_add(a, b)) } @@ -238,8 +246,7 @@ mod sealed { #[inline] #[target_feature(enable = "vector")] - // FIXME: "vfssb" is part of vector enhancements 1, add a test for it when possible - // #[cfg_attr(test, assert_instr(vfasb))] + #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vfssb))] pub unsafe fn vs_float(a: vector_float, b: vector_float) -> vector_float { transmute(simd_sub(a, b)) } @@ -326,6 +333,12 @@ mod sealed { impl_vec_trait! { [VectorMax vec_max] ~(vmxlb, vmxb, vmxlh, vmxh, vmxlf, vmxf, vmxlg, vmxg) } + test_impl! { vec_vfmaxsb (a: vector_float, b: vector_float) -> vector_float [simd_fmax, "vector-enhancements-1" vfmaxsb ] } + test_impl! { vec_vfmaxdb (a: vector_double, b: vector_double) -> vector_double [simd_fmax, "vector-enhancements-1" vfmaxdb] } + + impl_vec_trait!([VectorMax vec_max] vec_vfmaxsb (vector_float, vector_float) -> vector_float); + impl_vec_trait!([VectorMax vec_max] vec_vfmaxdb (vector_double, vector_double) -> vector_double); + #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorMin { type Result; @@ -344,6 +357,12 @@ mod sealed { impl_vec_trait! { [VectorMin vec_min] ~(vmxlb, vmxb, vmxlh, vmxh, vmxlf, vmxf, vmxlg, vmxg) } + test_impl! { vec_vfminsb (a: vector_float, b: vector_float) -> vector_float [simd_fmin, "vector-enhancements-1" vfminsb] } + test_impl! { vec_vfmindb (a: vector_double, b: vector_double) -> vector_double [simd_fmin, "vector-enhancements-1" vfmindb] } + + impl_vec_trait!([VectorMin vec_min] vec_vfminsb (vector_float, vector_float) -> vector_float); + impl_vec_trait!([VectorMin vec_min] vec_vfmindb (vector_double, vector_double) -> vector_double); + #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorAbs { unsafe fn vec_abs(self) -> Self; @@ -366,21 +385,8 @@ mod sealed { impl_abs! { vec_abs_i32, i32x4 } impl_abs! { vec_abs_i64, i64x2 } - #[inline] - #[target_feature(enable = "vector")] - unsafe fn vec_abs_f32(v: vector_float) -> vector_float { - let v: u32x4 = transmute(v); - - transmute(simd_and(v, u32x4::splat(0x7FFFFFFF))) - } - - #[inline] - #[target_feature(enable = "vector")] - unsafe fn vec_abs_f64(v: vector_double) -> vector_double { - let v: u64x2 = transmute(v); - - transmute(simd_and(v, u64x2::splat(0x7FFFFFFF_FFFFFFFF))) - } + test_impl! { vec_abs_f32 (v: vector_float) -> vector_float [ simd_fabs, "vector-enhancements-1" vflpsb ] } + test_impl! { vec_abs_f64 (v: vector_double) -> vector_double [ simd_fabs, vflpdb ] } impl_vec_trait! { [VectorAbs vec_abs] vec_abs_f32 (vector_float) } impl_vec_trait! { [VectorAbs vec_abs] vec_abs_f64 (vector_double) } @@ -516,10 +522,15 @@ mod sealed { test_impl! { vec_ctzf_unsigned +(a: vector_unsigned_int) -> vector_unsigned_int [simd_cttz, vctzf] } test_impl! { vec_ctzg_unsigned +(a: vector_unsigned_long_long) -> vector_unsigned_long_long [simd_cttz, vctzg] } - // FIXME(vector-enhancements-1) other integer types are emulated, but get their own - // instructions in later facilities. Add tests when possible. - test_impl! { vec_popcnt_signed +(a: vector_signed_char) -> vector_signed_char [simd_ctpop, vpopctb] } - test_impl! { vec_popcnt_unsigned +(a: vector_unsigned_char) -> vector_unsigned_char [simd_ctpop, vpopctb] } + test_impl! { vec_vpopctb_signed +(a: vector_signed_char) -> vector_signed_char [simd_ctpop, vpopctb] } + test_impl! { vec_vpopcth_signed +(a: vector_signed_short) -> vector_signed_short [simd_ctpop, "vector-enhancements-1" vpopcth] } + test_impl! { vec_vpopctf_signed +(a: vector_signed_int) -> vector_signed_int [simd_ctpop, "vector-enhancements-1" vpopctf] } + test_impl! { vec_vpopctg_signed +(a: vector_signed_long_long) -> vector_signed_long_long [simd_ctpop, "vector-enhancements-1" vpopctg] } + + test_impl! { vec_vpopctb_unsigned +(a: vector_unsigned_char) -> vector_unsigned_char [simd_ctpop, vpopctb] } + test_impl! { vec_vpopcth_unsigned +(a: vector_unsigned_short) -> vector_unsigned_short [simd_ctpop, "vector-enhancements-1" vpopcth] } + test_impl! { vec_vpopctf_unsigned +(a: vector_unsigned_int) -> vector_unsigned_int [simd_ctpop, "vector-enhancements-1" vpopctf] } + test_impl! { vec_vpopctg_unsigned +(a: vector_unsigned_long_long) -> vector_unsigned_long_long [simd_ctpop, "vector-enhancements-1" vpopctg] } #[unstable(feature = "stdarch_s390x", issue = "135681")] pub trait VectorAnd { @@ -547,7 +558,7 @@ mod sealed { #[inline] #[target_feature(enable = "vector")] - // FIXME(vector-enhancements-1) #[cfg_attr(test, assert_instr(vno))] + #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vno))] unsafe fn nor(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char { let a: u8x16 = transmute(a); let b: u8x16 = transmute(b); @@ -564,7 +575,7 @@ mod sealed { #[inline] #[target_feature(enable = "vector")] - // FIXME(vector-enhancements-1) #[cfg_attr(test, assert_instr(vnn))] + #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vnn))] unsafe fn nand(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char { let a: u8x16 = transmute(a); let b: u8x16 = transmute(b); @@ -581,7 +592,7 @@ mod sealed { #[inline] #[target_feature(enable = "vector")] - // FIXME(vector-enhancements-1) #[cfg_attr(test, assert_instr(vnx))] + #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vnx))] unsafe fn eqv(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char { let a: u8x16 = transmute(a); let b: u8x16 = transmute(b); @@ -598,7 +609,7 @@ mod sealed { #[inline] #[target_feature(enable = "vector")] - // FIXME(vector-enhancements-1) #[cfg_attr(test, assert_instr(vnc))] + #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vnc))] unsafe fn andc(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char { let a = transmute(a); let b = transmute(b); @@ -615,7 +626,7 @@ mod sealed { #[inline] #[target_feature(enable = "vector")] - // FIXME(vector-enhancements-1) #[cfg_attr(test, assert_instr(voc))] + #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(voc))] unsafe fn orc(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char { let a = transmute(a); let b = transmute(b); @@ -629,6 +640,67 @@ mod sealed { } impl_vec_trait! { [VectorOrc vec_orc]+ 2c (orc) } + + test_impl! { vec_roundc_f32 (a: vector_float) -> vector_float [nearbyint_v4f32, "vector-enhancements-1" vfisb] } + test_impl! { vec_roundc_f64 (a: vector_double) -> vector_double [nearbyint_v2f64, vfidb] } + + // FIXME(llvm) roundeven does not yet lower to vfidb (but should in the future) + test_impl! { vec_round_f32 (a: vector_float) -> vector_float [roundeven_v4f32, _] } + test_impl! { vec_round_f64 (a: vector_double) -> vector_double [roundeven_v2f64, _] } + + test_impl! { vec_rint_f32 (a: vector_float) -> vector_float [rint_v4f32, "vector-enhancements-1" vfisb] } + test_impl! { vec_rint_f64 (a: vector_double) -> vector_double [rint_v2f64, vfidb] } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorRoundc { + unsafe fn vec_roundc(self) -> Self; + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorRound { + unsafe fn vec_round(self) -> Self; + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorRint { + unsafe fn vec_rint(self) -> Self; + } + + impl_vec_trait! { [VectorRoundc vec_roundc] vec_roundc_f32 (vector_float) } + impl_vec_trait! { [VectorRoundc vec_roundc] vec_roundc_f64 (vector_double) } + + impl_vec_trait! { [VectorRound vec_round] vec_round_f32 (vector_float) } + impl_vec_trait! { [VectorRound vec_round] vec_round_f64 (vector_double) } + + impl_vec_trait! { [VectorRint vec_rint] vec_rint_f32 (vector_float) } + impl_vec_trait! { [VectorRint vec_rint] vec_rint_f64 (vector_double) } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorTrunc { + // same as vec_roundz + unsafe fn vec_trunc(self) -> Self; + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorCeil { + // same as vec_roundp + unsafe fn vec_ceil(self) -> Self; + } + + #[unstable(feature = "stdarch_s390x", issue = "135681")] + pub trait VectorFloor { + // same as vec_roundm + unsafe fn vec_floor(self) -> Self; + } + + impl_vec_trait! { [VectorTrunc vec_trunc] simd_trunc (vector_float) } + impl_vec_trait! { [VectorTrunc vec_trunc] simd_trunc (vector_double) } + + impl_vec_trait! { [VectorCeil vec_ceil] simd_ceil (vector_float) } + impl_vec_trait! { [VectorCeil vec_ceil] simd_ceil (vector_double) } + + impl_vec_trait! { [VectorFloor vec_floor] simd_floor (vector_float) } + impl_vec_trait! { [VectorFloor vec_floor] simd_floor (vector_double) } } /// Vector element-wise addition. @@ -843,6 +915,126 @@ where a.vec_orc(b) } +/// Vector floor. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_floor(a: T) -> T +where + T: sealed::VectorFloor, +{ + a.vec_floor() +} + +/// Vector ceil. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_ceil(a: T) -> T +where + T: sealed::VectorCeil, +{ + a.vec_ceil() +} + +/// Returns a vector containing the truncated values of the corresponding elements of the given vector. +/// Each element of the result contains the value of the corresponding element of a, truncated to an integral value. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_trunc(a: T) -> T +where + T: sealed::VectorTrunc, +{ + a.vec_trunc() +} + +/// Returns a vector containing the rounded values to the nearest representable floating-point integer, +/// using IEEE round-to-nearest rounding, of the corresponding elements of the given vector +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_round(a: T) -> T +where + T: sealed::VectorRound, +{ + a.vec_round() +} + +/// Returns a vector by using the current rounding mode to round every +/// floating-point element in the given vector to integer. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_roundc(a: T) -> T +where + T: sealed::VectorRoundc, +{ + a.vec_roundc() +} + +/// Returns a vector containing the largest representable floating-point integral values less +/// than or equal to the values of the corresponding elements of the given vector. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_roundm(a: T) -> T +where + T: sealed::VectorFloor, +{ + // the IBM docs note + // + // > vec_roundm provides the same functionality as vec_floor, except that vec_roundz would not trigger the IEEE-inexact exception. + // + // but in practice `vec_floor` also does not trigger that exception, so both are equivalent + a.vec_floor() +} + +/// Returns a vector containing the smallest representable floating-point integral values greater +/// than or equal to the values of the corresponding elements of the given vector. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_roundp(a: T) -> T +where + T: sealed::VectorCeil, +{ + // the IBM docs note + // + // > vec_roundp provides the same functionality as vec_ceil, except that vec_roundz would not trigger the IEEE-inexact exception. + // + // but in practice `vec_ceil` also does not trigger that exception, so both are equivalent + a.vec_ceil() +} + +/// Returns a vector containing the truncated values of the corresponding elements of the given vector. +/// Each element of the result contains the value of the corresponding element of a, truncated to an integral value. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_roundz(a: T) -> T +where + T: sealed::VectorTrunc, +{ + // the IBM docs note + // + // > vec_roundz provides the same functionality as vec_trunc, except that vec_roundz would not trigger the IEEE-inexact exception. + // + // but in practice `vec_trunc` also does not trigger that exception, so both are equivalent + a.vec_trunc() +} + +/// Returns a vector by using the current rounding mode to round every floating-point element in the given vector to integer. +#[inline] +#[target_feature(enable = "vector")] +#[unstable(feature = "stdarch_s390x", issue = "135681")] +pub unsafe fn vec_rint(a: T) -> T +where + T: sealed::VectorRint, +{ + a.vec_rint() +} + #[cfg(test)] mod tests { use super::*; @@ -852,6 +1044,33 @@ mod tests { use crate::core_arch::simd::*; use stdarch_test::simd_test; + macro_rules! test_vec_1 { + { $name: ident, $fn:ident, f32x4, [$($a:expr),+], ~[$($d:expr),+] } => { + #[simd_test(enable = "vector")] + unsafe fn $name() { + let a: vector_float = transmute(f32x4::new($($a),+)); + + let d: vector_float = transmute(f32x4::new($($d),+)); + let r = transmute(vec_cmple(vec_abs(vec_sub($fn(a), d)), vec_splats(f32::EPSILON))); + let e = m32x4::new(true, true, true, true); + assert_eq!(e, r); + } + }; + { $name: ident, $fn:ident, $ty: ident, [$($a:expr),+], [$($d:expr),+] } => { + test_vec_1! { $name, $fn, $ty -> $ty, [$($a),+], [$($d),+] } + }; + { $name: ident, $fn:ident, $ty: ident -> $ty_out: ident, [$($a:expr),+], [$($d:expr),+] } => { + #[simd_test(enable = "vector")] + unsafe fn $name() { + let a: s_t_l!($ty) = transmute($ty::new($($a),+)); + + let d = $ty_out::new($($d),+); + let r : $ty_out = transmute($fn(a)); + assert_eq!(d, r); + } + } + } + macro_rules! test_vec_2 { { $name: ident, $fn:ident, $ty: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => { test_vec_2! { $name, $fn, $ty -> $ty, [$($a),+], [$($b),+], [$($d),+] } @@ -1059,4 +1278,88 @@ mod tests { [0b11001100, 0b11001100, 0b11001100, 0b11001100], [0b00110011, 0b11110011, 0b00001100, 0b00000000], [!0b11111111, !0b00111111, !0b11000000, !0b11001100] } + + test_vec_1! { test_vec_floor_f32, vec_floor, f32x4, + [1.1, 1.9, -0.5, -0.9], + [1.0, 1.0, -1.0, -1.0] + } + + test_vec_1! { test_vec_floor_f64_1, vec_floor, f64x2, + [1.1, 1.9], + [1.0, 1.0] + } + test_vec_1! { test_vec_floor_f64_2, vec_floor, f64x2, + [-0.5, -0.9], + [-1.0, -1.0] + } + + test_vec_1! { test_vec_ceil_f32, vec_ceil, f32x4, + [0.1, 0.5, 0.6, 0.9], + [1.0, 1.0, 1.0, 1.0] + } + test_vec_1! { test_vec_ceil_f64_1, vec_ceil, f64x2, + [0.1, 0.5], + [1.0, 1.0] + } + test_vec_1! { test_vec_ceil_f64_2, vec_ceil, f64x2, + [0.6, 0.9], + [1.0, 1.0] + } + + test_vec_1! { test_vec_round_f32, vec_round, f32x4, + [0.1, 0.5, 0.6, 0.9], + [0.0, 0.0, 1.0, 1.0] + } + + test_vec_1! { test_vec_round_f32_even_odd, vec_round, f32x4, + [0.5, 1.5, 2.5, 3.5], + [0.0, 2.0, 2.0, 4.0] + } + + test_vec_1! { test_vec_round_f64_1, vec_round, f64x2, + [0.1, 0.5], + [0.0, 0.0] + } + test_vec_1! { test_vec_round_f64_2, vec_round, f64x2, + [0.6, 0.9], + [1.0, 1.0] + } + + test_vec_1! { test_vec_roundc_f32, vec_roundc, f32x4, + [0.1, 0.5, 0.6, 0.9], + [0.0, 0.0, 1.0, 1.0] + } + + test_vec_1! { test_vec_roundc_f32_even_odd, vec_roundc, f32x4, + [0.5, 1.5, 2.5, 3.5], + [0.0, 2.0, 2.0, 4.0] + } + + test_vec_1! { test_vec_roundc_f64_1, vec_roundc, f64x2, + [0.1, 0.5], + [0.0, 0.0] + } + test_vec_1! { test_vec_roundc_f64_2, vec_roundc, f64x2, + [0.6, 0.9], + [1.0, 1.0] + } + + test_vec_1! { test_vec_rint_f32, vec_rint, f32x4, + [0.1, 0.5, 0.6, 0.9], + [0.0, 0.0, 1.0, 1.0] + } + + test_vec_1! { test_vec_rint_f32_even_odd, vec_rint, f32x4, + [0.5, 1.5, 2.5, 3.5], + [0.0, 2.0, 2.0, 4.0] + } + + test_vec_1! { test_vec_rint_f64_1, vec_rint, f64x2, + [0.1, 0.5], + [0.0, 0.0] + } + test_vec_1! { test_vec_rint_f64_2, vec_rint, f64x2, + [0.6, 0.9], + [1.0, 1.0] + } }