diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index a4cc960bd29921..ca38c26ab7c845 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -2685,6 +2685,15 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) break; } + case NI_Sve2_AddCarryWideningLower: + case NI_Sve2_AddCarryWideningUpper: + if (targetReg != op3Reg) + { + GetEmitter()->emitIns_Mov(INS_mov, emitTypeSize(node), targetReg, op3Reg, /* canSkip */ true); + } + GetEmitter()->emitInsSve_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, opt); + break; + case NI_Sve2_BitwiseClearXor: case NI_Sve2_Xor: if (targetReg != op1Reg) diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index e70db726ce152f..5a943763b993ec 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -318,6 +318,8 @@ HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceAddWideningLower, HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceAddWideningUpper, -1, 3, {INS_invalid, INS_invalid, INS_sve_sabalt, INS_sve_uabalt, INS_sve_sabalt, INS_sve_uabalt, INS_sve_sabalt, INS_sve_uabalt, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceWideningLower, -1, 2, {INS_invalid, INS_invalid, INS_sve_sabdlb, INS_sve_uabdlb, INS_sve_sabdlb, INS_sve_uabdlb, INS_sve_sabdlb, INS_sve_uabdlb, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable) HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceWideningUpper, -1, 2, {INS_invalid, INS_invalid, INS_sve_sabdlt, INS_sve_uabdlt, INS_sve_sabdlt, INS_sve_uabdlt, INS_sve_sabdlt, INS_sve_uabdlt, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable) +HARDWARE_INTRINSIC(Sve2, AddCarryWideningLower, -1, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adclb, INS_invalid, INS_sve_adclb, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen) +HARDWARE_INTRINSIC(Sve2, AddCarryWideningUpper, -1, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_adclt, INS_invalid, INS_sve_adclt, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Sve2, BitwiseClearXor, -1, 3, {INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_sve_bcax, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve2, BitwiseSelect, -1, 3, {INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_sve_bsl, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) HARDWARE_INTRINSIC(Sve2, BitwiseSelectLeftInverted, -1, 3, {INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_sve_bsl1n, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_HasRMWSemantics) diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index a1e81422b27082..b47fe8f962780a 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1935,6 +1935,7 @@ class LinearScan : public LinearScanInterface // 'tgtPrefUse' to that RefPosition. RefPosition* tgtPrefUse = nullptr; RefPosition* tgtPrefUse2 = nullptr; + RefPosition* tgtPrefUse3 = nullptr; public: // The following keep track of information about internal (temporary register) intervals @@ -1957,6 +1958,7 @@ class LinearScan : public LinearScanInterface { tgtPrefUse = nullptr; tgtPrefUse2 = nullptr; + tgtPrefUse3 = nullptr; internalCount = 0; setInternalRegsDelayFree = false; pendingDelayFree = false; diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index cb19df849ada8f..06d832d10307ba 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1464,15 +1464,25 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou { assert(tgtPrefUse == nullptr); assert(tgtPrefUse2 == nullptr); + assert(tgtPrefUse3 == nullptr); tgtPrefUse = delayUse; } - else + else if (opNum == 2) { assert(opNum == 2); assert(tgtPrefUse == nullptr); assert(tgtPrefUse2 == nullptr); + assert(tgtPrefUse3 == nullptr); tgtPrefUse2 = delayUse; } + else + { + assert(opNum == 3); + assert(tgtPrefUse == nullptr); + assert(tgtPrefUse2 == nullptr); + assert(tgtPrefUse3 == nullptr); + tgtPrefUse3 = delayUse; + } } } else if (containedCselOp == operand) @@ -2292,6 +2302,14 @@ GenTree* LinearScan::getDelayFreeOperand(GenTreeHWIntrinsic* intrinsicTree, bool assert(delayFreeOp != nullptr); break; + case NI_Sve2_AddCarryWideningLower: + case NI_Sve2_AddCarryWideningUpper: + // RMW operates on the third op. + assert(isRMW); + delayFreeOp = intrinsicTree->Op(3); + assert(delayFreeOp != nullptr); + break; + default: if (isRMW) { diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 0d297d92a929e5..a57a3bebaebdca 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -3062,6 +3062,7 @@ RefPosition* LinearScan::BuildDef(GenTree* tree, SingleTypeRegSet dstCandidates, #ifndef TARGET_ARM setTgtPref(interval, tgtPrefUse); setTgtPref(interval, tgtPrefUse2); + setTgtPref(interval, tgtPrefUse3); #endif // !TARGET_ARM #if FEATURE_PARTIAL_SIMD_CALLEE_SAVE diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve2.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve2.PlatformNotSupported.cs index ff9b4b05b22bb9..1750d1e17a5d16 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve2.PlatformNotSupported.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve2.PlatformNotSupported.cs @@ -230,6 +230,34 @@ internal Arm64() { } /// public static Vector AbsoluteDifferenceWideningUpper(Vector left, Vector right) { throw new PlatformNotSupportedException(); } + // Add with carry long (bottom) + + /// + /// svuint32_t svadclb[_u32](svuint32_t op1, svuint32_t op2, svuint32_t op3) + /// ADCLB Ztied1.S, Zop2.S, Zop3.S + /// + public static unsafe Vector AddCarryWideningLower(Vector op1, Vector op2, Vector op3) { throw new PlatformNotSupportedException(); } + + /// + /// svuint64_t svadclb[_u64](svuint64_t op1, svuint64_t op2, svuint64_t op3) + /// ADCLB Ztied1.D, Zop2.D, Zop3.D + /// + public static unsafe Vector AddCarryWideningLower(Vector op1, Vector op2, Vector op3) { throw new PlatformNotSupportedException(); } + + // Add with carry long (top) + + /// + /// svuint32_t svadclt[_u32](svuint32_t op1, svuint32_t op2, svuint32_t op3) + /// ADCLT Ztied1.S, Zop2.S, Zop3.S + /// + public static unsafe Vector AddCarryWideningUpper(Vector op1, Vector op2, Vector op3) { throw new PlatformNotSupportedException(); } + + /// + /// svuint64_t svadclt[_u64](svuint64_t op1, svuint64_t op2, svuint64_t op3) + /// ADCLT Ztied1.D, Zop2.D, Zop3.D + /// + public static unsafe Vector AddCarryWideningUpper(Vector op1, Vector op2, Vector op3) { throw new PlatformNotSupportedException(); } + // Bitwise clear and exclusive OR /// diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve2.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve2.cs index 77bd276cb5bad0..77da21a5005738 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve2.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve2.cs @@ -230,6 +230,34 @@ internal Arm64() { } /// public static Vector AbsoluteDifferenceWideningUpper(Vector left, Vector right) => AbsoluteDifferenceWideningUpper(left, right); + // Add with carry long (bottom) + + /// + /// svuint32_t svadclb[_u32](svuint32_t op1, svuint32_t op2, svuint32_t op3) + /// ADCLB Ztied1.S, Zop2.S, Zop3.S + /// + public static unsafe Vector AddCarryWideningLower(Vector op1, Vector op2, Vector op3) => AddCarryWideningLower(op1, op2, op3); + + /// + /// svuint64_t svadclb[_u64](svuint64_t op1, svuint64_t op2, svuint64_t op3) + /// ADCLB Ztied1.D, Zop2.D, Zop3.D + /// + public static unsafe Vector AddCarryWideningLower(Vector op1, Vector op2, Vector op3) => AddCarryWideningLower(op1, op2, op3); + + // Add with carry long (top) + + /// + /// svuint32_t svadclt[_u32](svuint32_t op1, svuint32_t op2, svuint32_t op3) + /// ADCLT Ztied1.S, Zop2.S, Zop3.S + /// + public static unsafe Vector AddCarryWideningUpper(Vector op1, Vector op2, Vector op3) => AddCarryWideningUpper(op1, op2, op3); + + /// + /// svuint64_t svadclt[_u64](svuint64_t op1, svuint64_t op2, svuint64_t op3) + /// ADCLT Ztied1.D, Zop2.D, Zop3.D + /// + public static unsafe Vector AddCarryWideningUpper(Vector op1, Vector op2, Vector op3) => AddCarryWideningUpper(op1, op2, op3); + // Bitwise clear and exclusive OR /// diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index 76b5f394db6336..70fbfc6277e9e3 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -6116,6 +6116,10 @@ internal Arm64() { } public static System.Numerics.Vector AbsoluteDifferenceWideningUpper(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } public static System.Numerics.Vector AbsoluteDifferenceWideningUpper(System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector AddCarryWideningLower(System.Numerics.Vector op1, System.Numerics.Vector op2, System.Numerics.Vector op3) { throw null; } + public static System.Numerics.Vector AddCarryWideningLower(System.Numerics.Vector op1, System.Numerics.Vector op2, System.Numerics.Vector op3) { throw null; } + public static System.Numerics.Vector AddCarryWideningUpper(System.Numerics.Vector op1, System.Numerics.Vector op2, System.Numerics.Vector op3) { throw null; } + public static System.Numerics.Vector AddCarryWideningUpper(System.Numerics.Vector op1, System.Numerics.Vector op2, System.Numerics.Vector op3) { throw null; } public static System.Numerics.Vector BitwiseClearXor(System.Numerics.Vector xor, System.Numerics.Vector value, System.Numerics.Vector mask) { throw null; } public static System.Numerics.Vector BitwiseClearXor(System.Numerics.Vector xor, System.Numerics.Vector value, System.Numerics.Vector mask) { throw null; } public static System.Numerics.Vector BitwiseClearXor(System.Numerics.Vector xor, System.Numerics.Vector value, System.Numerics.Vector mask) { throw null; } diff --git a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs index f7812710903cb5..ac8c9a9771aa16 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs @@ -4817,6 +4817,11 @@ ("SveVecBinOpDifferentRetType.template", new Dictionary {["TestName"] = "Sve2_AbsoluteDifferenceWideningUpper_uint_ushort", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "AbsoluteDifferenceWideningUpper", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ConvertFunc"] = "", ["ValidateIterResult"] = "Helpers.AbsoluteDifferenceWideningOdd(left, right, i) != result[i]", ["GetIterResult"] = "(UInt32) Helpers.AbsoluteDifferenceWideningOdd(left, right, i)"}), ("SveVecBinOpDifferentRetType.template", new Dictionary {["TestName"] = "Sve2_AbsoluteDifferenceWideningUpper_ulong_uint", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "AbsoluteDifferenceWideningUpper", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ConvertFunc"] = "", ["ValidateIterResult"] = "Helpers.AbsoluteDifferenceWideningOdd(left, right, i) != result[i]", ["GetIterResult"] = "(UInt64) Helpers.AbsoluteDifferenceWideningOdd(left, right, i)"}), + ("SveVecTernOpTest.template", new Dictionary { ["TestName"] = "Sve2_AddCarryWideningLower_uint", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "AddCarryWideningLower", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()", ["ConvertFunc"] = "", ["ValidateIterResult"] = "Helpers.AddCarryWideningEven(firstOp, secondOp, thirdOp, i) != result[i]", ["GetIterResult"] = "(UInt32) Helpers.AddCarryWideningEven(first, second, third, i)"}), + ("SveVecTernOpTest.template", new Dictionary { ["TestName"] = "Sve2_AddCarryWideningLower_ulong", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "AddCarryWideningLower", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()", ["ConvertFunc"] = "", ["ValidateIterResult"] = "Helpers.AddCarryWideningEven(firstOp, secondOp, thirdOp, i) != result[i]", ["GetIterResult"] = "(UInt64) Helpers.AddCarryWideningEven(first, second, third, i)"}), + ("SveVecTernOpTest.template", new Dictionary { ["TestName"] = "Sve2_AddCarryWideningUpper_uint", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "AddCarryWideningUpper", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt32()", ["ConvertFunc"] = "", ["ValidateIterResult"] = "Helpers.AddCarryWideningOdd(firstOp, secondOp, thirdOp, i) != result[i]", ["GetIterResult"] = "(UInt32) Helpers.AddCarryWideningOdd(first, second, third, i)"}), + ("SveVecTernOpTest.template", new Dictionary { ["TestName"] = "Sve2_AddCarryWideningUpper_ulong", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "AddCarryWideningUpper", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp3"] = "TestLibrary.Generator.GetUInt64()", ["ConvertFunc"] = "", ["ValidateIterResult"] = "Helpers.AddCarryWideningOdd(firstOp, secondOp, thirdOp, i) != result[i]", ["GetIterResult"] = "(UInt64) Helpers.AddCarryWideningOdd(first, second, third, i)"}), + ("SveVecTernOpTest.template", new Dictionary { ["TestName"] = "Sve2_BitwiseClearXor_sbyte", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "BitwiseClearXor", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["NextValueOp3"] = "TestLibrary.Generator.GetSByte()", ["ConvertFunc"] = "", ["ValidateIterResult"] = "result[i] != (firstOp[i] ^ (secondOp[i] &~ thirdOp[i]))", ["GetIterResult"] = "(SByte) (firstOp[i] ^ (secondOp[i] &~ thirdOp[i]))"}), ("SveVecTernOpTest.template", new Dictionary { ["TestName"] = "Sve2_BitwiseClearXor_short", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "BitwiseClearXor", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int16", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt16()", ["ConvertFunc"] = "", ["ValidateIterResult"] = "result[i] != (firstOp[i] ^ (secondOp[i] &~ thirdOp[i]))", ["GetIterResult"] = "(Int16) (firstOp[i] ^ (secondOp[i] &~ thirdOp[i]))"}), ("SveVecTernOpTest.template", new Dictionary { ["TestName"] = "Sve2_BitwiseClearXor_int", ["Isa"] = "Sve2", ["LoadIsa"] = "Sve2", ["Method"] = "BitwiseClearXor", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int32", ["Op3VectorType"] = "Vector", ["Op3BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp3"] = "TestLibrary.Generator.GetInt32()", ["ConvertFunc"] = "", ["ValidateIterResult"] = "result[i] != (firstOp[i] ^ (secondOp[i] &~ thirdOp[i]))", ["GetIterResult"] = "(Int32) (firstOp[i] ^ (secondOp[i] &~ thirdOp[i]))"}), diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs index 355730e6ddab56..1ad711276c3941 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs @@ -2293,6 +2293,79 @@ private static long Reduce(Func reduceOp, sbyte[] op1) public static int AddPairwiseWideningAndAdd(int[] op1, short[] op2, int i) => (int)(op1[i] + AddWidening(op2[2 * i], op2[2 * i + 1])); + public static uint AddCarryWideningEven(uint[] op1, uint[] op2, uint[] op3, int i) + { + uint lsb; + ulong res; + + if ((i < 0) || (i >= op1.Length) || (i >= op2.Length) || (i >= op3.Length)) + { + throw new ArgumentOutOfRangeException(nameof(i), "Index i is out of range"); + } + + if (i % 2 == 0) + { + if (i + 1 >= op2.Length) + { + throw new ArgumentOutOfRangeException(nameof(i), "Index i + 1 is out of range."); + } + + lsb = op2[i + 1] & 1u; + res = (ulong)op1[i] + op3[i] + lsb; + return (uint)res; + } + else + { + if (((i - 1) < 0) || ((i - 1) >= op1.Length) || ((i - 1) >= op3.Length)) + { + throw new ArgumentOutOfRangeException(nameof(i), "Index i - 1 is out of range."); + } + + lsb = op2[i] & 1u; + res = (ulong)op1[i - 1] + op3[i - 1] + lsb; + + // Shift result to get the carry bit + return (uint)(res >> 32); + } + } + + public static uint AddCarryWideningOdd(uint[] op1, uint[] op2, uint[] op3, int i) + { + uint lsb; + ulong res; + + if ((i < 0) || (i >= op1.Length) || (i >= op2.Length) || (i >= op3.Length)) + { + throw new ArgumentOutOfRangeException(nameof(i), "Index i is out of range"); + } + + if (i % 2 == 0) + { + if (((i + 1) >= op1.Length) || ((i + 1) >= op2.Length)) + { + throw new ArgumentOutOfRangeException(nameof(i), "Index i + 1 is out of range."); + } + + lsb = op2[i + 1] & 1u; + res = (ulong)op1[i + 1] + op3[i] + lsb; + return (uint)res; + } + else + { + if (((i - 1) < 0) || ((i - 1) >= op3.Length)) + { + throw new ArgumentOutOfRangeException(nameof(i), "Index i - 1 is out of range."); + } + + lsb = op2[i] & 1u; + res = (ulong)op1[i] + op3[i - 1] + lsb; + + // Shift result to get the carry bit + return (uint)(res >> 32); + } + } + + private static short HighNarrowing(int op1, bool round) { uint roundConst = 0; @@ -2415,6 +2488,88 @@ private static long Reduce(Func reduceOp, short[] op1) public static long AddPairwiseWideningAndAdd(long[] op1, int[] op2, int i) => (long)(op1[i] + AddWidening(op2[2 * i], op2[2 * i + 1])); + public static ulong AddCarryWideningEven(ulong[] op1, ulong[] op2, ulong[] op3, int i) + { + ulong lsb; + ulong res; + + if ((i < 0) || (i >= op1.Length) || (i >= op2.Length) || (i >= op3.Length)) + { + throw new ArgumentOutOfRangeException(nameof(i), "Index i is out of range"); + } + + if (i % 2 == 0) + { + if ((i + 1) >= op2.Length) + { + throw new ArgumentOutOfRangeException(nameof(i), "Index i + 1 is out of range for op3."); + } + + lsb = op2[i + 1] & 1UL; + res = op1[i] + op3[i] + lsb; + return res; + } + else + { + if (((i - 1) < 0) || ((i - 1) >= op1.Length) || ((i - 1) >= op3.Length)) + { + throw new ArgumentOutOfRangeException(nameof(i), "Index i - 1 is out of range."); + } + + lsb = op2[i] & 1UL; + + // Look for an overflow in the addition to get the carry bit + ulong sum1 = op1[i - 1] + op3[i - 1]; + bool overflow1 = sum1 < op1[i - 1]; + + ulong sum2 = sum1 + lsb; + bool overflow2 = sum2 < sum1; + + return (overflow1 || overflow2) ? 1UL : 0UL; + } + } + + public static ulong AddCarryWideningOdd(ulong[] op1, ulong[] op2, ulong[] op3, int i) + { + ulong lsb; + ulong res; + + if ((i < 0) || (i >= op1.Length) || (i >= op2.Length) || (i >= op3.Length)) + { + throw new ArgumentOutOfRangeException(nameof(i), "Index i is out of range"); + } + + if (i % 2 == 0) + { + if (((i + 1) >= op1.Length) || ((i + 1) >= op2.Length)) + { + throw new ArgumentOutOfRangeException(nameof(i), "Index i + 1 is out of range."); + } + + lsb = op2[i + 1] & 1UL; + res = op1[i + 1] + op3[i] + lsb; + return res; + } + else + { + if (((i - 1) < 0) || ((i - 1) >= op3.Length)) + { + throw new ArgumentOutOfRangeException(nameof(i), "Index i - 1 is out of range."); + } + + lsb = op2[i] & 1UL; + + // Look for an overflow in the addition to get the carry bit + ulong sum1 = op1[i] + op3[i - 1]; + bool overflow1 = sum1 < op1[i]; + + ulong sum2 = sum1 + lsb; + bool overflow2 = sum2 < sum1; + + return (overflow1 || overflow2) ? 1UL : 0UL; + } + } + private static int HighNarrowing(long op1, bool round) { ulong roundConst = 0;