[vm] Intrinsify Int32x4 operator >>>

Intrinsified on x64, arm64 and arm. ia32 and RISC-V keep the Dart
fallback.

Issue: #64170
Issue: #64345
Issue: #64511
TEST=lib/typed_data/int32x4_arithmetic_test

Change-Id: I1a1637242d4145a4a7c53c56924e625559808a96
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/560562
Commit-Queue: Slava Egorov <vegorov@google.com>
Reviewed-by: Slava Egorov <vegorov@google.com>
Reviewed-by: Martin Kustermann <kustermann@google.com>
Auto-Submit: Modestas Valauskas <valauskasmodestas@gmail.com>

Author: modulovalue

runtime/vm/compiler/backend/il.h

@@ -11312,6 +11312,7 @@ class LoadThreadInstr : public TemplateDefinition<0, NoThrow, PureAndMovable> {
   M(2, _, Int32x4NotEqual, (Int32x4, Int32x4), Int32x4)                        \
   M(2, _, Int32x4Shl, (Int32x4, Int32), Int32x4)                               \
   M(2, _, Int32x4ShrS, (Int32x4, Int32), Int32x4)                              \
+  M(2, _, Int32x4ShrU, (Int32x4, Int32), Int32x4)                              \
   M(2, _, Int32x4AndNot, (Int32x4, Int32x4), Int32x4)                          \
   M(2, _, Float32x4GreaterThan, (Float32x4, Float32x4), Int32x4)               \
   M(2, _, Float32x4GreaterThanOrEqual, (Float32x4, Float32x4), Int32x4)        \

runtime/vm/compiler/backend/il_arm.cc

@@ -5277,13 +5277,17 @@ DEFINE_EMIT(Int32x4Shift,
              Register shift,
              Temp<QRegister> shift_vector)) {
   __ and_(TMP, shift, compiler::Operand(31));
-  if (instr->kind() == SimdOpInstr::kInt32x4ShrS) {
-    // vshlqi shifts right when the per-lane count is negative.
+  if (instr->kind() != SimdOpInstr::kInt32x4Shl) {
+    // vshlqi and vshlqu shift right when the per-lane count is negative.
     __ rsb(TMP, TMP, compiler::Operand(0));
   }
   __ vmovdr(DTMP, 0, TMP);
   __ vdup(compiler::kFourBytes, shift_vector, DTMP, 0);
-  __ vshlqi(compiler::kFourBytes, result, value, shift_vector);
+  if (instr->kind() == SimdOpInstr::kInt32x4ShrU) {
+    __ vshlqu(compiler::kFourBytes, result, value, shift_vector);
+  } else {
+    __ vshlqi(compiler::kFourBytes, result, value, shift_vector);
+  }
 }
 
 // Map SimdOpInstr::Kind-s to corresponding emit functions. Uses the following
@@ -5295,6 +5299,7 @@ DEFINE_EMIT(Int32x4Shift,
 #define SIMD_OP_VARIANTS(CASE, ____, SIMPLE)                                   \
   CASE(Int32x4Shl)                                                             \
   CASE(Int32x4ShrS)                                                            \
+  CASE(Int32x4ShrU)                                                            \
   ____(Int32x4Shift)                                                           \
   CASE(Float32x4Add)                                                           \
   CASE(Float32x4Sub)                                                           \

runtime/vm/compiler/backend/il_arm64.cc

@@ -4380,12 +4380,16 @@ DEFINE_EMIT(Int32x4Shift,
              Temp<Register> count,
              Temp<VRegister> shift_vector)) {
   __ AndImmediate(count, shift, 31);
-  if (instr->kind() == SimdOpInstr::kInt32x4ShrS) {
-    // SSHL shifts right when the per-lane count is negative.
+  if (instr->kind() != SimdOpInstr::kInt32x4Shl) {
+    // SSHL and USHL shift right when the per-lane count is negative.
     __ neg(count, count);
   }
   __ vdupw(shift_vector, count);
-  __ vsshlw(result, value, shift_vector);
+  if (instr->kind() == SimdOpInstr::kInt32x4ShrU) {
+    __ vushlw(result, value, shift_vector);
+  } else {
+    __ vsshlw(result, value, shift_vector);
+  }
 }
 
 // Map SimdOpInstr::Kind-s to corresponding emit functions. Uses the following
@@ -4397,6 +4401,7 @@ DEFINE_EMIT(Int32x4Shift,
 #define SIMD_OP_VARIANTS(CASE, ____, SIMPLE)                                   \
   CASE(Int32x4Shl)                                                             \
   CASE(Int32x4ShrS)                                                            \
+  CASE(Int32x4ShrU)                                                            \
   ____(Int32x4Shift)                                                           \
   SIMD_OP_SIMPLE_BINARY(CASE)                                                  \
   CASE(Float32x4ShuffleMix)                                                    \

runtime/vm/compiler/backend/il_ia32.cc

@@ -4511,6 +4511,7 @@ LocationSummary* SimdOpInstr::MakeLocationSummary(Zone* zone, bool opt) const {
     case SimdOpInstr::kInt32x4NotEqual:
     case SimdOpInstr::kInt32x4Shl:
     case SimdOpInstr::kInt32x4ShrS:
+    case SimdOpInstr::kInt32x4ShrU:
     case SimdOpInstr::kInt32x4AndNot:
     case SimdOpInstr::kFloat32x4GreaterThan:
     case SimdOpInstr::kFloat32x4GreaterThanOrEqual:
@@ -4539,6 +4540,7 @@ void SimdOpInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
     case SimdOpInstr::kInt32x4NotEqual:
     case SimdOpInstr::kInt32x4Shl:
     case SimdOpInstr::kInt32x4ShrS:
+    case SimdOpInstr::kInt32x4ShrU:
     case SimdOpInstr::kInt32x4AndNot:
     case SimdOpInstr::kFloat32x4GreaterThan:
     case SimdOpInstr::kFloat32x4GreaterThanOrEqual:

runtime/vm/compiler/backend/il_x64.cc

@@ -4677,6 +4677,9 @@ DEFINE_EMIT(Int32x4Shift,
     case SimdOpInstr::kInt32x4ShrS:
       __ psrad(value, count_xmm);
       break;
+    case SimdOpInstr::kInt32x4ShrU:
+      __ psrld(value, count_xmm);
+      break;
     default:
       UNREACHABLE();
   }
@@ -4691,6 +4694,7 @@ DEFINE_EMIT(Int32x4Shift,
 #define SIMD_OP_VARIANTS(CASE, ____, SIMPLE)                                   \
   CASE(Int32x4Shl)                                                             \
   CASE(Int32x4ShrS)                                                            \
+  CASE(Int32x4ShrU)                                                            \
   ____(Int32x4Shift)                                                           \
   SIMD_OP_SIMPLE_BINARY(CASE)                                                  \
   CASE(Int32x4AndNot)                                                          \

runtime/vm/compiler/call_specializer.cc

@@ -2773,6 +2773,7 @@ class SimdLowering : public ValueObject {
       case MethodRecognizer::kInt32x4AllTrue:
       case MethodRecognizer::kInt32x4Shl:
       case MethodRecognizer::kInt32x4ShrS:
+      case MethodRecognizer::kInt32x4ShrU:
       case MethodRecognizer::kInt32x4AndNot:
         // TODO(riscv)
         return false;
@@ -3557,6 +3558,7 @@ bool CallSpecializer::TryInlineRecognizedMethod(
     case MethodRecognizer::kInt32x4NotEqual:
     case MethodRecognizer::kInt32x4Shl:
     case MethodRecognizer::kInt32x4ShrS:
+    case MethodRecognizer::kInt32x4ShrU:
     case MethodRecognizer::kInt32x4AndNot:
 #endif
       return InlineSimdOp(flow_graph, is_dynamic_call, call, receiver, kind,

runtime/vm/compiler/recognized_methods_list.h

@@ -529,6 +529,7 @@ namespace dart {
   V(TypedDataLibrary, _Int32x4, notEqual, Int32x4NotEqual, 0xafde9ab3)         \
   V(TypedDataLibrary, _Int32x4, <<, Int32x4Shl, 0xfc92031f)                    \
   V(TypedDataLibrary, _Int32x4, >>, Int32x4ShrS, 0xf0fc5fc4)                   \
+  V(TypedDataLibrary, _Int32x4, >>>, Int32x4ShrU, 0xedd3f6c6)                  \
   V(TypedDataLibrary, _Int32x4, get:x, Int32x4GetX, 0x55b951d5)                \
   V(TypedDataLibrary, _Int32x4, get:y, Int32x4GetY, 0x434aacf8)                \
   V(TypedDataLibrary, _Int32x4, get:z, Int32x4GetZ, 0x7916188e)                \

sdk/lib/_internal/vm/lib/typed_data_patch.dart

@@ -2271,6 +2271,7 @@ final class _Int32x4 implements Int32x4 {
     return Int32x4(x >> n, y >> n, z >> n, w >> n);
   }
 
+  @pragma("vm:recognized", "other")
   Int32x4 operator >>>(int shiftAmount) {
     final int n = shiftAmount & 31;
     return Int32x4(