[vm/compiler] Perform word-sized copies in MemoryMove when possible.

When copying elements that are smaller than words on non-X86
architectures, only copy element by element until the remaining elements
can be copied in word-sized chunks, and then do so.

TEST=vm/cc/IRTest_Memory, co19{,_2}/LibTest/typed_data,
     lib{,_2}/typed_data, corelib{,_2}/list_test

Issue: #42072

Cq-Include-Trybots: luci.dart.try:vm-aot-linux-debug-simarm_x64-try,vm-aot-linux-debug-simriscv64-try,vm-linux-debug-simriscv64-try,vm-mac-debug-arm64-try,vm-aot-linux-release-simarm64-try,vm-aot-linux-release-simarm_x64-try,vm-aot-mac-release-arm64-try,vm-ffi-qemu-linux-release-riscv64-try,vm-ffi-qemu-linux-release-arm-try,vm-linux-release-simarm-try,vm-linux-release-simarm64-try,vm-mac-release-arm64-try,vm-aot-android-release-arm64c-try,vm-ffi-android-debug-arm64c-try
Change-Id: I61eab310b92a6bc5ebd88fa63d562103d887cb74
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/324280
Reviewed-by: Alexander Markov <[email protected]>
Commit-Queue: Tess Strickland <[email protected]>
Reviewed-by: Ryan Macnak <[email protected]>

Author: sstrickl

runtime/vm/compiler/assembler/assembler_arm.cc

@@ -2821,16 +2821,29 @@ Address Assembler::PrepareLargeLoadOffset(const Address& address,
   if (address.kind() != Address::Immediate) {
     return address;
   }
-  Register base = address.base();
   int32_t offset = address.offset();
   int32_t offset_mask = 0;
-  if (!Address::CanHoldLoadOffset(size, offset, &offset_mask)) {
-    ASSERT(base != IP);
-    AddImmediate(IP, base, offset & ~offset_mask, cond);
-    base = IP;
-    offset = offset & offset_mask;
+  if (Address::CanHoldLoadOffset(size, offset, &offset_mask)) {
+    return address;
   }
-  return Address(base, offset);
+  auto mode = address.mode();
+  // If the retrieved offset is negative, then the U bit was flipped during
+  // encoding, so re-flip it.
+  if (offset < 0) {
+    mode = static_cast<Address::Mode>(mode ^ U);
+  }
+  // If writing back post-indexing, we can't separate the instruction into
+  // two parts and the offset must fit.
+  ASSERT((mode | U) != Address::PostIndex);
+  // If we're writing back pre-indexing, we must add directly to the base,
+  // otherwise we use TMP.
+  Register base = address.base();
+  ASSERT(base != TMP || address.has_writeback());
+  Register temp = address.has_writeback() ? base : TMP;
+  AddImmediate(temp, base, offset & ~offset_mask, cond);
+  base = temp;
+  offset = offset & offset_mask;
+  return Address(base, offset, mode);
 }
 
 Address Assembler::PrepareLargeStoreOffset(const Address& address,
@@ -2840,16 +2853,29 @@ Address Assembler::PrepareLargeStoreOffset(const Address& address,
   if (address.kind() != Address::Immediate) {
     return address;
   }
-  Register base = address.base();
   int32_t offset = address.offset();
   int32_t offset_mask = 0;
-  if (!Address::CanHoldStoreOffset(size, offset, &offset_mask)) {
-    ASSERT(base != IP);
-    AddImmediate(IP, base, offset & ~offset_mask, cond);
-    base = IP;
-    offset = offset & offset_mask;
+  if (Address::CanHoldStoreOffset(size, offset, &offset_mask)) {
+    return address;
   }
-  return Address(base, offset);
+  auto mode = address.mode();
+  // If the retrieved offset is negative, then the U bit was flipped during
+  // encoding, so re-flip it.
+  if (offset < 0) {
+    mode = static_cast<Address::Mode>(mode ^ U);
+  }
+  // If writing back post-indexing, we can't separate the instruction into
+  // two parts and the offset must fit.
+  ASSERT((mode | U) != Address::PostIndex);
+  // If we're writing back pre-indexing, we must add directly to the base,
+  // otherwise we use TMP.
+  Register base = address.base();
+  ASSERT(base != TMP || address.has_writeback());
+  Register temp = address.has_writeback() ? base : TMP;
+  AddImmediate(temp, base, offset & ~offset_mask, cond);
+  base = temp;
+  offset = offset & offset_mask;
+  return Address(base, offset, mode);
 }
 
 void Assembler::LoadFromOffset(Register reg,

runtime/vm/compiler/assembler/assembler_arm.h

@@ -257,6 +257,9 @@ class Address : public ValueObject {
     kind_ = Immediate;
     base_ = rn;
     offset_ = offset;
+    // If the offset can't be encoded in fewer bits, then it'll conflict with
+    // the encoding of the mode and we won't be able to retrieve it later.
+    ASSERT(Utils::MagnitudeIsUint(kOpcodeShift, offset));
     if (offset < 0) {
       encoding_ = (am ^ (1 << kUShift)) | -offset;  // Flip U to adjust sign.
     } else {
@@ -310,7 +313,7 @@ class Address : public ValueObject {
                : kNoRegister;
   }
 
-  Mode mode() const { return static_cast<Mode>(encoding() & kModeMask); }
+  Mode mode() const { return static_cast<Mode>(encoding_ & kModeMask); }
 
   bool has_writeback() const {
     return (mode() == PreIndex) || (mode() == PostIndex) ||

runtime/vm/compiler/backend/il.cc

@@ -6775,6 +6775,7 @@ void MemoryCopyInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
     // the loop case.
     __ BranchIf(UNSIGNED_LESS_EQUAL, &copy_forwards,
                 compiler::Assembler::kNearJump);
+    __ Comment("Copying backwards");
     if (constant_length) {
       EmitUnrolledCopy(compiler, dest_reg, src_reg, num_elements,
                        /*reversed=*/true);
@@ -6783,6 +6784,7 @@ void MemoryCopyInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
                    &copy_forwards);
     }
     __ Jump(&done, compiler::Assembler::kNearJump);
+    __ Comment("Copying forwards");
   }
   __ Bind(&copy_forwards);
   if (constant_length) {

runtime/vm/compiler/backend/il_arm.cc

@@ -243,33 +243,88 @@ void MemoryCopyInstr::PrepareLengthRegForLoop(FlowGraphCompiler* compiler,
   __ BranchIfZero(length_reg, done);
 }
 
+static compiler::OperandSize OperandSizeFor(intptr_t bytes) {
+  ASSERT(Utils::IsPowerOfTwo(bytes));
+  switch (bytes) {
+    case 1:
+      return compiler::kUnsignedByte;
+    case 2:
+      return compiler::kUnsignedTwoBytes;
+    case 4:
+      return compiler::kUnsignedFourBytes;
+    case 8:
+      return compiler::kEightBytes;
+    default:
+      UNREACHABLE();
+      return compiler::kEightBytes;
+  }
+}
+
+static void CopyUpToWordMultiple(FlowGraphCompiler* compiler,
+                                 Register dest_reg,
+                                 Register src_reg,
+                                 Register length_reg,
+                                 intptr_t element_size,
+                                 bool unboxed_inputs,
+                                 bool reversed,
+                                 compiler::Label* done) {
+  ASSERT(Utils::IsPowerOfTwo(element_size));
+  if (element_size >= compiler::target::kWordSize) return;
+
+  const intptr_t base_shift = (unboxed_inputs ? 0 : kSmiTagShift) -
+                              Utils::ShiftForPowerOfTwo(element_size);
+  auto const mode =
+      reversed ? compiler::Address::NegPreIndex : compiler::Address::PostIndex;
+  intptr_t tested_bits = 0;
+
+  __ Comment("Copying until region is a multiple of word size");
+
+  for (intptr_t bit = compiler::target::kWordSizeLog2 - 1; bit >= 0; bit--) {
+    const intptr_t bytes = 1 << bit;
+    if (element_size > bytes) continue;
+    auto const sz = OperandSizeFor(bytes);
+    const intptr_t tested_bit = bit + base_shift;
+    tested_bits |= (1 << tested_bit);
+    __ tst(length_reg, compiler::Operand(1 << tested_bit));
+    __ LoadFromOffset(TMP, compiler::Address(src_reg, bytes, mode), sz,
+                      NOT_ZERO);
+    __ StoreToOffset(TMP, compiler::Address(dest_reg, bytes, mode), sz,
+                     NOT_ZERO);
+  }
+
+  __ bics(length_reg, length_reg, compiler::Operand(tested_bits));
+  __ b(done, ZERO);
+}
+
 void MemoryCopyInstr::EmitLoopCopy(FlowGraphCompiler* compiler,
                                    Register dest_reg,
                                    Register src_reg,
                                    Register length_reg,
                                    compiler::Label* done,
                                    compiler::Label* copy_forwards) {
-  const intptr_t loop_subtract = unboxed_inputs() ? 1 : Smi::RawValue(1);
-  auto load_mode = compiler::Address::PostIndex;
-  auto load_multiple_mode = BlockAddressMode::IA_W;
-  if (copy_forwards != nullptr) {
-    // When reversed, start the src and dest registers with the end addresses
-    // and apply the negated offset prior to indexing.
-    load_mode = compiler::Address::NegPreIndex;
-    load_multiple_mode = BlockAddressMode::DB_W;
+  const bool reversed = copy_forwards != nullptr;
+  if (reversed) {
     // Verify that the overlap actually exists by checking to see if
     // dest_start < src_end.
     const intptr_t shift = Utils::ShiftForPowerOfTwo(element_size_) -
                            (unboxed_inputs() ? 0 : kSmiTagShift);
     if (shift < 0) {
-      __ add(TMP, src_reg, compiler::Operand(length_reg, ASR, -shift));
+      __ add(src_reg, src_reg, compiler::Operand(length_reg, ASR, -shift));
     } else {
-      __ add(TMP, src_reg, compiler::Operand(length_reg, LSL, shift));
+      __ add(src_reg, src_reg, compiler::Operand(length_reg, LSL, shift));
     }
-    __ CompareRegisters(dest_reg, TMP);
-    __ BranchIf(UNSIGNED_GREATER_EQUAL, copy_forwards);
-    // There is overlap, so mov TMP to src_reg and adjust dest_reg now.
-    __ MoveRegister(src_reg, TMP);
+    __ CompareRegisters(dest_reg, src_reg);
+    // If dest_reg >= src_reg, then set src_reg back to the start of the source
+    // region before branching to the forwards-copying loop.
+    if (shift < 0) {
+      __ sub(src_reg, src_reg, compiler::Operand(length_reg, ASR, -shift),
+             UNSIGNED_GREATER_EQUAL);
+    } else {
+      __ sub(src_reg, src_reg, compiler::Operand(length_reg, LSL, shift),
+             UNSIGNED_GREATER_EQUAL);
+    }
+    __ b(copy_forwards, UNSIGNED_GREATER_EQUAL);
+    // There is overlap, so adjust dest_reg now.
     if (shift < 0) {
       __ add(dest_reg, dest_reg, compiler::Operand(length_reg, ASR, -shift));
     } else {
@@ -279,29 +334,34 @@ void MemoryCopyInstr::EmitLoopCopy(FlowGraphCompiler* compiler,
   // We can use TMP for all instructions below because element_size_ is
   // guaranteed to fit in the offset portion of the instruction in the
   // non-LDM/STM cases.
-  compiler::Address src_address =
-      compiler::Address(src_reg, element_size_, load_mode);
-  compiler::Address dest_address =
-      compiler::Address(dest_reg, element_size_, load_mode);
+  CopyUpToWordMultiple(compiler, dest_reg, src_reg, length_reg, element_size_,
+                       unboxed_inputs_, reversed, done);
+  // When reversed, the src and dest registers have been adjusted to start at
+  // the end addresses, so apply the negated offset prior to indexing.
+  const auto load_mode =
+      reversed ? compiler::Address::NegPreIndex : compiler::Address::PostIndex;
+  const auto load_multiple_mode =
+      reversed ? BlockAddressMode::DB_W : BlockAddressMode::IA_W;
+  // The size of the uncopied region is a multiple of the word size, so now we
+  // copy the rest by word (unless the element size is larger).
+  const intptr_t loop_subtract =
+      Utils::Maximum<intptr_t>(1, compiler::target::kWordSize / element_size_)
+      << (unboxed_inputs_ ? 0 : kSmiTagShift);
   // Used only for LDM/STM below.
   RegList temp_regs = (1 << TMP);
   for (intptr_t i = 0; i < locs()->temp_count(); i++) {
     temp_regs |= 1 << locs()->temp(i).reg();
   }
+  __ Comment("Copying by multiples of word size");
   compiler::Label loop;
   __ Bind(&loop);
   switch (element_size_) {
+    // Fall through for the sizes smaller than compiler::target::kWordSize.
     case 1:
-      __ ldrb(TMP, src_address);
-      __ strb(TMP, dest_address);
-      break;
     case 2:
-      __ ldrh(TMP, src_address);
-      __ strh(TMP, dest_address);
-      break;
     case 4:
-      __ ldr(TMP, src_address);
-      __ str(TMP, dest_address);
+      __ ldr(TMP, compiler::Address(src_reg, 4, load_mode));
+      __ str(TMP, compiler::Address(dest_reg, 4, load_mode));
       break;
     case 8:
       COMPILE_ASSERT(8 == kMaxMemoryCopyElementSize);

runtime/vm/compiler/backend/il_arm64.cc

@@ -175,22 +175,71 @@ void MemoryCopyInstr::PrepareLengthRegForLoop(FlowGraphCompiler* compiler,
   __ BranchIfZero(length_reg, done);
 }
 
+static compiler::OperandSize OperandSizeFor(intptr_t bytes) {
+  ASSERT(Utils::IsPowerOfTwo(bytes));
+  switch (bytes) {
+    case 1:
+      return compiler::kUnsignedByte;
+    case 2:
+      return compiler::kUnsignedTwoBytes;
+    case 4:
+      return compiler::kUnsignedFourBytes;
+    case 8:
+      return compiler::kEightBytes;
+    default:
+      UNREACHABLE();
+      return compiler::kEightBytes;
+  }
+}
+
+static void CopyUpToWordMultiple(FlowGraphCompiler* compiler,
+                                 Register dest_reg,
+                                 Register src_reg,
+                                 Register length_reg,
+                                 intptr_t element_size,
+                                 bool unboxed_inputs,
+                                 bool reversed,
+                                 compiler::Label* done) {
+  ASSERT(Utils::IsPowerOfTwo(element_size));
+  if (element_size >= compiler::target::kWordSize) return;
+
+  const intptr_t base_shift = (unboxed_inputs ? 0 : kSmiTagShift) -
+                              Utils::ShiftForPowerOfTwo(element_size);
+  const intptr_t offset_sign = reversed ? -1 : 1;
+  auto const mode =
+      reversed ? compiler::Address::PreIndex : compiler::Address::PostIndex;
+  intptr_t tested_bits = 0;
+
+  __ Comment("Copying until region is a multiple of word size");
+
+  for (intptr_t bit = compiler::target::kWordSizeLog2 - 1; bit >= 0; bit--) {
+    const intptr_t bytes = 1 << bit;
+    if (element_size > bytes) continue;
+    auto const sz = OperandSizeFor(bytes);
+    const intptr_t tested_bit = bit + base_shift;
+    tested_bits |= (1 << tested_bit);
+    const intptr_t offset = offset_sign * bytes;
+    compiler::Label skip_copy;
+    __ tbz(&skip_copy, length_reg, tested_bit);
+    __ ldr(TMP, compiler::Address(src_reg, offset, mode), sz);
+    __ str(TMP, compiler::Address(dest_reg, offset, mode), sz);
+    __ Bind(&skip_copy);
+  }
+
+  ASSERT(tested_bits != 0);
+  __ andis(length_reg, length_reg, compiler::Immediate(~tested_bits),
+           compiler::kObjectBytes);
+  __ b(done, ZERO);
+}
+
 void MemoryCopyInstr::EmitLoopCopy(FlowGraphCompiler* compiler,
                                    Register dest_reg,
                                    Register src_reg,
                                    Register length_reg,
                                    compiler::Label* done,
                                    compiler::Label* copy_forwards) {
-  const intptr_t loop_subtract = unboxed_inputs() ? 1 : Smi::RawValue(1);
-  intptr_t offset = element_size_;
-  auto mode = element_size_ == 16 ? compiler::Address::PairPostIndex
-                                  : compiler::Address::PostIndex;
-  if (copy_forwards != nullptr) {
-    // When reversed, start the src and dest registers with the end addresses
-    // and apply the negated offset prior to indexing.
-    offset = -element_size_;
-    mode = element_size_ == 16 ? compiler::Address::PairPreIndex
-                               : compiler::Address::PreIndex;
+  const bool reversed = copy_forwards != nullptr;
+  if (reversed) {
     // Verify that the overlap actually exists by checking to see if
     // dest_start < src_end.
     if (!unboxed_inputs()) {
@@ -213,24 +262,33 @@ void MemoryCopyInstr::EmitLoopCopy(FlowGraphCompiler* compiler,
       __ add(dest_reg, dest_reg, compiler::Operand(length_reg, LSL, shift));
     }
   }
-
-  compiler::Address src_address = compiler::Address(src_reg, offset, mode);
-  compiler::Address dest_address = compiler::Address(dest_reg, offset, mode);
+  CopyUpToWordMultiple(compiler, dest_reg, src_reg, length_reg, element_size_,
+                       unboxed_inputs_, reversed, done);
+  // When reversed, the src and dest registers are adjusted to start with the
+  // end addresses, so apply the negated offset prior to indexing.
+  const intptr_t offset =
+      (reversed ? -1 : 1) *
+      Utils::Maximum<intptr_t>(compiler::target::kWordSize, element_size_);
+  const auto mode = element_size_ == 16
+                        ? (reversed ? compiler::Address::PairPreIndex
+                                    : compiler::Address::PairPostIndex)
+                        : (reversed ? compiler::Address::PreIndex
+                                    : compiler::Address::PostIndex);
+  // The size of the uncopied region is a multiple of the word size, so now we
+  // copy the rest by word (unless the element size is larger).
+  const intptr_t loop_subtract =
+      Utils::Maximum<intptr_t>(1, compiler::target::kWordSize / element_size_)
+      << (unboxed_inputs_ ? 0 : kSmiTagShift);
+  const auto src_address = compiler::Address(src_reg, offset, mode);
+  const auto dest_address = compiler::Address(dest_reg, offset, mode);
+  __ Comment("Copying by multiples of word size");
   compiler::Label loop;
   __ Bind(&loop);
   switch (element_size_) {
+    // Fall through for the sizes smaller than compiler::target::kWordSize.
     case 1:
-      __ ldr(TMP, src_address, compiler::kUnsignedByte);
-      __ str(TMP, dest_address, compiler::kUnsignedByte);
-      break;
     case 2:
-      __ ldr(TMP, src_address, compiler::kUnsignedTwoBytes);
-      __ str(TMP, dest_address, compiler::kUnsignedTwoBytes);
-      break;
     case 4:
-      __ ldr(TMP, src_address, compiler::kUnsignedFourBytes);
-      __ str(TMP, dest_address, compiler::kUnsignedFourBytes);
-      break;
     case 8:
       __ ldr(TMP, src_address, compiler::kEightBytes);
       __ str(TMP, dest_address, compiler::kEightBytes);
@@ -243,7 +301,6 @@ void MemoryCopyInstr::EmitLoopCopy(FlowGraphCompiler* compiler,
       UNREACHABLE();
       break;
   }
-
   __ subs(length_reg, length_reg, compiler::Operand(loop_subtract),
           compiler::kObjectBytes);
   __ b(&loop, NOT_ZERO);