Skip to content

Commit 04941b5

Browse files
rmacnak-googlecommit-bot@chromium.org
rmacnak-google
authored and
[email protected]
committed
[vm, gc] For large arrays (arrays on a large heap page), use a card-based remembered set.
Bug: #30433 Change-Id: I7c710d9e36ab593295d00fc5e9e72a86a1b843af Reviewed-on: https://dart-review.googlesource.com/c/63688 Commit-Queue: Ryan Macnak <[email protected]> Reviewed-by: Vyacheslav Egorov <[email protected]>
1 parent 3687415 commit 04941b5

33 files changed

+802
-107
lines changed

‎runtime/vm/compiler/assembler/assembler_arm.cc

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1645,6 +1645,55 @@ void Assembler::StoreIntoObject(Register object,
16451645
Bind(&done);
16461646
}
16471647

1648+
void Assembler::StoreIntoArray(Register object,
1649+
Register slot,
1650+
Register value,
1651+
CanBeSmi can_be_smi,
1652+
bool lr_reserved) {
1653+
// x.slot = x. Barrier should have be removed at the IL level.
1654+
ASSERT(object != value);
1655+
ASSERT(object != LR);
1656+
ASSERT(value != LR);
1657+
ASSERT(slot != LR);
1658+
ASSERT(object != TMP);
1659+
ASSERT(value != TMP);
1660+
ASSERT(slot != TMP);
1661+
1662+
str(value, Address(slot, 0));
1663+
1664+
// In parallel, test whether
1665+
// - object is old and not remembered and value is new, or
1666+
// - object is old and value is old and not marked and concurrent marking is
1667+
// in progress
1668+
// If so, call the WriteBarrier stub, which will either add object to the
1669+
// store buffer (case 1) or add value to the marking stack (case 2).
1670+
// Compare RawObject::StorePointer.
1671+
Label done;
1672+
if (can_be_smi == kValueCanBeSmi) {
1673+
BranchIfSmi(value, &done);
1674+
}
1675+
if (!lr_reserved) Push(LR);
1676+
ldrb(TMP, FieldAddress(object, Object::tags_offset()));
1677+
ldrb(LR, FieldAddress(value, Object::tags_offset()));
1678+
and_(TMP, LR, Operand(TMP, LSR, RawObject::kBarrierOverlapShift));
1679+
ldr(LR, Address(THR, Thread::write_barrier_mask_offset()));
1680+
tst(TMP, Operand(LR));
1681+
1682+
if ((object != kWriteBarrierObjectReg) || (value != kWriteBarrierValueReg) ||
1683+
(slot != kWriteBarrierSlotReg)) {
1684+
// Spill and shuffle unimplemented. Currently StoreIntoArray is only used
1685+
// from StoreIndexInstr, which gets these exact registers from the register
1686+
// allocator.
1687+
UNIMPLEMENTED();
1688+
}
1689+
1690+
ldr(LR, Address(THR, Thread::array_write_barrier_entry_point_offset()), NE);
1691+
blx(LR, NE);
1692+
1693+
if (!lr_reserved) Pop(LR);
1694+
Bind(&done);
1695+
}
1696+
16481697
void Assembler::StoreIntoObjectOffset(Register object,
16491698
int32_t offset,
16501699
Register value,

‎runtime/vm/compiler/assembler/assembler_arm.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -783,6 +783,11 @@ class Assembler : public AssemblerBase {
783783
Register value, // Value we are storing.
784784
CanBeSmi can_value_be_smi = kValueCanBeSmi,
785785
bool lr_reserved = false);
786+
void StoreIntoArray(Register object,
787+
Register slot,
788+
Register value,
789+
CanBeSmi can_value_be_smi = kValueCanBeSmi,
790+
bool lr_reserved = false);
786791
void StoreIntoObjectOffset(Register object,
787792
int32_t offset,
788793
Register value,

‎runtime/vm/compiler/assembler/assembler_arm64.cc

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1028,6 +1028,53 @@ void Assembler::StoreIntoObject(Register object,
10281028
Bind(&done);
10291029
}
10301030

1031+
void Assembler::StoreIntoArray(Register object,
1032+
Register slot,
1033+
Register value,
1034+
CanBeSmi can_be_smi,
1035+
bool lr_reserved) {
1036+
ASSERT(object != TMP);
1037+
ASSERT(object != TMP2);
1038+
ASSERT(value != TMP);
1039+
ASSERT(value != TMP2);
1040+
ASSERT(slot != TMP);
1041+
ASSERT(slot != TMP2);
1042+
1043+
str(value, Address(slot, 0));
1044+
1045+
// In parallel, test whether
1046+
// - object is old and not remembered and value is new, or
1047+
// - object is old and value is old and not marked and concurrent marking is
1048+
// in progress
1049+
// If so, call the WriteBarrier stub, which will either add object to the
1050+
// store buffer (case 1) or add value to the marking stack (case 2).
1051+
// Compare RawObject::StorePointer.
1052+
Label done;
1053+
if (can_be_smi == kValueCanBeSmi) {
1054+
BranchIfSmi(value, &done);
1055+
}
1056+
ldr(TMP, FieldAddress(object, Object::tags_offset()), kUnsignedByte);
1057+
ldr(TMP2, FieldAddress(value, Object::tags_offset()), kUnsignedByte);
1058+
and_(TMP, TMP2, Operand(TMP, LSR, RawObject::kBarrierOverlapShift));
1059+
tst(TMP, Operand(BARRIER_MASK));
1060+
b(&done, ZERO);
1061+
if (!lr_reserved) Push(LR);
1062+
1063+
if ((object != kWriteBarrierObjectReg) || (value != kWriteBarrierValueReg) ||
1064+
(slot != kWriteBarrierSlotReg)) {
1065+
// Spill and shuffle unimplemented. Currently StoreIntoArray is only used
1066+
// from StoreIndexInstr, which gets these exact registers from the register
1067+
// allocator.
1068+
UNIMPLEMENTED();
1069+
}
1070+
1071+
ldr(LR, Address(THR, Thread::array_write_barrier_entry_point_offset()));
1072+
blr(LR);
1073+
1074+
if (!lr_reserved) Pop(LR);
1075+
Bind(&done);
1076+
}
1077+
10311078
void Assembler::StoreIntoObjectNoBarrier(Register object,
10321079
const Address& dest,
10331080
Register value) {

‎runtime/vm/compiler/assembler/assembler_arm64.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1445,6 +1445,12 @@ class Assembler : public AssemblerBase {
14451445
Register value,
14461446
CanBeSmi can_value_be_smi = kValueCanBeSmi,
14471447
bool lr_reserved = false);
1448+
void StoreIntoArray(Register object,
1449+
Register slot,
1450+
Register value,
1451+
CanBeSmi can_value_be_smi = kValueCanBeSmi,
1452+
bool lr_reserved = false);
1453+
14481454
void StoreIntoObjectOffset(Register object,
14491455
int32_t offset,
14501456
Register value,

‎runtime/vm/compiler/assembler/assembler_ia32.cc

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1909,6 +1909,48 @@ void Assembler::StoreIntoObjectNoBarrier(Register object,
19091909
// No store buffer update.
19101910
}
19111911

1912+
// Destroys the value register.
1913+
void Assembler::StoreIntoArray(Register object,
1914+
Register slot,
1915+
Register value,
1916+
CanBeSmi can_be_smi) {
1917+
ASSERT(object != value);
1918+
movl(Address(slot, 0), value);
1919+
1920+
Label done;
1921+
StoreIntoObjectFilter(object, value, &done, can_be_smi, kJumpToNoUpdate);
1922+
// A store buffer update is required.
1923+
if (value != kWriteBarrierObjectReg) {
1924+
pushl(kWriteBarrierObjectReg); // Preserve kWriteBarrierObjectReg.
1925+
}
1926+
if (value != kWriteBarrierSlotReg && slot != kWriteBarrierSlotReg) {
1927+
pushl(kWriteBarrierSlotReg); // Preserve kWriteBarrierSlotReg.
1928+
}
1929+
if (object != kWriteBarrierObjectReg && slot != kWriteBarrierSlotReg) {
1930+
if (slot == kWriteBarrierObjectReg && object == kWriteBarrierSlotReg) {
1931+
xchgl(slot, object);
1932+
} else if (slot == kWriteBarrierObjectReg) {
1933+
movl(kWriteBarrierSlotReg, slot);
1934+
movl(kWriteBarrierObjectReg, object);
1935+
} else {
1936+
movl(kWriteBarrierObjectReg, object);
1937+
movl(kWriteBarrierSlotReg, slot);
1938+
}
1939+
} else if (object != kWriteBarrierObjectReg) {
1940+
movl(kWriteBarrierObjectReg, object);
1941+
} else if (slot != kWriteBarrierSlotReg) {
1942+
movl(kWriteBarrierSlotReg, slot);
1943+
}
1944+
call(Address(THR, Thread::array_write_barrier_entry_point_offset()));
1945+
if (value != kWriteBarrierSlotReg && slot != kWriteBarrierSlotReg) {
1946+
popl(kWriteBarrierSlotReg); // Restore kWriteBarrierSlotReg.
1947+
}
1948+
if (value != kWriteBarrierObjectReg) {
1949+
popl(kWriteBarrierObjectReg); // Restore kWriteBarrierObjectReg.
1950+
}
1951+
Bind(&done);
1952+
}
1953+
19121954
void Assembler::UnverifiedStoreOldObject(const Address& dest,
19131955
const Object& value) {
19141956
ASSERT(!value.IsICData() || ICData::Cast(value).IsOriginal());

‎runtime/vm/compiler/assembler/assembler_ia32.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -610,6 +610,10 @@ class Assembler : public AssemblerBase {
610610
const Address& dest, // Where we are storing into.
611611
Register value, // Value we are storing.
612612
CanBeSmi can_value_be_smi = kValueCanBeSmi);
613+
void StoreIntoArray(Register object, // Object we are storing into.
614+
Register slot, // Where we are storing into.
615+
Register value, // Value we are storing.
616+
CanBeSmi can_value_be_smi = kValueCanBeSmi);
613617

614618
void StoreIntoObjectNoBarrier(Register object,
615619
const Address& dest,

‎runtime/vm/compiler/assembler/assembler_x64.cc

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1308,6 +1308,47 @@ void Assembler::StoreIntoObject(Register object,
13081308
Bind(&done);
13091309
}
13101310

1311+
void Assembler::StoreIntoArray(Register object,
1312+
Register slot,
1313+
Register value,
1314+
CanBeSmi can_be_smi) {
1315+
ASSERT(object != TMP);
1316+
ASSERT(value != TMP);
1317+
ASSERT(slot != TMP);
1318+
1319+
movq(Address(slot, 0), value);
1320+
1321+
// In parallel, test whether
1322+
// - object is old and not remembered and value is new, or
1323+
// - object is old and value is old and not marked and concurrent marking is
1324+
// in progress
1325+
// If so, call the WriteBarrier stub, which will either add object to the
1326+
// store buffer (case 1) or add value to the marking stack (case 2).
1327+
// Compare RawObject::StorePointer.
1328+
Label done;
1329+
if (can_be_smi == kValueCanBeSmi) {
1330+
testq(value, Immediate(kSmiTagMask));
1331+
j(ZERO, &done, kNearJump);
1332+
}
1333+
movb(TMP, FieldAddress(object, Object::tags_offset()));
1334+
shrl(TMP, Immediate(RawObject::kBarrierOverlapShift));
1335+
andl(TMP, Address(THR, Thread::write_barrier_mask_offset()));
1336+
testb(FieldAddress(value, Object::tags_offset()), TMP);
1337+
j(ZERO, &done, kNearJump);
1338+
1339+
if ((object != kWriteBarrierObjectReg) || (value != kWriteBarrierValueReg) ||
1340+
(slot != kWriteBarrierSlotReg)) {
1341+
// Spill and shuffle unimplemented. Currently StoreIntoArray is only used
1342+
// from StoreIndexInstr, which gets these exact registers from the register
1343+
// allocator.
1344+
UNIMPLEMENTED();
1345+
}
1346+
1347+
call(Address(THR, Thread::array_write_barrier_entry_point_offset()));
1348+
1349+
Bind(&done);
1350+
}
1351+
13111352
void Assembler::StoreIntoObjectNoBarrier(Register object,
13121353
const Address& dest,
13131354
Register value) {

‎runtime/vm/compiler/assembler/assembler_x64.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -726,6 +726,10 @@ class Assembler : public AssemblerBase {
726726
const Address& dest, // Where we are storing into.
727727
Register value, // Value we are storing.
728728
CanBeSmi can_be_smi = kValueCanBeSmi);
729+
void StoreIntoArray(Register object, // Object we are storing into.
730+
Register slot, // Where we are storing into.
731+
Register value, // Value we are storing.
732+
CanBeSmi can_be_smi = kValueCanBeSmi);
729733

730734
void StoreIntoObjectNoBarrier(Register object,
731735
const Address& dest,

‎runtime/vm/compiler/backend/il.h

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5124,7 +5124,15 @@ class CreateArrayInstr : public TemplateAllocation<2, Throws> {
51245124
virtual AliasIdentity Identity() const { return identity_; }
51255125
virtual void SetIdentity(AliasIdentity identity) { identity_ = identity; }
51265126

5127-
virtual bool WillAllocateNewOrRemembered() const { return true; }
5127+
virtual bool WillAllocateNewOrRemembered() const {
5128+
// Large arrays will use cards instead; cannot skip write barrier.
5129+
if (!num_elements()->BindsToConstant()) return false;
5130+
const Object& length = num_elements()->BoundConstant();
5131+
if (!length.IsSmi()) return false;
5132+
intptr_t raw_length = Smi::Cast(length).Value();
5133+
// Compare Array::New.
5134+
return (raw_length >= 0) && (raw_length < Array::kMaxNewSpaceElements);
5135+
}
51285136

51295137
private:
51305138
const TokenPosition token_pos_;

‎runtime/vm/compiler/backend/il_arm.cc

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1484,9 +1484,9 @@ Representation StoreIndexedInstr::RequiredInputRepresentation(
14841484

14851485
LocationSummary* StoreIndexedInstr::MakeLocationSummary(Zone* zone,
14861486
bool opt) const {
1487-
const bool directly_addressable = aligned() &&
1488-
class_id() != kTypedDataInt64ArrayCid &&
1489-
class_id() != kTypedDataUint64ArrayCid;
1487+
const bool directly_addressable =
1488+
aligned() && class_id() != kTypedDataInt64ArrayCid &&
1489+
class_id() != kTypedDataUint64ArrayCid && class_id() != kArrayCid;
14901490
const intptr_t kNumInputs = 3;
14911491
LocationSummary* locs;
14921492

@@ -1526,6 +1526,10 @@ LocationSummary* StoreIndexedInstr::MakeLocationSummary(Zone* zone,
15261526
locs->set_in(2, ShouldEmitStoreBarrier()
15271527
? Location::RegisterLocation(kWriteBarrierValueReg)
15281528
: Location::RegisterOrConstant(value()));
1529+
if (ShouldEmitStoreBarrier()) {
1530+
locs->set_in(0, Location::RegisterLocation(kWriteBarrierObjectReg));
1531+
locs->set_temp(0, Location::RegisterLocation(kWriteBarrierSlotReg));
1532+
}
15291533
break;
15301534
case kExternalTypedDataUint8ArrayCid:
15311535
case kExternalTypedDataUint8ClampedArrayCid:
@@ -1563,9 +1567,9 @@ LocationSummary* StoreIndexedInstr::MakeLocationSummary(Zone* zone,
15631567
}
15641568

15651569
void StoreIndexedInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
1566-
const bool directly_addressable = aligned() &&
1567-
class_id() != kTypedDataInt64ArrayCid &&
1568-
class_id() != kTypedDataUint64ArrayCid;
1570+
const bool directly_addressable =
1571+
aligned() && class_id() != kTypedDataInt64ArrayCid &&
1572+
class_id() != kTypedDataUint64ArrayCid && class_id() != kArrayCid;
15691573
// The array register points to the backing store for external arrays.
15701574
const Register array = locs()->in(0).reg();
15711575
const Location index = locs()->in(1);
@@ -1604,14 +1608,14 @@ void StoreIndexedInstr::EmitNativeCode(FlowGraphCompiler* compiler) {
16041608
case kArrayCid:
16051609
if (ShouldEmitStoreBarrier()) {
16061610
const Register value = locs()->in(2).reg();
1607-
__ StoreIntoObject(array, element_address, value, CanValueBeSmi(),
1608-
/*lr_reserved=*/!compiler->intrinsic_mode());
1611+
__ StoreIntoArray(array, temp, value, CanValueBeSmi(),
1612+
/*lr_reserved=*/!compiler->intrinsic_mode());
16091613
} else if (locs()->in(2).IsConstant()) {
16101614
const Object& constant = locs()->in(2).constant();
1611-
__ StoreIntoObjectNoBarrier(array, element_address, constant);
1615+
__ StoreIntoObjectNoBarrier(array, Address(temp), constant);
16121616
} else {
16131617
const Register value = locs()->in(2).reg();
1614-
__ StoreIntoObjectNoBarrier(array, element_address, value);
1618+
__ StoreIntoObjectNoBarrier(array, Address(temp), value);
16151619
}
16161620
break;
16171621
case kTypedDataInt8ArrayCid:

0 commit comments

Comments
 (0)