Skip to content

Commit 4c6a2ab

Browse files
sjindel-googlecommit-bot@chromium.org
sjindel-google
authored and
[email protected]
committed
Re-land "[vm] Aggressive write-barrier elimination."
Three bugs were fixed: 1. BitVector::Equals was not fully fixed by the original CL. 2. We need to add old objects to the deferred marking queue during RememberLiveTemporaries(). 3. The thread being scanned in RestoreWriteBarrierInvariant may not be scheduled, so we cannot use its store buffer block. In addition, this changed uncovered another bug fixed in: https://dart-review.googlesource.com/c/sdk/+/138960. Original CL is in patchset 3. This reverts commit 30a12a3. Change-Id: I36169b09563998ed5b3c3eac70ee0ebe78853e62 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/138920 Reviewed-by: Ryan Macnak <[email protected]> Commit-Queue: Samir Jindel <[email protected]>
1 parent ba68d04 commit 4c6a2ab

28 files changed

+924
-91
lines changed

‎runtime/vm/bit_vector.cc

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
// BSD-style license that can be found in the LICENSE file.
44

55
#include "vm/bit_vector.h"
6-
6+
#include "vm/log.h"
77
#include "vm/os.h"
88

99
namespace dart {
@@ -39,8 +39,10 @@ bool BitVector::Equals(const BitVector& other) const {
3939
if (data_[i] != other.data_[i]) return false;
4040
}
4141
if (i < data_length_) {
42+
if (length_ % kBitsPerWord == 0) return data_[i] == other.data_[i];
43+
4244
// Don't compare bits beyond length_.
43-
const intptr_t shift_size = (kBitsPerWord - length_) & (kBitsPerWord - 1);
45+
const intptr_t shift_size = kBitsPerWord - (length_ % kBitsPerWord);
4446
const uword mask = static_cast<uword>(-1) >> shift_size;
4547
if ((data_[i] & mask) != (other.data_[i] & mask)) return false;
4648
}
@@ -105,11 +107,11 @@ bool BitVector::IsEmpty() const {
105107
}
106108

107109
void BitVector::Print() const {
108-
OS::PrintErr("[");
110+
THR_Print("[");
109111
for (intptr_t i = 0; i < length_; i++) {
110-
OS::PrintErr(Contains(i) ? "1" : "0");
112+
THR_Print(Contains(i) ? "1" : "0");
111113
}
112-
OS::PrintErr("]");
114+
THR_Print("]");
113115
}
114116

115117
} // namespace dart

‎runtime/vm/bit_vector.h

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,8 @@ class BitVector : public ZoneAllocated {
7070
data_[i / kBitsPerWord] &= ~(static_cast<uword>(1) << (i % kBitsPerWord));
7171
}
7272

73+
void Set(intptr_t i, bool value) { value ? Add(i) : Remove(i); }
74+
7375
bool Equals(const BitVector& other) const;
7476

7577
// Add all elements that are in the bitvector from.
@@ -92,6 +94,14 @@ class BitVector : public ZoneAllocated {
9294
return (block & (static_cast<uword>(1) << (i % kBitsPerWord))) != 0;
9395
}
9496

97+
bool SubsetOf(const BitVector& other) {
98+
ASSERT(length_ == other.length_);
99+
for (intptr_t i = 0; i < data_length_; ++i) {
100+
if ((data_[i] & other.data_[i]) != data_[i]) return false;
101+
}
102+
return true;
103+
}
104+
95105
void Clear() {
96106
for (intptr_t i = 0; i < data_length_; i++) {
97107
data_[i] = 0;

‎runtime/vm/compiler/backend/flow_graph_compiler.cc

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -615,7 +615,9 @@ void FlowGraphCompiler::VisitBlocks() {
615615
EmitInstructionPrologue(instr);
616616
ASSERT(pending_deoptimization_env_ == NULL);
617617
pending_deoptimization_env_ = instr->env();
618+
DEBUG_ONLY(current_instruction_ = instr);
618619
instr->EmitNativeCode(this);
620+
DEBUG_ONLY(current_instruction_ = nullptr);
619621
pending_deoptimization_env_ = NULL;
620622
if (IsPeephole(instr)) {
621623
ASSERT(top_of_stack_ == nullptr);
@@ -708,7 +710,9 @@ void FlowGraphCompiler::GenerateDeferredCode() {
708710
set_current_instruction(slow_path->instruction());
709711
SpecialStatsBegin(stats_tag);
710712
BeginCodeSourceRange();
713+
DEBUG_ONLY(current_instruction_ = slow_path->instruction());
711714
slow_path->GenerateCode(this);
715+
DEBUG_ONLY(current_instruction_ = nullptr);
712716
EndCodeSourceRange(slow_path->instruction()->token_pos());
713717
SpecialStatsEnd(stats_tag);
714718
set_current_instruction(nullptr);

‎runtime/vm/compiler/backend/flow_graph_compiler.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1121,6 +1121,15 @@ class FlowGraphCompiler : public ValueObject {
11211121
// is amenable to a peephole optimization.
11221122
bool IsPeephole(Instruction* instr) const;
11231123

1124+
#if defined(DEBUG)
1125+
bool CanCallDart() const {
1126+
return current_instruction_ == nullptr ||
1127+
current_instruction_->CanCallDart();
1128+
}
1129+
#else
1130+
bool CanCallDart() const { return true; }
1131+
#endif
1132+
11241133
// This struct contains either function or code, the other one being NULL.
11251134
class StaticCallsStruct : public ZoneAllocated {
11261135
public:

‎runtime/vm/compiler/backend/flow_graph_compiler_arm.cc

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -995,6 +995,7 @@ void FlowGraphCompiler::GenerateDartCall(intptr_t deopt_id,
995995
RawPcDescriptors::Kind kind,
996996
LocationSummary* locs,
997997
Code::EntryKind entry_kind) {
998+
ASSERT(CanCallDart());
998999
__ BranchLinkPatchable(stub, entry_kind);
9991000
EmitCallsiteMetadata(token_pos, deopt_id, kind, locs);
10001001
}
@@ -1005,6 +1006,7 @@ void FlowGraphCompiler::GenerateStaticDartCall(intptr_t deopt_id,
10051006
LocationSummary* locs,
10061007
const Function& target,
10071008
Code::EntryKind entry_kind) {
1009+
ASSERT(CanCallDart());
10081010
if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
10091011
__ GenerateUnRelocatedPcRelativeCall();
10101012
AddPcRelativeCallTarget(target, entry_kind);
@@ -1061,6 +1063,7 @@ void FlowGraphCompiler::EmitOptimizedInstanceCall(const Code& stub,
10611063
TokenPosition token_pos,
10621064
LocationSummary* locs,
10631065
Code::EntryKind entry_kind) {
1066+
ASSERT(CanCallDart());
10641067
ASSERT(Array::Handle(zone(), ic_data.arguments_descriptor()).Length() > 0);
10651068
// Each ICData propagated from unoptimized to optimized code contains the
10661069
// function that corresponds to the Dart function of that IC call. Due
@@ -1084,6 +1087,7 @@ void FlowGraphCompiler::EmitInstanceCallJIT(const Code& stub,
10841087
TokenPosition token_pos,
10851088
LocationSummary* locs,
10861089
Code::EntryKind entry_kind) {
1090+
ASSERT(CanCallDart());
10871091
ASSERT(entry_kind == Code::EntryKind::kNormal ||
10881092
entry_kind == Code::EntryKind::kUnchecked);
10891093
ASSERT(Array::Handle(zone(), ic_data.arguments_descriptor()).Length() > 0);
@@ -1109,6 +1113,7 @@ void FlowGraphCompiler::EmitMegamorphicInstanceCall(
11091113
LocationSummary* locs,
11101114
intptr_t try_index,
11111115
intptr_t slow_path_argument_count) {
1116+
ASSERT(CanCallDart());
11121117
ASSERT(!arguments_descriptor.IsNull() && (arguments_descriptor.Length() > 0));
11131118
const ArgumentsDescriptor args_desc(arguments_descriptor);
11141119
const MegamorphicCache& cache = MegamorphicCache::ZoneHandle(
@@ -1156,6 +1161,7 @@ void FlowGraphCompiler::EmitInstanceCallAOT(const ICData& ic_data,
11561161
LocationSummary* locs,
11571162
Code::EntryKind entry_kind,
11581163
bool receiver_can_be_smi) {
1164+
ASSERT(CanCallDart());
11591165
ASSERT(entry_kind == Code::EntryKind::kNormal ||
11601166
entry_kind == Code::EntryKind::kUnchecked);
11611167
ASSERT(ic_data.NumArgsTested() == 1);
@@ -1200,6 +1206,7 @@ void FlowGraphCompiler::EmitUnoptimizedStaticCall(intptr_t size_with_type_args,
12001206
LocationSummary* locs,
12011207
const ICData& ic_data,
12021208
Code::EntryKind entry_kind) {
1209+
ASSERT(CanCallDart());
12031210
const Code& stub =
12041211
StubCode::UnoptimizedStaticCallEntry(ic_data.NumArgsTested());
12051212
__ LoadObject(R9, ic_data);
@@ -1216,6 +1223,7 @@ void FlowGraphCompiler::EmitOptimizedStaticCall(
12161223
TokenPosition token_pos,
12171224
LocationSummary* locs,
12181225
Code::EntryKind entry_kind) {
1226+
ASSERT(CanCallDart());
12191227
ASSERT(!function.IsClosureFunction());
12201228
if (function.HasOptionalParameters() || function.IsGeneric()) {
12211229
__ LoadObject(R4, arguments_descriptor);
@@ -1235,6 +1243,7 @@ void FlowGraphCompiler::EmitDispatchTableCall(
12351243
Register cid_reg,
12361244
int32_t selector_offset,
12371245
const Array& arguments_descriptor) {
1246+
ASSERT(CanCallDart());
12381247
ASSERT(cid_reg != ARGS_DESC_REG);
12391248
if (!arguments_descriptor.IsNull()) {
12401249
__ LoadObject(ARGS_DESC_REG, arguments_descriptor);

‎runtime/vm/compiler/backend/flow_graph_compiler_arm64.cc

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -966,6 +966,7 @@ void FlowGraphCompiler::GenerateDartCall(intptr_t deopt_id,
966966
RawPcDescriptors::Kind kind,
967967
LocationSummary* locs,
968968
Code::EntryKind entry_kind) {
969+
ASSERT(CanCallDart());
969970
__ BranchLinkPatchable(stub, entry_kind);
970971
EmitCallsiteMetadata(token_pos, deopt_id, kind, locs);
971972
}
@@ -976,6 +977,7 @@ void FlowGraphCompiler::GenerateStaticDartCall(intptr_t deopt_id,
976977
LocationSummary* locs,
977978
const Function& target,
978979
Code::EntryKind entry_kind) {
980+
ASSERT(CanCallDart());
979981
if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
980982
__ GenerateUnRelocatedPcRelativeCall();
981983
AddPcRelativeCallTarget(target, entry_kind);
@@ -1023,6 +1025,7 @@ void FlowGraphCompiler::EmitOptimizedInstanceCall(const Code& stub,
10231025
TokenPosition token_pos,
10241026
LocationSummary* locs,
10251027
Code::EntryKind entry_kind) {
1028+
ASSERT(CanCallDart());
10261029
ASSERT(Array::Handle(zone(), ic_data.arguments_descriptor()).Length() > 0);
10271030
// Each ICData propagated from unoptimized to optimized code contains the
10281031
// function that corresponds to the Dart function of that IC call. Due
@@ -1045,6 +1048,7 @@ void FlowGraphCompiler::EmitInstanceCallJIT(const Code& stub,
10451048
TokenPosition token_pos,
10461049
LocationSummary* locs,
10471050
Code::EntryKind entry_kind) {
1051+
ASSERT(CanCallDart());
10481052
ASSERT(entry_kind == Code::EntryKind::kNormal ||
10491053
entry_kind == Code::EntryKind::kUnchecked);
10501054
ASSERT(Array::Handle(zone(), ic_data.arguments_descriptor()).Length() > 0);
@@ -1076,6 +1080,7 @@ void FlowGraphCompiler::EmitMegamorphicInstanceCall(
10761080
LocationSummary* locs,
10771081
intptr_t try_index,
10781082
intptr_t slow_path_argument_count) {
1083+
ASSERT(CanCallDart());
10791084
ASSERT(!arguments_descriptor.IsNull() && (arguments_descriptor.Length() > 0));
10801085
const ArgumentsDescriptor args_desc(arguments_descriptor);
10811086
const MegamorphicCache& cache = MegamorphicCache::ZoneHandle(
@@ -1120,6 +1125,7 @@ void FlowGraphCompiler::EmitInstanceCallAOT(const ICData& ic_data,
11201125
LocationSummary* locs,
11211126
Code::EntryKind entry_kind,
11221127
bool receiver_can_be_smi) {
1128+
ASSERT(CanCallDart());
11231129
ASSERT(ic_data.NumArgsTested() == 1);
11241130
const Code& initial_stub = StubCode::UnlinkedCall();
11251131
const char* switchable_call_mode = "smiable";
@@ -1170,6 +1176,7 @@ void FlowGraphCompiler::EmitUnoptimizedStaticCall(intptr_t size_with_type_args,
11701176
LocationSummary* locs,
11711177
const ICData& ic_data,
11721178
Code::EntryKind entry_kind) {
1179+
ASSERT(CanCallDart());
11731180
const Code& stub =
11741181
StubCode::UnoptimizedStaticCallEntry(ic_data.NumArgsTested());
11751182
__ LoadObject(R5, ic_data);
@@ -1186,6 +1193,7 @@ void FlowGraphCompiler::EmitOptimizedStaticCall(
11861193
TokenPosition token_pos,
11871194
LocationSummary* locs,
11881195
Code::EntryKind entry_kind) {
1196+
ASSERT(CanCallDart());
11891197
ASSERT(!function.IsClosureFunction());
11901198
if (function.HasOptionalParameters() || function.IsGeneric()) {
11911199
__ LoadObject(R4, arguments_descriptor);
@@ -1205,6 +1213,7 @@ void FlowGraphCompiler::EmitDispatchTableCall(
12051213
Register cid_reg,
12061214
int32_t selector_offset,
12071215
const Array& arguments_descriptor) {
1216+
ASSERT(CanCallDart());
12081217
ASSERT(cid_reg != ARGS_DESC_REG);
12091218
if (!arguments_descriptor.IsNull()) {
12101219
__ LoadObject(ARGS_DESC_REG, arguments_descriptor);

‎runtime/vm/compiler/backend/flow_graph_compiler_ia32.cc

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -861,6 +861,7 @@ void FlowGraphCompiler::GenerateDartCall(intptr_t deopt_id,
861861
RawPcDescriptors::Kind kind,
862862
LocationSummary* locs,
863863
Code::EntryKind entry_kind) {
864+
ASSERT(CanCallDart());
864865
__ Call(stub, /*moveable_target=*/false, entry_kind);
865866
EmitCallsiteMetadata(token_pos, deopt_id, kind, locs);
866867
}
@@ -871,6 +872,7 @@ void FlowGraphCompiler::GenerateStaticDartCall(intptr_t deopt_id,
871872
LocationSummary* locs,
872873
const Function& target,
873874
Code::EntryKind entry_kind) {
875+
ASSERT(CanCallDart());
874876
const auto& stub = StubCode::CallStaticFunction();
875877
__ Call(stub, /*movable_target=*/true, entry_kind);
876878
EmitCallsiteMetadata(token_pos, deopt_id, kind, locs);
@@ -892,6 +894,7 @@ void FlowGraphCompiler::EmitUnoptimizedStaticCall(intptr_t size_with_type_args,
892894
LocationSummary* locs,
893895
const ICData& ic_data,
894896
Code::EntryKind entry_kind) {
897+
ASSERT(CanCallDart());
895898
const Code& stub =
896899
StubCode::UnoptimizedStaticCallEntry(ic_data.NumArgsTested());
897900
__ LoadObject(ECX, ic_data);
@@ -919,6 +922,7 @@ void FlowGraphCompiler::EmitOptimizedInstanceCall(const Code& stub,
919922
TokenPosition token_pos,
920923
LocationSummary* locs,
921924
Code::EntryKind entry_kind) {
925+
ASSERT(CanCallDart());
922926
ASSERT(Array::Handle(ic_data.arguments_descriptor()).Length() > 0);
923927
// Each ICData propagated from unoptimized to optimized code contains the
924928
// function that corresponds to the Dart function of that IC call. Due
@@ -942,6 +946,7 @@ void FlowGraphCompiler::EmitInstanceCallJIT(const Code& stub,
942946
TokenPosition token_pos,
943947
LocationSummary* locs,
944948
Code::EntryKind entry_kind) {
949+
ASSERT(CanCallDart());
945950
ASSERT(entry_kind == Code::EntryKind::kNormal ||
946951
entry_kind == Code::EntryKind::kUnchecked);
947952
ASSERT(Array::Handle(ic_data.arguments_descriptor()).Length() > 0);
@@ -967,6 +972,7 @@ void FlowGraphCompiler::EmitMegamorphicInstanceCall(
967972
LocationSummary* locs,
968973
intptr_t try_index,
969974
intptr_t slow_path_argument_count) {
975+
ASSERT(CanCallDart());
970976
ASSERT(!arguments_descriptor.IsNull() && (arguments_descriptor.Length() > 0));
971977
const ArgumentsDescriptor args_desc(arguments_descriptor);
972978
const MegamorphicCache& cache = MegamorphicCache::ZoneHandle(
@@ -1014,6 +1020,7 @@ void FlowGraphCompiler::EmitOptimizedStaticCall(
10141020
TokenPosition token_pos,
10151021
LocationSummary* locs,
10161022
Code::EntryKind entry_kind) {
1023+
ASSERT(CanCallDart());
10171024
if (function.HasOptionalParameters() || function.IsGeneric()) {
10181025
__ LoadObject(EDX, arguments_descriptor);
10191026
} else {

‎runtime/vm/compiler/backend/flow_graph_compiler_x64.cc

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -977,6 +977,7 @@ void FlowGraphCompiler::GenerateDartCall(intptr_t deopt_id,
977977
RawPcDescriptors::Kind kind,
978978
LocationSummary* locs,
979979
Code::EntryKind entry_kind) {
980+
ASSERT(CanCallDart());
980981
__ CallPatchable(stub, entry_kind);
981982
EmitCallsiteMetadata(token_pos, deopt_id, kind, locs);
982983
}
@@ -987,6 +988,7 @@ void FlowGraphCompiler::GenerateStaticDartCall(intptr_t deopt_id,
987988
LocationSummary* locs,
988989
const Function& target,
989990
Code::EntryKind entry_kind) {
991+
ASSERT(CanCallDart());
990992
ASSERT(is_optimizing());
991993
if (FLAG_precompiled_mode && FLAG_use_bare_instructions) {
992994
__ GenerateUnRelocatedPcRelativeCall();
@@ -1019,6 +1021,7 @@ void FlowGraphCompiler::EmitUnoptimizedStaticCall(intptr_t size_with_type_args,
10191021
LocationSummary* locs,
10201022
const ICData& ic_data,
10211023
Code::EntryKind entry_kind) {
1024+
ASSERT(CanCallDart());
10221025
const Code& stub =
10231026
StubCode::UnoptimizedStaticCallEntry(ic_data.NumArgsTested());
10241027
__ LoadObject(RBX, ic_data);
@@ -1047,6 +1050,7 @@ void FlowGraphCompiler::EmitOptimizedInstanceCall(const Code& stub,
10471050
TokenPosition token_pos,
10481051
LocationSummary* locs,
10491052
Code::EntryKind entry_kind) {
1053+
ASSERT(CanCallDart());
10501054
ASSERT(Array::Handle(zone(), ic_data.arguments_descriptor()).Length() > 0);
10511055
// Each ICData propagated from unoptimized to optimized code contains the
10521056
// function that corresponds to the Dart function of that IC call. Due
@@ -1070,6 +1074,7 @@ void FlowGraphCompiler::EmitInstanceCallJIT(const Code& stub,
10701074
TokenPosition token_pos,
10711075
LocationSummary* locs,
10721076
Code::EntryKind entry_kind) {
1077+
ASSERT(CanCallDart());
10731078
ASSERT(entry_kind == Code::EntryKind::kNormal ||
10741079
entry_kind == Code::EntryKind::kUnchecked);
10751080
ASSERT(Array::Handle(zone(), ic_data.arguments_descriptor()).Length() > 0);
@@ -1095,6 +1100,7 @@ void FlowGraphCompiler::EmitMegamorphicInstanceCall(
10951100
LocationSummary* locs,
10961101
intptr_t try_index,
10971102
intptr_t slow_path_argument_count) {
1103+
ASSERT(CanCallDart());
10981104
ASSERT(!arguments_descriptor.IsNull() && (arguments_descriptor.Length() > 0));
10991105
const ArgumentsDescriptor args_desc(arguments_descriptor);
11001106
const MegamorphicCache& cache = MegamorphicCache::ZoneHandle(
@@ -1136,6 +1142,7 @@ void FlowGraphCompiler::EmitInstanceCallAOT(const ICData& ic_data,
11361142
LocationSummary* locs,
11371143
Code::EntryKind entry_kind,
11381144
bool receiver_can_be_smi) {
1145+
ASSERT(CanCallDart());
11391146
ASSERT(entry_kind == Code::EntryKind::kNormal ||
11401147
entry_kind == Code::EntryKind::kUnchecked);
11411148
ASSERT(ic_data.NumArgsTested() == 1);
@@ -1178,6 +1185,7 @@ void FlowGraphCompiler::EmitOptimizedStaticCall(
11781185
TokenPosition token_pos,
11791186
LocationSummary* locs,
11801187
Code::EntryKind entry_kind) {
1188+
ASSERT(CanCallDart());
11811189
ASSERT(!function.IsClosureFunction());
11821190
if (function.HasOptionalParameters() || function.IsGeneric()) {
11831191
__ LoadObject(R10, arguments_descriptor);
@@ -1197,6 +1205,7 @@ void FlowGraphCompiler::EmitDispatchTableCall(
11971205
Register cid_reg,
11981206
int32_t selector_offset,
11991207
const Array& arguments_descriptor) {
1208+
ASSERT(CanCallDart());
12001209
const Register table_reg = RAX;
12011210
ASSERT(cid_reg != table_reg);
12021211
ASSERT(cid_reg != ARGS_DESC_REG);

0 commit comments

Comments
 (0)