Skip to content

Commit 265a544

Browse files
fsc8000fsc8000
committed
Object pool with support for untagged entries.
This adds meta-information to object pool entries to allow storing untagged immediates or code addresses (ExternalLabel) directly. This eliminates the need to generate extra code to preserve the LSB when storing immediates as smis (x64, arm64). BUG= Review URL: https://codereview.chromium.org//1175523002.
1 parent 5cc8fca commit 265a544

37 files changed

+530
-259
lines changed

‎runtime/observatory/tests/service/graph_test.dart‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ var tests = [
7171
bVertex.shallowSize +
7272
rVertex.shallowSize));
7373

74-
const int fixedSizeListCid = 61;
74+
const int fixedSizeListCid = 62;
7575
List<ObjectVertex> lists = new List.from(graph.vertices.where(
7676
(ObjectVertex obj) => obj.vmCid == fixedSizeListCid));
7777
expect(lists.length >= 2, isTrue);

‎runtime/vm/assembler.cc‎

Lines changed: 60 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -242,61 +242,94 @@ const Code::Comments& Assembler::GetCodeComments() const {
242242
}
243243

244244

245-
intptr_t ObjectPool::AddObject(const Object& obj, Patchability patchable) {
245+
intptr_t ObjectPoolWrapper::AddObject(const Object& obj) {
246+
return AddObject(ObjectPool::Entry(&obj), kNotPatchable);
247+
}
248+
249+
250+
intptr_t ObjectPoolWrapper::AddImmediate(uword imm) {
251+
return AddObject(ObjectPool::Entry(imm, ObjectPool::kImmediate),
252+
kNotPatchable);
253+
}
254+
255+
intptr_t ObjectPoolWrapper::AddObject(ObjectPool::Entry entry,
256+
Patchability patchable) {
246257
// The object pool cannot be used in the vm isolate.
247258
ASSERT(Isolate::Current() != Dart::vm_isolate());
248-
if (object_pool_.IsNull()) {
249-
object_pool_ = GrowableObjectArray::New(Heap::kOld);
250-
}
251-
object_pool_.Add(obj, Heap::kOld);
252-
patchable_pool_entries_.Add(patchable);
259+
object_pool_.Add(entry);
253260
if (patchable == kNotPatchable) {
254261
// The object isn't patchable. Record the index for fast lookup.
255262
object_pool_index_table_.Insert(
256-
ObjIndexPair(&obj, object_pool_.Length() - 1));
263+
ObjIndexPair(entry, object_pool_.length() - 1));
257264
}
258-
return object_pool_.Length() - 1;
265+
return object_pool_.length() - 1;
259266
}
260267

261268

262-
intptr_t ObjectPool::AddExternalLabel(const ExternalLabel* label,
263-
Patchability patchable) {
269+
intptr_t ObjectPoolWrapper::AddExternalLabel(const ExternalLabel* label,
270+
Patchability patchable) {
264271
ASSERT(Isolate::Current() != Dart::vm_isolate());
265-
const uword address = label->address();
266-
ASSERT(Utils::IsAligned(address, 4));
267-
// The address is stored in the object array as a RawSmi.
268-
const Smi& smi = Smi::Handle(reinterpret_cast<RawSmi*>(address));
269-
return AddObject(smi, patchable);
272+
return AddObject(ObjectPool::Entry(label->address(),
273+
ObjectPool::kImmediate),
274+
patchable);
270275
}
271276

272277

273-
intptr_t ObjectPool::FindObject(const Object& obj, Patchability patchable) {
278+
intptr_t ObjectPoolWrapper::FindObject(ObjectPool::Entry entry,
279+
Patchability patchable) {
274280
// The object pool cannot be used in the vm isolate.
275281
ASSERT(Isolate::Current() != Dart::vm_isolate());
276282

277283
// If the object is not patchable, check if we've already got it in the
278284
// object pool.
279-
if (patchable == kNotPatchable && !object_pool_.IsNull()) {
280-
intptr_t idx = object_pool_index_table_.Lookup(&obj);
285+
if (patchable == kNotPatchable) {
286+
intptr_t idx = object_pool_index_table_.Lookup(entry);
281287
if (idx != ObjIndexPair::kNoIndex) {
282-
ASSERT(patchable_pool_entries_[idx] == kNotPatchable);
283288
return idx;
284289
}
285290
}
286291

287-
return AddObject(obj, patchable);
292+
return AddObject(entry, patchable);
288293
}
289294

290295

291-
intptr_t ObjectPool::FindExternalLabel(const ExternalLabel* label,
292-
Patchability patchable) {
296+
intptr_t ObjectPoolWrapper::FindObject(const Object& obj) {
297+
return FindObject(ObjectPool::Entry(&obj), kNotPatchable);
298+
}
299+
300+
301+
intptr_t ObjectPoolWrapper::FindImmediate(uword imm) {
302+
return FindObject(ObjectPool::Entry(imm, ObjectPool::kImmediate),
303+
kNotPatchable);
304+
}
305+
306+
307+
intptr_t ObjectPoolWrapper::FindExternalLabel(const ExternalLabel* label,
308+
Patchability patchable) {
293309
// The object pool cannot be used in the vm isolate.
294310
ASSERT(Isolate::Current() != Dart::vm_isolate());
295-
const uword address = label->address();
296-
ASSERT(Utils::IsAligned(address, 4));
297-
// The address is stored in the object array as a RawSmi.
298-
const Smi& smi = Smi::Handle(reinterpret_cast<RawSmi*>(address));
299-
return FindObject(smi, patchable);
311+
return FindObject(ObjectPool::Entry(label->address(),
312+
ObjectPool::kImmediate),
313+
patchable);
314+
}
315+
316+
317+
RawObjectPool* ObjectPoolWrapper::MakeObjectPool() {
318+
intptr_t len = object_pool_.length();
319+
if (len == 0) {
320+
return Object::empty_object_pool().raw();
321+
}
322+
const ObjectPool& result = ObjectPool::Handle(ObjectPool::New(len));
323+
for (intptr_t i = 0; i < len; ++i) {
324+
ObjectPool::EntryType info = object_pool_[i].type_;
325+
result.SetInfoAt(i, info);
326+
if (info == ObjectPool::kTaggedObject) {
327+
result.SetObjectAt(i, *object_pool_[i].obj_);
328+
} else {
329+
result.SetRawValueAt(i, object_pool_[i].raw_value_);
330+
}
331+
}
332+
return result.raw();
300333
}
301334

302335

‎runtime/vm/assembler.h‎

Lines changed: 44 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -216,45 +216,61 @@ class AssemblerBuffer : public ValueObject {
216216
class ObjIndexPair {
217217
public:
218218
// Typedefs needed for the DirectChainedHashMap template.
219-
typedef const Object* Key;
219+
typedef ObjectPool::Entry Key;
220220
typedef intptr_t Value;
221221
typedef ObjIndexPair Pair;
222222

223223
static const intptr_t kNoIndex = -1;
224224

225-
ObjIndexPair() : key_(NULL), value_(kNoIndex) { }
226-
227-
ObjIndexPair(Key key, Value value)
228-
: key_(key->IsNotTemporaryScopedHandle()
229-
? key : &Object::ZoneHandle(key->raw())),
230-
value_(value) { }
225+
ObjIndexPair() : key_(static_cast<uword>(NULL), ObjectPool::kTaggedObject),
226+
value_(kNoIndex) { }
227+
228+
ObjIndexPair(Key key, Value value) : value_(value) {
229+
key_.type_ = key.type_;
230+
if (key.type_ == ObjectPool::kTaggedObject) {
231+
if (key.obj_->IsNotTemporaryScopedHandle()) {
232+
key_.obj_ = key.obj_;
233+
} else {
234+
key_.obj_ = &Object::ZoneHandle(key.obj_->raw());
235+
}
236+
} else {
237+
key_.raw_value_ = key.raw_value_;
238+
}
239+
}
231240

232241
static Key KeyOf(Pair kv) { return kv.key_; }
233242

234243
static Value ValueOf(Pair kv) { return kv.value_; }
235244

236245
static intptr_t Hashcode(Key key) {
237-
if (key->IsSmi()) {
238-
return Smi::Cast(*key).Value();
246+
if (key.type_ != ObjectPool::kTaggedObject) {
247+
return key.raw_value_;
239248
}
240-
if (key->IsDouble()) {
249+
if (key.obj_->IsSmi()) {
250+
return Smi::Cast(*key.obj_).Value();
251+
}
252+
if (key.obj_->IsDouble()) {
241253
return static_cast<intptr_t>(
242254
bit_cast<int32_t, float>(
243-
static_cast<float>(Double::Cast(*key).value())));
255+
static_cast<float>(Double::Cast(*key.obj_).value())));
244256
}
245-
if (key->IsMint()) {
246-
return static_cast<intptr_t>(Mint::Cast(*key).value());
257+
if (key.obj_->IsMint()) {
258+
return static_cast<intptr_t>(Mint::Cast(*key.obj_).value());
247259
}
248-
if (key->IsString()) {
249-
return String::Cast(*key).Hash();
260+
if (key.obj_->IsString()) {
261+
return String::Cast(*key.obj_).Hash();
250262
}
251263
// TODO(fschneider): Add hash function for other classes commonly used as
252264
// compile-time constants.
253-
return key->GetClassId();
265+
return key.obj_->GetClassId();
254266
}
255267

256268
static inline bool IsKeyEqual(Pair kv, Key key) {
257-
return kv.key_->raw() == key->raw();
269+
if (kv.key_.type_ != key.type_) return false;
270+
if (kv.key_.type_ == ObjectPool::kTaggedObject) {
271+
return kv.key_.obj_->raw() == key.obj_->raw();
272+
}
273+
return kv.key_.raw_value_ == key.raw_value_;
258274
}
259275

260276
private:
@@ -269,25 +285,26 @@ enum Patchability {
269285
};
270286

271287

272-
class ObjectPool : public ValueObject {
288+
class ObjectPoolWrapper : public ValueObject {
273289
public:
274-
ObjectPool() : object_pool_(GrowableObjectArray::Handle()) { }
275-
276-
intptr_t AddObject(const Object& obj, Patchability patchable);
290+
intptr_t AddObject(const Object& obj);
291+
intptr_t AddImmediate(uword imm);
277292
intptr_t AddExternalLabel(const ExternalLabel* label,
278293
Patchability patchable);
279294

280-
intptr_t FindObject(const Object& obj, Patchability patchable);
295+
intptr_t FindObject(const Object& obj);
296+
intptr_t FindImmediate(uword imm);
281297
intptr_t FindExternalLabel(const ExternalLabel* label,
282298
Patchability patchable);
283-
const GrowableObjectArray& data() const { return object_pool_; }
299+
300+
RawObjectPool* MakeObjectPool();
284301

285302
private:
286-
// Objects and jump targets.
287-
GrowableObjectArray& object_pool_;
303+
intptr_t AddObject(ObjectPool::Entry entry, Patchability patchable);
304+
intptr_t FindObject(ObjectPool::Entry entry, Patchability patchable);
288305

289-
// Patchability of pool entries.
290-
GrowableArray<Patchability> patchable_pool_entries_;
306+
// Objects and jump targets.
307+
GrowableArray<ObjectPool::Entry> object_pool_;
291308

292309
// Hashmap for fast lookup in object pool.
293310
DirectChainedHashMap<ObjIndexPair> object_pool_index_table_;

‎runtime/vm/assembler_arm.cc‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1575,7 +1575,7 @@ void Assembler::LoadObject(Register rd, const Object& object, Condition cond) {
15751575
// Make sure that class CallPattern is able to decode this load from the
15761576
// object pool.
15771577
const int32_t offset =
1578-
Array::element_offset(object_pool_.FindObject(object, kNotPatchable));
1578+
ObjectPool::element_offset(object_pool_wrapper_.FindObject(object));
15791579
LoadWordFromPoolOffset(rd, offset - kHeapObjectTag, cond);
15801580
}
15811581
}
@@ -2660,8 +2660,8 @@ void Assembler::BranchLinkPatchable(const ExternalLabel* label) {
26602660
// to by this code sequence.
26612661
// For added code robustness, use 'blx lr' in a patchable sequence and
26622662
// use 'blx ip' in a non-patchable sequence (see other BranchLink flavors).
2663-
const int32_t offset =
2664-
Array::element_offset(object_pool_.FindExternalLabel(label, kPatchable));
2663+
const int32_t offset = ObjectPool::element_offset(
2664+
object_pool_wrapper_.FindExternalLabel(label, kPatchable));
26652665
LoadWordFromPoolOffset(LR, offset - kHeapObjectTag);
26662666
blx(LR); // Use blx instruction so that the return branch prediction works.
26672667
}

‎runtime/vm/assembler_arm.h‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -341,11 +341,11 @@ class Assembler : public ValueObject {
341341
return buffer_.pointer_offsets();
342342
}
343343

344-
const GrowableObjectArray& object_pool_data() const {
345-
return object_pool_.data();
346-
}
344+
ObjectPoolWrapper& object_pool_wrapper() { return object_pool_wrapper_; }
347345

348-
ObjectPool& object_pool() { return object_pool_; }
346+
RawObjectPool* MakeObjectPool() {
347+
return object_pool_wrapper_.MakeObjectPool();
348+
}
349349

350350
bool use_far_branches() const {
351351
return FLAG_use_far_branches || use_far_branches_;
@@ -961,7 +961,7 @@ class Assembler : public ValueObject {
961961

962962
private:
963963
AssemblerBuffer buffer_; // Contains position independent code.
964-
ObjectPool object_pool_; // Objects and patchable jump targets.
964+
ObjectPoolWrapper object_pool_wrapper_;
965965

966966
int32_t prologue_offset_;
967967

‎runtime/vm/assembler_arm64.cc‎

Lines changed: 15 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -35,29 +35,29 @@ Assembler::Assembler(bool use_far_branches)
3535
// These objects and labels need to be accessible through every pool-pointer
3636
// at the same index.
3737
intptr_t index =
38-
object_pool_.AddObject(Object::null_object(), kNotPatchable);
38+
object_pool_wrapper_.AddObject(Object::null_object());
3939
ASSERT(index == 0);
4040

41-
index = object_pool_.AddObject(Bool::True(), kNotPatchable);
41+
index = object_pool_wrapper_.AddObject(Bool::True());
4242
ASSERT(index == 1);
4343

44-
index = object_pool_.AddObject(Bool::False(), kNotPatchable);
44+
index = object_pool_wrapper_.AddObject(Bool::False());
4545
ASSERT(index == 2);
4646

4747
const Smi& vacant = Smi::Handle(Smi::New(0xfa >> kSmiTagShift));
4848
StubCode* stub_code = Isolate::Current()->stub_code();
4949
if (stub_code->UpdateStoreBuffer_entry() != NULL) {
50-
object_pool_.AddExternalLabel(
50+
object_pool_wrapper_.AddExternalLabel(
5151
&stub_code->UpdateStoreBufferLabel(), kNotPatchable);
5252
} else {
53-
object_pool_.AddObject(vacant, kNotPatchable);
53+
object_pool_wrapper_.AddObject(vacant);
5454
}
5555

5656
if (stub_code->CallToRuntime_entry() != NULL) {
57-
object_pool_.AddExternalLabel(
57+
object_pool_wrapper_.AddExternalLabel(
5858
&stub_code->CallToRuntimeLabel(), kNotPatchable);
5959
} else {
60-
object_pool_.AddObject(vacant, kNotPatchable);
60+
object_pool_wrapper_.AddObject(vacant);
6161
}
6262
}
6363
}
@@ -390,8 +390,7 @@ void Assembler::LoadWordFromPoolOffsetFixed(Register dst, Register pp,
390390

391391
intptr_t Assembler::FindImmediate(int64_t imm) {
392392
ASSERT(Isolate::Current() != Dart::vm_isolate());
393-
const Smi& smi = Smi::Handle(reinterpret_cast<RawSmi*>(imm));
394-
return object_pool_.FindObject(smi, kNotPatchable);
393+
return object_pool_wrapper_.FindImmediate(imm);
395394
}
396395

397396

@@ -441,8 +440,8 @@ void Assembler::LoadExternalLabel(Register dst,
441440
Register pp) {
442441
const int64_t target = static_cast<int64_t>(label->address());
443442
if (CanLoadImmediateFromPool(target, pp)) {
444-
const int32_t offset =
445-
Array::element_offset(object_pool_.FindExternalLabel(label, patchable));
443+
const int32_t offset = ObjectPool::element_offset(
444+
object_pool_wrapper_.FindExternalLabel(label, patchable));
446445
LoadWordFromPoolOffset(dst, pp, offset);
447446
} else {
448447
LoadImmediate(dst, target, kNoPP);
@@ -454,8 +453,8 @@ void Assembler::LoadExternalLabelFixed(Register dst,
454453
const ExternalLabel* label,
455454
Patchability patchable,
456455
Register pp) {
457-
const int32_t offset =
458-
Array::element_offset(object_pool_.FindExternalLabel(label, patchable));
456+
const int32_t offset = ObjectPool::element_offset(
457+
object_pool_wrapper_.FindExternalLabel(label, patchable));
459458
LoadWordFromPoolOffsetFixed(dst, pp, offset);
460459
}
461460

@@ -468,7 +467,7 @@ void Assembler::LoadIsolate(Register dst, Register pp) {
468467
void Assembler::LoadObject(Register dst, const Object& object, Register pp) {
469468
if (CanLoadObjectFromPool(object)) {
470469
const int32_t offset =
471-
Array::element_offset(object_pool_.FindObject(object, kNotPatchable));
470+
ObjectPool::element_offset(object_pool_wrapper_.FindObject(object));
472471
LoadWordFromPoolOffset(dst, pp, offset);
473472
} else {
474473
ASSERT((Isolate::Current() == Dart::vm_isolate()) ||
@@ -495,7 +494,7 @@ void Assembler::LoadDecodableImmediate(Register reg, int64_t imm, Register pp) {
495494
allow_constant_pool()) {
496495
int64_t val_smi_tag = imm & kSmiTagMask;
497496
imm &= ~kSmiTagMask; // Mask off the tag bits.
498-
const int32_t offset = Array::element_offset(FindImmediate(imm));
497+
const int32_t offset = ObjectPool::element_offset(FindImmediate(imm));
499498
LoadWordFromPoolOffset(reg, pp, offset);
500499
if (val_smi_tag != 0) {
501500
// Add back the tag bits.
@@ -531,7 +530,7 @@ void Assembler::LoadImmediate(Register reg, int64_t imm, Register pp) {
531530
// Save the bits that must be masked-off for the SmiTag
532531
int64_t val_smi_tag = imm & kSmiTagMask;
533532
imm &= ~kSmiTagMask; // Mask off the tag bits.
534-
const int32_t offset = Array::element_offset(FindImmediate(imm));
533+
const int32_t offset = ObjectPool::element_offset(FindImmediate(imm));
535534
LoadWordFromPoolOffset(reg, pp, offset);
536535
if (val_smi_tag != 0) {
537536
// Add back the tag bits.

0 commit comments

Comments
 (0)