[vm, isolate] Merge heaps without evacuating new-space.

Change-Id: I03d65e866215f243f2ab6a4a1d9d41625be4d56f
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/148682
Reviewed-by: Martin Kustermann <[email protected]>
Commit-Queue: Ryan Macnak <[email protected]>

Author: rmacnak-google

runtime/vm/dart_api_impl.cc

@@ -1309,12 +1309,10 @@ Isolate* CreateWithinExistingIsolateGroup(IsolateGroup* group,
   {
     TransitionNativeToVM native_to_vm(thread);
 
-    // Ensure new space is empty and there are no threads running.
+    // Ensure there are no helper threads running.
     BackgroundCompiler::Stop(isolate);
-    isolate->heap()->new_space()->Evacuate();
     isolate->heap()->WaitForMarkerTasks(thread);
     isolate->heap()->WaitForSweeperTasks(thread);
-    RELEASE_ASSERT(isolate->heap()->new_space()->UsedInWords() == 0);
     RELEASE_ASSERT(isolate->heap()->old_space()->tasks() == 0);
   }
 
@@ -1344,7 +1342,7 @@ Isolate* CreateWithinExistingIsolateGroup(IsolateGroup* group,
         // Merge the heap from [spawning_group] to [group].
         {
           SafepointOperationScope safepoint_scope(thread);
-          group->heap()->MergeOtherHeap(isolate->group()->heap());
+          group->heap()->MergeFrom(isolate->group()->heap());
         }
 
         spawning_group->UnregisterIsolate(isolate);

runtime/vm/heap/freelist.cc

@@ -377,13 +377,13 @@ FreeListElement* FreeList::TryAllocateLargeLocked(intptr_t minimum_size) {
   return NULL;
 }
 
-void FreeList::MergeOtherFreelist(FreeList* other, bool is_protected) {
+void FreeList::MergeFrom(FreeList* donor, bool is_protected) {
   // The [other] free list is from a dying isolate. There are no other threads
   // accessing it, so there is no need to lock here.
   MutexLocker ml(&mutex_);
   for (intptr_t i = 0; i < (kNumLists + 1); ++i) {
-    FreeListElement* other_head = other->free_lists_[i];
-    if (other_head != nullptr) {
+    FreeListElement* donor_head = donor->free_lists_[i];
+    if (donor_head != nullptr) {
       // If we didn't have a freelist element before we have to set the bit now,
       // since we will get 1+ elements from [other].
       FreeListElement* old_head = free_lists_[i];
@@ -392,7 +392,7 @@ void FreeList::MergeOtherFreelist(FreeList* other, bool is_protected) {
       }
 
       // Chain other's list in.
-      FreeListElement* last = other_head;
+      FreeListElement* last = donor_head;
       while (last->next() != nullptr) {
         last = last->next();
       }
@@ -406,12 +406,12 @@ void FreeList::MergeOtherFreelist(FreeList* other, bool is_protected) {
         VirtualMemory::Protect(reinterpret_cast<void*>(last), sizeof(*last),
                                VirtualMemory::kReadExecute);
       }
-      free_lists_[i] = other_head;
+      free_lists_[i] = donor_head;
     }
   }
 
   last_free_small_size_ =
-      Utils::Maximum(last_free_small_size_, other->last_free_small_size_);
+      Utils::Maximum(last_free_small_size_, donor->last_free_small_size_);
 }
 
 }  // namespace dart

runtime/vm/heap/freelist.h

@@ -157,7 +157,7 @@ class FreeList {
   void set_end(uword value) { end_ = value; }
   void AddUnaccountedSize(intptr_t size) { unaccounted_size_ += size; }
 
-  void MergeOtherFreelist(FreeList* freelist, bool is_protected);
+  void MergeFrom(FreeList* donor, bool is_protected);
 
  private:
   static const int kNumLists = 128;

runtime/vm/heap/heap.cc

@@ -739,19 +739,19 @@ void Heap::CollectOnNthAllocation(intptr_t num_allocations) {
   gc_on_nth_allocation_ = num_allocations;
 }
 
-void Heap::MergeOtherHeap(Heap* other) {
-  ASSERT(!other->gc_new_space_in_progress_);
-  ASSERT(!other->gc_old_space_in_progress_);
-  ASSERT(!other->read_only_);
-  ASSERT(other->new_space()->UsedInWords() == 0);
-  ASSERT(other->old_space()->tasks() == 0);
+void Heap::MergeFrom(Heap* donor) {
+  ASSERT(!donor->gc_new_space_in_progress_);
+  ASSERT(!donor->gc_old_space_in_progress_);
+  ASSERT(!donor->read_only_);
+  ASSERT(donor->old_space()->tasks() == 0);
 
-  old_space_.MergeOtherPageSpace(other->old_space());
+  new_space_.MergeFrom(donor->new_space());
+  old_space_.MergeFrom(donor->old_space());
 
   for (intptr_t i = 0; i < kNumWeakSelectors; ++i) {
     // The new space rehashing should not be necessary.
-    new_weak_tables_[i]->MergeOtherWeakTable(other->new_weak_tables_[i]);
-    old_weak_tables_[i]->MergeOtherWeakTable(other->old_weak_tables_[i]);
+    new_weak_tables_[i]->MergeFrom(donor->new_weak_tables_[i]);
+    old_weak_tables_[i]->MergeFrom(donor->old_weak_tables_[i]);
   }
 }
 

runtime/vm/heap/heap.h

@@ -307,7 +307,7 @@ class Heap {
 
   void CollectOnNthAllocation(intptr_t num_allocations);
 
-  void MergeOtherHeap(Heap* other);
+  void MergeFrom(Heap* donor);
 
  private:
   class GCStats : public ValueObject {

runtime/vm/heap/pages.cc

@@ -1437,54 +1437,53 @@ static void EnsureEqualImagePages(OldPage* pages, OldPage* other_pages) {
 #endif
 }
 
-void PageSpace::MergeOtherPageSpace(PageSpace* other) {
-  other->AbandonBumpAllocation();
+void PageSpace::MergeFrom(PageSpace* donor) {
+  donor->AbandonBumpAllocation();
 
-  ASSERT(other->tasks_ == 0);
-  ASSERT(other->concurrent_marker_tasks_ == 0);
-  ASSERT(other->phase_ == kDone);
-  DEBUG_ASSERT(other->iterating_thread_ == nullptr);
-  ASSERT(other->marker_ == nullptr);
+  ASSERT(donor->tasks_ == 0);
+  ASSERT(donor->concurrent_marker_tasks_ == 0);
+  ASSERT(donor->phase_ == kDone);
+  DEBUG_ASSERT(donor->iterating_thread_ == nullptr);
+  ASSERT(donor->marker_ == nullptr);
 
   for (intptr_t i = 0; i < num_freelists_; ++i) {
-    ASSERT(other->freelists_[i].top() == 0);
-    ASSERT(other->freelists_[i].end() == 0);
+    ASSERT(donor->freelists_[i].top() == 0);
+    ASSERT(donor->freelists_[i].end() == 0);
     const bool is_protected =
         FLAG_write_protect_code && i == OldPage::kExecutable;
-    freelists_[i].MergeOtherFreelist(&other->freelists_[i], is_protected);
-    other->freelists_[i].Reset();
+    freelists_[i].MergeFrom(&donor->freelists_[i], is_protected);
+    donor->freelists_[i].Reset();
   }
 
   // The freelist locks will be taken in MergeOtherFreelist above, and the
   // locking order is the freelist locks are taken before the page list locks,
   // so don't take the pages lock until after MergeOtherFreelist.
   MutexLocker ml(&pages_lock_);
-  MutexLocker ml2(&other->pages_lock_);
+  MutexLocker ml2(&donor->pages_lock_);
 
-  AppendList(&pages_, &pages_tail_, &other->pages_, &other->pages_tail_);
-  AppendList(&exec_pages_, &exec_pages_tail_, &other->exec_pages_,
-             &other->exec_pages_tail_);
-  AppendList(&large_pages_, &large_pages_tail_, &other->large_pages_,
-             &other->large_pages_tail_);
+  AppendList(&pages_, &pages_tail_, &donor->pages_, &donor->pages_tail_);
+  AppendList(&exec_pages_, &exec_pages_tail_, &donor->exec_pages_,
+             &donor->exec_pages_tail_);
+  AppendList(&large_pages_, &large_pages_tail_, &donor->large_pages_,
+             &donor->large_pages_tail_);
   // We intentionall do not merge [image_pages_] beause [this] and [other] have
   // the same mmap()ed image page areas.
-  EnsureEqualImagePages(image_pages_, other->image_pages_);
+  EnsureEqualImagePages(image_pages_, donor->image_pages_);
 
   // We intentionaly do not increase [max_capacity_in_words_] because this can
   // lead [max_capacity_in_words_] to become larger and larger and eventually
   // wrap-around and become negative.
-  allocated_black_in_words_ += other->allocated_black_in_words_;
-  gc_time_micros_ += other->gc_time_micros_;
-  collections_ += other->collections_;
+  allocated_black_in_words_ += donor->allocated_black_in_words_;
+  gc_time_micros_ += donor->gc_time_micros_;
+  collections_ += donor->collections_;
 
-  usage_.capacity_in_words += other->usage_.capacity_in_words;
-  usage_.used_in_words += other->usage_.used_in_words;
-  usage_.external_in_words += other->usage_.external_in_words;
+  usage_.capacity_in_words += donor->usage_.capacity_in_words;
+  usage_.used_in_words += donor->usage_.used_in_words;
+  usage_.external_in_words += donor->usage_.external_in_words;
 
-  page_space_controller_.MergeOtherPageSpaceController(
-      &other->page_space_controller_);
+  page_space_controller_.MergeFrom(&donor->page_space_controller_);
 
-  ASSERT(FLAG_concurrent_mark || other->enable_concurrent_mark_ == false);
+  ASSERT(FLAG_concurrent_mark || donor->enable_concurrent_mark_ == false);
 }
 
 PageSpaceController::PageSpaceController(Heap* heap,
@@ -1697,11 +1696,10 @@ void PageSpaceController::RecordUpdate(SpaceUsage before,
   }
 }
 
-void PageSpaceController::MergeOtherPageSpaceController(
-    PageSpaceController* other) {
-  last_usage_.capacity_in_words += other->last_usage_.capacity_in_words;
-  last_usage_.used_in_words += other->last_usage_.used_in_words;
-  last_usage_.external_in_words += other->last_usage_.external_in_words;
+void PageSpaceController::MergeFrom(PageSpaceController* donor) {
+  last_usage_.capacity_in_words += donor->last_usage_.capacity_in_words;
+  last_usage_.used_in_words += donor->last_usage_.used_in_words;
+  last_usage_.external_in_words += donor->last_usage_.external_in_words;
 }
 
 void PageSpaceGarbageCollectionHistory::AddGarbageCollectionTime(int64_t start,

runtime/vm/heap/pages.h

@@ -239,7 +239,7 @@ class PageSpaceController {
   friend class PageSpace;  // For MergeOtherPageSpaceController
 
   void RecordUpdate(SpaceUsage before, SpaceUsage after, const char* reason);
-  void MergeOtherPageSpaceController(PageSpaceController* other);
+  void MergeFrom(PageSpaceController* donor);
 
   void RecordUpdate(SpaceUsage before,
                     SpaceUsage after,
@@ -482,7 +482,7 @@ class PageSpace {
 
   bool IsObjectFromImagePages(ObjectPtr object);
 
-  void MergeOtherPageSpace(PageSpace* other);
+  void MergeFrom(PageSpace* donor);
 
  private:
   // Ids for time and data records in Heap::GCStats.

runtime/vm/heap/scavenger.cc

@@ -673,6 +673,19 @@ void SemiSpace::AddList(NewPage* head, NewPage* tail) {
   tail_ = tail;
 }
 
+void SemiSpace::MergeFrom(SemiSpace* donor) {
+  for (NewPage* page = donor->head_; page != nullptr; page = page->next()) {
+    page->Release();
+  }
+
+  AddList(donor->head_, donor->tail_);
+  capacity_in_words_ += donor->capacity_in_words_;
+
+  donor->head_ = nullptr;
+  donor->tail_ = nullptr;
+  donor->capacity_in_words_ = 0;
+}
+
 // The initial estimate of how many words we can scavenge per microsecond (usage
 // before / scavenge time). This is a conservative value observed running
 // Flutter on a Nexus 4. After the first scavenge, we instead use a value based
@@ -1578,4 +1591,13 @@ void Scavenger::Evacuate() {
   ASSERT((UsedInWords() == 0) || failed_to_promote_);
 }
 
+void Scavenger::MergeFrom(Scavenger* donor) {
+  MutexLocker ml(&space_lock_);
+  MutexLocker ml2(&donor->space_lock_);
+  to_->MergeFrom(donor->to_);
+
+  external_size_ += donor->external_size_;
+  donor->external_size_ = 0;
+}
+
 }  // namespace dart

runtime/vm/heap/scavenger.h

@@ -183,6 +183,7 @@ class SemiSpace {
   NewPage* head() const { return head_; }
 
   void AddList(NewPage* head, NewPage* tail);
+  void MergeFrom(SemiSpace* donor);
 
  private:
   // Size of NewPages in this semi-space.
@@ -279,6 +280,8 @@ class Scavenger {
   // Promote all live objects.
   void Evacuate();
 
+  void MergeFrom(Scavenger* donor);
+
   int64_t UsedInWords() const {
     MutexLocker ml(&space_lock_);
     return to_->capacity_in_words();

runtime/vm/heap/weak_table.cc

@@ -132,10 +132,10 @@ void WeakTable::Rehash() {
   free(old_data);
 }
 
-void WeakTable::MergeOtherWeakTable(WeakTable* other) {
-  for (intptr_t i = 0; i < other->size(); i++) {
-    if (other->IsValidEntryAtExclusive(i)) {
-      SetValueExclusive(other->ObjectAtExclusive(i), ValueIndex(i));
+void WeakTable::MergeFrom(WeakTable* donor) {
+  for (intptr_t i = 0; i < donor->size(); i++) {
+    if (donor->IsValidEntryAtExclusive(i)) {
+      SetValueExclusive(donor->ObjectAtExclusive(i), ValueIndex(i));
     }
   }
 }

runtime/vm/heap/weak_table.h

@@ -130,7 +130,7 @@ class WeakTable {
 
   void Reset();
 
-  void MergeOtherWeakTable(WeakTable* other);
+  void MergeFrom(WeakTable* donor);
 
  private:
   enum {