Move Software Write Watch's write barrier updates to use the new

GCToEEInterface::StompWriteBarrier to stomp the EE's write barrier.

Author: swgillespie

src/gc/gc.cpp

@@ -1402,9 +1402,6 @@ int mark_time, plan_time, sweep_time, reloc_time, compact_time;
 
 #ifndef MULTIPLE_HEAPS
 
-#define ephemeral_low           g_gc_ephemeral_low
-#define ephemeral_high          g_gc_ephemeral_high
-
 #endif // MULTIPLE_HEAPS
 
 #ifdef TRACE_GC
@@ -2187,26 +2184,32 @@ void stomp_write_barrier_resize(bool is_runtime_suspended, bool requires_upper_b
     args.card_table = g_gc_card_table;
     args.lowest_address = g_gc_lowest_address;
     args.highest_address = g_gc_highest_address;
+#ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+    if (SoftwareWriteWatch::IsEnabledForGCHeap())
+    {
+        args.write_watch_table = g_gc_sw_ww_table;
+    }
+#endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
     GCToEEInterface::StompWriteBarrier(&args);
 }
 
-void stomp_write_barrier_ephemeral(bool is_runtime_suspended, uint8_t* ephemeral_lo, uint8_t* ephemeral_hi)
+void stomp_write_barrier_ephemeral(bool is_runtime_suspended, uint8_t* ephemeral_low, uint8_t* ephemeral_high)
 {
     WriteBarrierParameters args = {};
     args.operation = WriteBarrierOp::StompEphemeral;
     args.is_runtime_suspended = is_runtime_suspended;
-    args.ephemeral_lo = g_gc_ephemeral_low;
-    args.ephemeral_hi = g_gc_ephemeral_high;
+    args.ephemeral_low = ephemeral_low;
+    args.ephemeral_high = ephemeral_high;
 #ifdef MULTIPLE_HEAPS
     // It is not correct to update the EE's g_ephemeral_low and g_ephemeral_high
     // to anything other than their default values when using Server GC, since
     // there is no single ephemeral generation across all of the heaps.
     // Server GC write barriers do not reference these two globals, but ErectWriteBarrier does.
     //
-    // When MULTIPLE_HEAPS is defined, g_gc_ephemeral_low and g_gc_ephemeral_high should
+    // When MULTIPLE_HEAPS is defined, the EE's g_ephemeral_low and g_ephemeral_high should
     // always have their default values.
-    assert(args.ephemeral_lo == (uint8_t*)1);
-    assert(args.ephemeral_hi == (uint8_t*)~0);
+    args.ephemeral_low = (uint8_t*)1;
+    args.ephemeral_high = (uint8_t*)~0;
 #endif // MULTIPLE_HEAPS
     GCToEEInterface::StompWriteBarrier(&args);
 }
@@ -2430,6 +2433,10 @@ BOOL        gc_heap::ro_segments_in_range;
 
 size_t      gc_heap::gen0_big_free_spaces = 0;
 
+uint8_t*    gc_heap::ephemeral_low;
+
+uint8_t*    gc_heap::ephemeral_high;
+
 uint8_t*    gc_heap::lowest_address;
 
 uint8_t*    gc_heap::highest_address;
@@ -7277,9 +7284,6 @@ int gc_heap::grow_brick_card_tables (uint8_t* start,
             }
 
             g_gc_card_table = translated_ct;
-            g_gc_lowest_address = saved_g_lowest_address;
-            g_gc_highest_address = saved_g_highest_address;
-
             SoftwareWriteWatch::SetResizedUntranslatedTable(
                 mem + sw_ww_table_offset,
                 saved_g_lowest_address,
@@ -7290,6 +7294,8 @@ int gc_heap::grow_brick_card_tables (uint8_t* start,
             // grow version of the write barrier.  This test tells us if the new
             // segment was allocated at a lower address than the old, requiring
             // that we start doing an upper bounds check in the write barrier.
+            g_gc_lowest_address = saved_g_lowest_address;
+            g_gc_highest_address = saved_g_highest_address;
             stomp_write_barrier_resize(true, la != saved_g_lowest_address);
             write_barrier_updated = true;
 
@@ -32778,8 +32784,8 @@ gc_heap::verify_heap (BOOL begin_gc_p)
 #endif //BACKGROUND_GC 
 
 #ifndef MULTIPLE_HEAPS
-    if ((g_gc_ephemeral_low != generation_allocation_start (generation_of (max_generation - 1))) ||
-        (g_gc_ephemeral_high != heap_segment_reserved (ephemeral_heap_segment)))
+    if ((ephemeral_low != generation_allocation_start (generation_of (max_generation - 1))) ||
+        (ephemeral_high != heap_segment_reserved (ephemeral_heap_segment)))
     {
         FATAL_GC_ERROR();
     }
@@ -35739,7 +35745,13 @@ GCHeap::SetCardsAfterBulkCopy( Object **StartPoint, size_t len )
     end = StartPoint + (len/sizeof(Object*));
     while (rover < end)
     {
-        if ( (((uint8_t*)*rover) >= g_gc_ephemeral_low) && (((uint8_t*)*rover) < g_gc_ephemeral_high) )
+        // For Workstation GC, where there is exactly one ephemeral region, we can avoid touching the
+        // card table if the rover is currently pointing into a region outside of the ephemeral generation.
+        //
+        // For Server GC, where there are multiple ephemeral regions, we must always touch the card table.
+#ifndef MULTIPLE_HEAPS
+        if ( (((uint8_t*)*rover) >= gc_heap::ephemeral_low) && (((uint8_t*)*rover) < gc_heap::ephemeral_high) )
+#endif // MULTIPLE_HEAPS
         {
             // Set Bit For Card and advance to next card
             size_t card = gcard_of ((uint8_t*)rover);
@@ -35748,10 +35760,12 @@ GCHeap::SetCardsAfterBulkCopy( Object **StartPoint, size_t len )
             // Skip to next card for the object
             rover = (Object**)align_on_card ((uint8_t*)(rover+1));
         }
+#ifndef MULTIPLE_HEAPS
         else
         {
             rover++;
         }
+#endif // MULTIPLE_HEAPS
     }
 }
 

src/gc/gc.h

@@ -140,8 +140,6 @@ class DacHeapWalker;
 extern "C" uint32_t* g_gc_card_table;
 extern "C" uint8_t* g_gc_lowest_address;
 extern "C" uint8_t* g_gc_highest_address;
-extern "C" uint8_t* g_gc_ephemeral_low;
-extern "C" uint8_t* g_gc_ephemeral_high;
 
 namespace WKS {
     ::IGCHeapInternal* CreateGCHeap();

src/gc/gccommon.cpp

@@ -41,8 +41,6 @@ uint8_t* g_shadow_lowest_address = NULL;
 uint32_t* g_gc_card_table;
 uint8_t* g_gc_lowest_address  = 0;
 uint8_t* g_gc_highest_address = 0;
-uint8_t* g_gc_ephemeral_low   = (uint8_t*)1;
-uint8_t* g_gc_ephemeral_high  = (uint8_t*)~0;
 
 VOLATILE(int32_t) m_GCLock = -1;
 

src/gc/gcinterface.h

@@ -46,7 +46,9 @@ enum class WriteBarrierOp
 {
     StompResize,
     StompEphemeral,
-    Initialize
+    Initialize,
+    SwitchToWriteWatch,
+    SwitchToNonWriteWatch
 };
 
 // Arguments to GCToEEInterface::StompWriteBarrier
@@ -85,11 +87,15 @@ struct WriteBarrierParameters
 
     // The new start of the ephemeral generation. 
     // Used for WriteBarrierOp::StompEphemeral.
-    uint8_t* ephemeral_lo;
+    uint8_t* ephemeral_low;
 
     // The new end of the ephemeral generation.
     // Used for WriteBarrierOp::StompEphemeral.
-    uint8_t* ephemeral_hi;
+    uint8_t* ephemeral_high;
+
+    // The new write watch table, if we are using our own write watch
+    // implementation. Used for WriteBarrierOp::SwitchToWriteWatch only.
+    uint8_t* write_watch_table;
 };
 
 #include "gcinterface.ee.h"
@@ -148,6 +154,10 @@ struct segment_info
 
 #define max_generation 2
 
+// The bit shift used to convert a memory address into an index into the
+// Software Write Watch table.
+#define SOFTWARE_WRITE_WATCH_AddressToTableByteIndexShift 0xc
+
 class Object;
 class IGCHeap;
 

src/gc/gcpriv.h

@@ -2802,13 +2802,11 @@ class gc_heap
     PER_HEAP
     void exit_gc_done_event_lock();
 
-#ifdef MULTIPLE_HEAPS
     PER_HEAP
     uint8_t*  ephemeral_low;      //lowest ephemeral address
 
     PER_HEAP
     uint8_t*  ephemeral_high;     //highest ephemeral address
-#endif //MULTIPLE_HEAPS
 
     PER_HEAP
     uint32_t* card_table;

src/gc/gcsvr.cpp

@@ -13,6 +13,7 @@
 #include "gc.h"
 #include "gcscan.h"
 #include "gcdesc.h"
+#include "softwarewritewatch.h"
 
 #define SERVER_GC 1
 

src/gc/gcwks.cpp

@@ -11,6 +11,7 @@
 #include "gc.h"
 #include "gcscan.h"
 #include "gcdesc.h"
+#include "softwarewritewatch.h"
 
 #ifdef SERVER_GC
 #undef SERVER_GC

src/gc/sample/GCSample.cpp

@@ -94,14 +94,11 @@ inline void ErectWriteBarrier(Object ** dst, Object * ref)
     if (((uint8_t*)dst < g_gc_lowest_address) || ((uint8_t*)dst >= g_gc_highest_address))
         return;
 
-    if((uint8_t*)ref >= g_gc_ephemeral_low && (uint8_t*)ref < g_gc_ephemeral_high)
-    {
-        // volatile is used here to prevent fetch of g_card_table from being reordered 
-        // with g_lowest/highest_address check above. See comment in code:gc_heap::grow_brick_card_tables.
-        uint8_t* pCardByte = (uint8_t *)*(volatile uint8_t **)(&g_gc_card_table) + card_byte((uint8_t *)dst);
-        if(*pCardByte != 0xFF)
-            *pCardByte = 0xFF;
-    }
+    // volatile is used here to prevent fetch of g_card_table from being reordered 
+    // with g_lowest/highest_address check above. See comment in code:gc_heap::grow_brick_card_tables.
+    uint8_t* pCardByte = (uint8_t *)*(volatile uint8_t **)(&g_gc_card_table) + card_byte((uint8_t *)dst);
+    if(*pCardByte != 0xFF)
+        *pCardByte = 0xFF;
 }
 
 void WriteBarrier(Object ** dst, Object * ref)

src/gc/softwarewritewatch.cpp

@@ -3,10 +3,9 @@
 // See the LICENSE file in the project root for more information.
 
 #include "common.h"
-#include "softwarewritewatch.h"
-
 #include "gcenv.h"
 #include "env/gcenv.os.h"
+#include "softwarewritewatch.h"
 
 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
 #ifndef DACCESS_COMPILE
@@ -15,8 +14,8 @@ static_assert((static_cast<size_t>(1) << SOFTWARE_WRITE_WATCH_AddressToTableByte
 
 extern "C"
 {
-    uint8_t *g_sw_ww_table = nullptr;
-    bool g_sw_ww_enabled_for_gc_heap = false;
+    uint8_t *g_gc_sw_ww_table = nullptr;
+    bool g_gc_sw_ww_enabled_for_gc_heap = false;
 }
 
 void SoftwareWriteWatch::StaticClose()
@@ -26,8 +25,8 @@ void SoftwareWriteWatch::StaticClose()
         return;
     }
 
-    g_sw_ww_enabled_for_gc_heap = false;
-    g_sw_ww_table = nullptr;
+    g_gc_sw_ww_enabled_for_gc_heap = false;
+    g_gc_sw_ww_table = nullptr;
 }
 
 bool SoftwareWriteWatch::GetDirtyFromBlock(

src/gc/softwarewritewatch.h

@@ -5,25 +5,20 @@
 #ifndef __SOFTWARE_WRITE_WATCH_H__
 #define __SOFTWARE_WRITE_WATCH_H__
 
+#include "gcinterface.h"
+#include "gc.h"
+
 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
 #ifndef DACCESS_COMPILE
 
-extern void SwitchToWriteWatchBarrier(bool isRuntimeSuspended);
-extern void SwitchToNonWriteWatchBarrier(bool isRuntimeSuspended);
-
-#define SOFTWARE_WRITE_WATCH_AddressToTableByteIndexShift 0xc
-
 extern "C"
 {
     // Table containing the dirty state. This table is translated to exclude the lowest address it represents, see
     // TranslateTableToExcludeHeapStartAddress.
-    extern uint8_t *g_sw_ww_table;
+    extern uint8_t *g_gc_sw_ww_table;
 
     // Write watch may be disabled when it is not needed (between GCs for instance). This indicates whether it is enabled.
-    extern bool g_sw_ww_enabled_for_gc_heap;
-
-    extern uint8_t *g_lowest_address; // start address of the GC heap
-    extern uint8_t *g_highest_address; // end address of the GC heap
+    extern bool g_gc_sw_ww_enabled_for_gc_heap;
 }
 
 class SoftwareWriteWatch
@@ -116,7 +111,7 @@ inline void SoftwareWriteWatch::VerifyMemoryRegion(
 
 inline uint8_t *SoftwareWriteWatch::GetTable()
 {
-    return g_sw_ww_table;
+    return g_gc_sw_ww_table;
 }
 
 inline uint8_t *SoftwareWriteWatch::GetUntranslatedTable()
@@ -163,7 +158,7 @@ inline void SoftwareWriteWatch::SetUntranslatedTable(uint8_t *untranslatedTable,
     assert(ALIGN_DOWN(untranslatedTable, sizeof(size_t)) == untranslatedTable);
     assert(heapStartAddress != nullptr);
 
-    g_sw_ww_table = TranslateTableToExcludeHeapStartAddress(untranslatedTable, heapStartAddress);
+    g_gc_sw_ww_table = TranslateTableToExcludeHeapStartAddress(untranslatedTable, heapStartAddress);
 }
 
 inline void SoftwareWriteWatch::SetResizedUntranslatedTable(
@@ -194,7 +189,7 @@ inline void SoftwareWriteWatch::SetResizedUntranslatedTable(
 
 inline bool SoftwareWriteWatch::IsEnabledForGCHeap()
 {
-    return g_sw_ww_enabled_for_gc_heap;
+    return g_gc_sw_ww_enabled_for_gc_heap;
 }
 
 inline void SoftwareWriteWatch::EnableForGCHeap()
@@ -204,9 +199,13 @@ inline void SoftwareWriteWatch::EnableForGCHeap()
 
     VerifyCreated();
     assert(!IsEnabledForGCHeap());
+    g_gc_sw_ww_enabled_for_gc_heap = true;
 
-    g_sw_ww_enabled_for_gc_heap = true;
-    SwitchToWriteWatchBarrier(true);
+    WriteBarrierParameters args = {};
+    args.operation = WriteBarrierOp::SwitchToWriteWatch;
+    args.write_watch_table = g_gc_sw_ww_table;
+    args.is_runtime_suspended = true;
+    GCToEEInterface::StompWriteBarrier(&args);
 }
 
 inline void SoftwareWriteWatch::DisableForGCHeap()
@@ -216,19 +215,22 @@ inline void SoftwareWriteWatch::DisableForGCHeap()
 
     VerifyCreated();
     assert(IsEnabledForGCHeap());
+    g_gc_sw_ww_enabled_for_gc_heap = false;     
 
-    g_sw_ww_enabled_for_gc_heap = false;
-    SwitchToNonWriteWatchBarrier(true);
+    WriteBarrierParameters args = {};
+    args.operation = WriteBarrierOp::SwitchToNonWriteWatch;
+    args.is_runtime_suspended = true;
+    GCToEEInterface::StompWriteBarrier(&args);
 }
 
 inline void *SoftwareWriteWatch::GetHeapStartAddress()
 {
-    return g_lowest_address;
+    return g_gc_lowest_address;
 }
 
 inline void *SoftwareWriteWatch::GetHeapEndAddress()
 {
-    return g_highest_address;
+    return g_gc_highest_address;
 }
 
 inline size_t SoftwareWriteWatch::GetTableByteIndex(void *address)