[GR-44787] Parallelize full garbage collections.

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java

@@ -114,6 +114,14 @@ static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) {
         return result;
     }
 
+    /** Retract the latest allocation. */
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    static void retractAllocation(AlignedHeader that, UnsignedWord size) {
+        Pointer newTop = HeapChunk.getTopPointer(that).subtract(size);
+        assert newTop.aboveOrEqual(getObjectsStart(that));
+        HeapChunk.setTopPointer(that, newTop);
+    }
+
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     static UnsignedWord getCommittedObjectMemory(AlignedHeader that) {
         return HeapChunk.getEndOffset(that).subtract(getObjectsStartOffset());
@@ -169,10 +177,5 @@ static final class MemoryWalkerAccessImpl extends HeapChunk.MemoryWalkerAccessIm
         public boolean isAligned(AlignedHeapChunk.AlignedHeader heapChunk) {
             return true;
         }
-
-        @Override
-        public UnsignedWord getAllocationStart(AlignedHeapChunk.AlignedHeader heapChunk) {
-            return getObjectsStart(heapChunk);
-        }
     }
 }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java

@@ -63,6 +63,7 @@
 import com.oracle.svm.core.genscavenge.BasicCollectionPolicies.NeverCollect;
 import com.oracle.svm.core.genscavenge.HeapChunk.Header;
 import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader;
+import com.oracle.svm.core.genscavenge.parallel.ParallelGC;
 import com.oracle.svm.core.genscavenge.remset.RememberedSet;
 import com.oracle.svm.core.graal.RuntimeCompilation;
 import com.oracle.svm.core.heap.CodeReferenceMapDecoder;
@@ -126,6 +127,8 @@ public final class GCImpl implements GC {
     public String getName() {
         if (SubstrateOptions.UseEpsilonGC.getValue()) {
             return "Epsilon GC";
+        } else if (SubstrateOptions.UseParallelGC.getValue()) {
+            return "Parallel GC";
         } else {
             return "Serial GC";
         }
@@ -196,6 +199,10 @@ private void collectOperation(CollectionVMOperationData data) {
         assert VMOperation.isGCInProgress() : "Collection should be a VMOperation.";
         assert getCollectionEpoch().equal(data.getRequestingEpoch());
 
+        if (ParallelGC.isEnabled()) {
+            ParallelGC.singleton().initialize();
+        }
+
         timers.mutator.closeAt(data.getRequestingNanoTime());
         startCollectionOrExit();
 
@@ -1070,6 +1077,8 @@ private void scanGreyObjects(boolean isIncremental) {
         try {
             if (isIncremental) {
                 scanGreyObjectsLoop();
+            } else if (ParallelGC.isEnabled()) {
+                ParallelGC.singleton().waitForIdle();
             } else {
                 HeapImpl.getHeapImpl().getOldGeneration().scanGreyObjects();
             }
@@ -1092,32 +1101,37 @@ private static void scanGreyObjectsLoop() {
 
     @AlwaysInline("GC performance")
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    @SuppressWarnings("static-method")
     Object promoteObject(Object original, UnsignedWord header) {
         HeapImpl heap = HeapImpl.getHeapImpl();
         boolean isAligned = ObjectHeaderImpl.isAlignedHeader(header);
         Header<?> originalChunk = getChunk(original, isAligned);
+
+        /* If the parallel GC is used, then the space may be outdated or null. */
         Space originalSpace = HeapChunk.getSpace(originalChunk);
-        if (!originalSpace.isFromSpace()) {
+        assert originalSpace != null || ParallelGC.isEnabled() && ParallelGC.singleton().isInParallelPhase();
+        if (originalSpace == null || !originalSpace.isFromSpace()) {
+            /* Object was already promoted or is currently being promoted. */
             return original;
         }
 
         Object result = null;
         if (!completeCollection && originalSpace.getNextAgeForPromotion() < policy.getTenuringAge()) {
             if (isAligned) {
-                result = heap.getYoungGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace);
+                result = heap.getYoungGeneration().promoteAlignedObject(original, originalSpace);
             } else {
                 result = heap.getYoungGeneration().promoteUnalignedObject(original, (UnalignedHeader) originalChunk, originalSpace);
             }
             if (result == null) {
                 accounting.onSurvivorOverflowed();
             }
         }
-        if (result == null) { // complete collection, tenuring age reached, or survivor space full
+
+        /* Complete collection, tenuring age reached, or survivor space full. */
+        if (result == null) {
             if (isAligned) {
-                result = heap.getOldGeneration().promoteAlignedObject(original, (AlignedHeader) originalChunk, originalSpace);
+                result = heap.getOldGeneration().promoteAlignedObject(original, originalSpace);
             } else {
-                result = heap.getOldGeneration().promoteUnalignedObject(original, (UnalignedHeader) originalChunk, originalSpace);
+                result = heap.getOldGeneration().promoteUnalignedObject(original, (UnalignedHeader) originalChunk);
             }
             assert result != null : "promotion failure in old generation must have been handled";
         }
@@ -1151,7 +1165,7 @@ private void promotePinnedObject(PinnedObjectImpl pinned) {
                     }
                 }
                 if (!promoted) {
-                    heap.getOldGeneration().promoteChunk(originalChunk, isAligned, originalSpace);
+                    heap.getOldGeneration().promoteChunk(originalChunk, isAligned);
                 }
             }
         }
@@ -1240,7 +1254,7 @@ public static boolean hasNeverCollectPolicy() {
     }
 
     @Fold
-    GreyToBlackObjectVisitor getGreyToBlackObjectVisitor() {
+    public GreyToBlackObjectVisitor getGreyToBlackObjectVisitor() {
         return greyToBlackObjectVisitor;
     }
 

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GenScavengeMemoryPoolMXBeans.java

@@ -46,7 +46,7 @@ public class GenScavengeMemoryPoolMXBeans {
 
     @Platforms(Platform.HOSTED_ONLY.class)
     public static MemoryPoolMXBean[] createMemoryPoolMXBeans() {
-        if (SubstrateOptions.UseSerialGC.getValue()) {
+        if (SubstrateOptions.useSerialOrParallelGC()) {
             mxBeans = new AbstractMemoryPoolMXBean[]{
                             new EdenMemoryPoolMXBean(YOUNG_GEN_SCAVENGER, COMPLETE_SCAVENGER),
                             new SurvivorMemoryPoolMXBean(YOUNG_GEN_SCAVENGER, COMPLETE_SCAVENGER),

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java

@@ -27,8 +27,6 @@
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
 
-import com.oracle.svm.core.AlwaysInline;
-import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.heap.ObjectVisitor;
 import com.oracle.svm.core.log.Log;
 
@@ -55,46 +53,4 @@ public String getName() {
 
     /** Report some statistics about the Generation to a Log. */
     public abstract Log report(Log log, boolean traceHeapChunks);
-
-    /**
-     * Promote an Object to this Generation, typically by copying and leaving a forwarding pointer
-     * to the new Object in place of the original Object. If the object cannot be promoted due to
-     * insufficient capacity, returns {@code null}.
-     *
-     * This turns an Object from white to grey: the object is in this Generation, but has not yet
-     * had its interior pointers visited.
-     *
-     * @return a reference to the promoted object, which is different to the original reference if
-     *         promotion was done by copying, or {@code null} if there was insufficient capacity in
-     *         this generation.
-     */
-    @AlwaysInline("GC performance")
-    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    protected abstract Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHeader originalChunk, Space originalSpace);
-
-    /**
-     * Promote an Object to this Generation, typically by HeapChunk motion. If the object cannot be
-     * promoted due to insufficient capacity, returns {@code null}.
-     *
-     * This turns an Object from white to grey: the object is in this Generation, but has not yet
-     * had its interior pointers visited.
-     *
-     * @return a reference to the promoted object, which is the same as the original if the object
-     *         was promoted through HeapChunk motion, or {@code null} if there was insufficient
-     *         capacity in this generation.
-     */
-    @AlwaysInline("GC performance")
-    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    protected abstract Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk, Space originalSpace);
-
-    /**
-     * Promote a HeapChunk from its original space to the appropriate space in this generation if
-     * there is sufficient capacity.
-     *
-     * This turns all the Objects in the chunk from white to grey: the objects are in the target
-     * Space, but have not yet had their interior pointers visited.
-     *
-     * @return true on success, false if the there was insufficient capacity.
-     */
-    protected abstract boolean promoteChunk(HeapChunk.Header<?> originalChunk, boolean isAligned, Space originalSpace);
 }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java

@@ -31,11 +31,13 @@
 
 import com.oracle.svm.core.AlwaysInline;
 import com.oracle.svm.core.Uninterruptible;
+import com.oracle.svm.core.genscavenge.parallel.ParallelGC;
 import com.oracle.svm.core.genscavenge.remset.RememberedSet;
 import com.oracle.svm.core.heap.ObjectHeader;
 import com.oracle.svm.core.heap.ObjectReferenceVisitor;
 import com.oracle.svm.core.heap.ReferenceAccess;
 import com.oracle.svm.core.hub.LayoutEncoding;
+import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicLong;
 import com.oracle.svm.core.log.Log;
 
 /**
@@ -97,6 +99,8 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole
                 counters.noteForwardedReferent();
                 // Update the reference to point to the forwarded Object.
                 Object obj = ohi.getForwardedObject(p, header);
+                assert ParallelGC.isEnabled() && ParallelGC.singleton().isInParallelPhase() ||
+                                innerOffset < LayoutEncoding.getSizeFromObjectInGC(obj).rawValue();
                 Object offsetObj = (innerOffset == 0) ? obj : Word.objectToUntrackedPointer(obj).add(innerOffset).toObject();
                 ReferenceAccess.singleton().writeObjectAt(objRef, offsetObj, compressed);
                 RememberedSet.get().dirtyCardIfNecessary(holderObject, obj);
@@ -105,11 +109,12 @@ public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boole
 
             // Promote the Object if necessary, making it at least grey, and ...
             Object obj = p.toObject();
-            assert innerOffset < LayoutEncoding.getSizeFromObjectInGC(obj).rawValue();
             Object copy = GCImpl.getGCImpl().promoteObject(obj, header);
             if (copy != obj) {
                 // ... update the reference to point to the copy, making the reference black.
                 counters.noteCopiedReferent();
+                assert ParallelGC.isEnabled() && ParallelGC.singleton().isInParallelPhase() ||
+                                innerOffset < LayoutEncoding.getSizeFromObjectInGC(copy).rawValue();
                 Object offsetCopy = (innerOffset == 0) ? copy : Word.objectToUntrackedPointer(copy).add(innerOffset).toObject();
                 ReferenceAccess.singleton().writeObjectAt(objRef, offsetCopy, compressed);
             } else {
@@ -152,33 +157,31 @@ public interface Counters extends AutoCloseable {
         @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         void noteUnmodifiedReference();
 
-        void toLog();
-
         void reset();
     }
 
     public static class RealCounters implements Counters {
-        private long objRef;
-        private long nullObjRef;
-        private long nullReferent;
-        private long forwardedReferent;
-        private long nonHeapReferent;
-        private long copiedReferent;
-        private long unmodifiedReference;
+        private final AtomicLong objRef = new AtomicLong(0);
+        private final AtomicLong nullObjRef = new AtomicLong(0);
+        private final AtomicLong nullReferent = new AtomicLong(0);
+        private final AtomicLong forwardedReferent = new AtomicLong(0);
+        private final AtomicLong nonHeapReferent = new AtomicLong(0);
+        private final AtomicLong copiedReferent = new AtomicLong(0);
+        private final AtomicLong unmodifiedReference = new AtomicLong(0);
 
         RealCounters() {
             reset();
         }
 
         @Override
         public void reset() {
-            objRef = 0L;
-            nullObjRef = 0L;
-            nullReferent = 0L;
-            forwardedReferent = 0L;
-            nonHeapReferent = 0L;
-            copiedReferent = 0L;
-            unmodifiedReference = 0L;
+            objRef.set(0L);
+            nullObjRef.set(0L);
+            nullReferent.set(0L);
+            forwardedReferent.set(0L);
+            nonHeapReferent.set(0L);
+            copiedReferent.set(0L);
+            unmodifiedReference.set(0L);
         }
 
         @Override
@@ -196,50 +199,49 @@ public void close() {
         @Override
         @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         public void noteObjRef() {
-            objRef += 1L;
+            objRef.incrementAndGet();
         }
 
         @Override
         @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         public void noteNullReferent() {
-            nullReferent += 1L;
+            nullReferent.incrementAndGet();
         }
 
         @Override
         @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         public void noteForwardedReferent() {
-            forwardedReferent += 1L;
+            forwardedReferent.incrementAndGet();
         }
 
         @Override
         @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         public void noteNonHeapReferent() {
-            nonHeapReferent += 1L;
+            nonHeapReferent.incrementAndGet();
         }
 
         @Override
         @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         public void noteCopiedReferent() {
-            copiedReferent += 1L;
+            copiedReferent.incrementAndGet();
         }
 
         @Override
         @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
         public void noteUnmodifiedReference() {
-            unmodifiedReference += 1L;
+            unmodifiedReference.incrementAndGet();
         }
 
-        @Override
-        public void toLog() {
+        private void toLog() {
             Log log = Log.log();
             log.string("[GreyToBlackObjRefVisitor.counters:");
-            log.string("  objRef: ").signed(objRef);
-            log.string("  nullObjRef: ").signed(nullObjRef);
-            log.string("  nullReferent: ").signed(nullReferent);
-            log.string("  forwardedReferent: ").signed(forwardedReferent);
-            log.string("  nonHeapReferent: ").signed(nonHeapReferent);
-            log.string("  copiedReferent: ").signed(copiedReferent);
-            log.string("  unmodifiedReference: ").signed(unmodifiedReference);
+            log.string("  objRef: ").signed(objRef.get());
+            log.string("  nullObjRef: ").signed(nullObjRef.get());
+            log.string("  nullReferent: ").signed(nullReferent.get());
+            log.string("  forwardedReferent: ").signed(forwardedReferent.get());
+            log.string("  nonHeapReferent: ").signed(nonHeapReferent.get());
+            log.string("  copiedReferent: ").signed(copiedReferent.get());
+            log.string("  unmodifiedReference: ").signed(unmodifiedReference.get());
             log.string("]").newline();
         }
     }
@@ -288,10 +290,6 @@ public void noteCopiedReferent() {
         public void noteUnmodifiedReference() {
         }
 
-        @Override
-        public void toLog() {
-        }
-
         @Override
         public void reset() {
         }