From 7b2c03e77447ed73112606b5e3c1e6f5ec9054f7 Mon Sep 17 00:00:00 2001 From: Ben Adams Date: Wed, 11 Nov 2015 12:24:39 +0000 Subject: [PATCH] Partitioned memory pool --- .../Http/ListenerContext.cs | 5 +- .../Infrastructure/KestrelThread.cs | 3 + .../Infrastructure/MemoryPool2.cs | 83 +++++++++++++++---- .../Infrastructure/MemoryPoolBlock2.cs | 21 ++++- .../KestrelServer.cs | 19 +++++ .../AsciiDecoder.cs | 14 ++-- .../UrlPathDecoder.cs | 2 +- 7 files changed, 119 insertions(+), 28 deletions(-) diff --git a/src/Microsoft.AspNet.Server.Kestrel/Http/ListenerContext.cs b/src/Microsoft.AspNet.Server.Kestrel/Http/ListenerContext.cs index 581720041..0ae440415 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Http/ListenerContext.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Http/ListenerContext.cs @@ -10,13 +10,11 @@ public class ListenerContext : ServiceContext { public ListenerContext() { - Memory2 = new MemoryPool2(); } public ListenerContext(ServiceContext serviceContext) : base(serviceContext) { - Memory2 = new MemoryPool2(); } public ListenerContext(ListenerContext listenerContext) @@ -25,7 +23,6 @@ public ListenerContext(ListenerContext listenerContext) ServerAddress = listenerContext.ServerAddress; Thread = listenerContext.Thread; Application = listenerContext.Application; - Memory2 = listenerContext.Memory2; Log = listenerContext.Log; } @@ -35,6 +32,6 @@ public ListenerContext(ListenerContext listenerContext) public RequestDelegate Application { get; set; } - public MemoryPool2 Memory2 { get; set; } + public MemoryPool2 Memory2 => Thread.Memory2; } } diff --git a/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/KestrelThread.cs b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/KestrelThread.cs index a8608de70..5381a49d6 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/KestrelThread.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/KestrelThread.cs @@ -42,6 +42,7 @@ public KestrelThread(KestrelEngine engine) _loop = new UvLoopHandle(_log); _post = new UvAsyncHandle(_log); _thread = new Thread(ThreadStart); + Memory2 = new MemoryPool2(); QueueCloseHandle = PostCloseHandle; } @@ -50,6 +51,8 @@ public KestrelThread(KestrelEngine engine) public Action, IntPtr> QueueCloseHandle { get; internal set; } + public MemoryPool2 Memory2 { get; } + public Task StartAsync() { var tcs = new TaskCompletionSource(); diff --git a/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPool2.cs b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPool2.cs index 37d64ddca..86e5df483 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPool2.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPool2.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Concurrent; +using System.Threading; namespace Microsoft.AspNet.Server.Kestrel.Infrastructure { @@ -36,11 +37,21 @@ public class MemoryPool2 : IDisposable /// private const int _slabLength = _blockStride * _blockCount; + /// + /// Max allocation block size for pooled blocks, + /// larger values can be leased but they will be disposed after use rather than returned to the pool. + /// + public const int MaxPooledBlockLength = _blockLength; + + // Processor count is a sys call https://github.com/dotnet/coreclr/blob/0e0ff9d17ab586f3cc7224dd33d8781cd77f3ca8/src/mscorlib/src/System/Environment.cs#L548 + // Nor does it look like its constant https://github.com/dotnet/corefx/blob/master/src/System.Threading.Tasks.Parallel/src/System/Threading/PlatformHelper.cs#L23 + public static int _partitionCount = Environment.ProcessorCount; + /// /// Thread-safe collection of blocks which are currently in the pool. A slab will pre-allocate all of the block tracking objects /// and add them to this collection. When memory is requested it is taken from here first, and when it is returned it is re-added. /// - private readonly ConcurrentQueue _blocks = new ConcurrentQueue(); + private readonly ConcurrentQueue[] _blocks; /// /// Thread-safe collection of slabs which have been allocated by this pool. As long as a slab is in this collection and slab.IsActive, @@ -53,13 +64,31 @@ public class MemoryPool2 : IDisposable /// private bool _disposedValue = false; // To detect redundant calls + public MemoryPool2() + { + _blocks = new ConcurrentQueue[_partitionCount]; + for (var i = 0; i < _blocks.Length; i++) + { + _blocks[i] = new ConcurrentQueue(); + } + } + + public void PopulatePools() + { + for (var i = 0; i < _blocks.Length; i++) + { + // Allocate the inital set + Return(AllocateSlab(i)); + } + } + /// /// Called to take a block from the pool. /// /// The block returned must be at least this size. It may be larger than this minimum size, and if so, /// the caller may write to the block's entire size rather than being limited to the minumumSize requested. /// The block that is reserved for the called. It must be passed to Return when it is no longer being used. - public MemoryPoolBlock2 Lease(int minimumSize) + public MemoryPoolBlock2 Lease(int minimumSize = MaxPooledBlockLength) { if (minimumSize > _blockLength) { @@ -67,28 +96,38 @@ public MemoryPoolBlock2 Lease(int minimumSize) // Because this is the degenerate case, a one-time-use byte[] array and tracking object are allocated. // When this block tracking object is returned it is not added to the pool - instead it will be // allowed to be garbage collected normally. - return MemoryPoolBlock2.Create( - new ArraySegment(new byte[minimumSize]), - dataPtr: IntPtr.Zero, - pool: null, - slab: null); + return MemoryPoolBlock2.Create(new ArraySegment(new byte[minimumSize])); } MemoryPoolBlock2 block; - if (_blocks.TryDequeue(out block)) + + var preferedPartition = Thread.CurrentThread.ManagedThreadId % _partitionCount; + + if (_blocks[preferedPartition].TryDequeue(out block)) { // block successfully taken from the stack - return it return block; } + + // no block, steal block from another parition + for (var i = 1; i < _partitionCount; i++) + { + if (_blocks[(preferedPartition + i) % _partitionCount].TryDequeue(out block)) + { + // block successfully taken from the stack - return it + return block; + } + } + // no blocks available - grow the pool - return AllocateSlab(); + return AllocateSlab(preferedPartition); } /// /// Internal method called when a block is requested and the pool is empty. It allocates one additional slab, creates all of the /// block tracking objects, and adds them all to the pool. /// - private MemoryPoolBlock2 AllocateSlab() + private MemoryPoolBlock2 AllocateSlab(int partition) { var slab = MemoryPoolSlab2.Create(_slabLength); _slabs.Push(slab); @@ -107,7 +146,8 @@ private MemoryPoolBlock2 AllocateSlab() new ArraySegment(slab.Array, offset, _blockLength), basePtr, this, - slab); + slab, + partition); Return(block); } @@ -116,7 +156,8 @@ private MemoryPoolBlock2 AllocateSlab() new ArraySegment(slab.Array, offset, _blockLength), basePtr, this, - slab); + slab, + partition); return newBlock; } @@ -131,8 +172,22 @@ private MemoryPoolBlock2 AllocateSlab() /// The block to return. It must have been acquired by calling Lease on the same memory pool instance. public void Return(MemoryPoolBlock2 block) { - block.Reset(); - _blocks.Enqueue(block); + var owningPool = block.Pool; + if (owningPool == null) + { + // not pooled block, throw away + return; + } + if (owningPool != this) + { + throw new InvalidOperationException("Returning " + nameof(MemoryPoolBlock2) + " to incorrect pool."); + } + if (owningPool == this) + { + block.Reset(); + // return to owning parition + _blocks[block.Partition].Enqueue(block); + } } protected virtual void Dispose(bool disposing) diff --git a/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolBlock2.cs b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolBlock2.cs index a93e03186..0a61f943c 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolBlock2.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolBlock2.cs @@ -66,6 +66,11 @@ protected MemoryPoolBlock2() /// public int End { get; set; } + /// + /// The Partition represents the memory pool partition the block belongs to. + /// + public int Partition { get; private set; } + /// /// Reference to the next block of data when the overall "active" bytes spans multiple blocks. At the point when the block is /// leased Next is guaranteed to be null. Start, End, and Next are used together in order to create a linked-list of discontiguous @@ -74,6 +79,7 @@ protected MemoryPoolBlock2() /// public MemoryPoolBlock2 Next { get; set; } +#if DEBUG ~MemoryPoolBlock2() { Debug.Assert(!_pinHandle.IsAllocated, "Ad-hoc memory block wasn't unpinned"); @@ -89,13 +95,17 @@ protected MemoryPoolBlock2() { Pool.Return(new MemoryPoolBlock2 { - _dataArrayPtr = _dataArrayPtr, Data = Data, + _dataArrayPtr = _dataArrayPtr, Pool = Pool, Slab = Slab, + Start = Start, + End = End, + Partition = Partition }); } } +#endif /// /// Called to ensure that a block is pinned, and return the pointer to native memory just after @@ -129,11 +139,17 @@ public void Unpin() } } + public static MemoryPoolBlock2 Create(ArraySegment data) + { + return Create(data, IntPtr.Zero, null, null, -1); + } + public static MemoryPoolBlock2 Create( ArraySegment data, IntPtr dataPtr, MemoryPool2 pool, - MemoryPoolSlab2 slab) + MemoryPoolSlab2 slab, + int partition) { return new MemoryPoolBlock2 { @@ -143,6 +159,7 @@ public static MemoryPoolBlock2 Create( Slab = slab, Start = data.Offset, End = data.Offset, + Partition = partition }; } diff --git a/src/Microsoft.AspNet.Server.Kestrel/KestrelServer.cs b/src/Microsoft.AspNet.Server.Kestrel/KestrelServer.cs index 346808ebc..ff70f0aac 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/KestrelServer.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/KestrelServer.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Runtime; using Microsoft.AspNet.Hosting; using Microsoft.AspNet.Hosting.Server; using Microsoft.AspNet.Http; @@ -106,6 +107,24 @@ public void Start(RequestDelegate requestDelegate) } engine.Start(threadCount); + + for (var i = 0; i < engine.Threads.Count; i++) + { + engine.Threads[i].Memory2.PopulatePools(); + } + + // Move all MemoryPoolBlock2 into Gen2 + GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce; + GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, true); + GC.WaitForPendingFinalizers(); + + GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce; + GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, true); + GC.WaitForPendingFinalizers(); + + GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce; + GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced, true); + var atLeastOneListener = false; foreach (var address in information.Addresses) diff --git a/test/Microsoft.AspNet.Server.KestrelTests/AsciiDecoder.cs b/test/Microsoft.AspNet.Server.KestrelTests/AsciiDecoder.cs index 26c034d2d..c002c6208 100644 --- a/test/Microsoft.AspNet.Server.KestrelTests/AsciiDecoder.cs +++ b/test/Microsoft.AspNet.Server.KestrelTests/AsciiDecoder.cs @@ -15,7 +15,7 @@ private void FullByteRangeSupported() { var byteRange = Enumerable.Range(0, 255).Select(x => (byte)x).ToArray(); - var mem = MemoryPoolBlock2.Create(new ArraySegment(byteRange), IntPtr.Zero, null, null); + var mem = MemoryPoolBlock2.Create(new ArraySegment(byteRange)); mem.End = byteRange.Length; var begin = mem.GetIterator(); @@ -44,10 +44,10 @@ private void MultiBlockProducesCorrectResults() .Concat(byteRange) .ToArray(); - var mem0 = MemoryPoolBlock2.Create(new ArraySegment(byteRange), IntPtr.Zero, null, null); - var mem1 = MemoryPoolBlock2.Create(new ArraySegment(byteRange), IntPtr.Zero, null, null); - var mem2 = MemoryPoolBlock2.Create(new ArraySegment(byteRange), IntPtr.Zero, null, null); - var mem3 = MemoryPoolBlock2.Create(new ArraySegment(byteRange), IntPtr.Zero, null, null); + var mem0 = MemoryPoolBlock2.Create(new ArraySegment(byteRange)); + var mem1 = MemoryPoolBlock2.Create(new ArraySegment(byteRange)); + var mem2 = MemoryPoolBlock2.Create(new ArraySegment(byteRange)); + var mem3 = MemoryPoolBlock2.Create(new ArraySegment(byteRange)); mem0.End = byteRange.Length; mem1.End = byteRange.Length; mem2.End = byteRange.Length; @@ -79,8 +79,8 @@ private void HeapAllocationProducesCorrectResults() var byteRange = Enumerable.Range(0, 16384 + 64).Select(x => (byte)x).ToArray(); var expectedByteRange = byteRange.Concat(byteRange).ToArray(); - var mem0 = MemoryPoolBlock2.Create(new ArraySegment(byteRange), IntPtr.Zero, null, null); - var mem1 = MemoryPoolBlock2.Create(new ArraySegment(byteRange), IntPtr.Zero, null, null); + var mem0 = MemoryPoolBlock2.Create(new ArraySegment(byteRange)); + var mem1 = MemoryPoolBlock2.Create(new ArraySegment(byteRange)); mem0.End = byteRange.Length; mem1.End = byteRange.Length; diff --git a/test/Microsoft.AspNet.Server.KestrelTests/UrlPathDecoder.cs b/test/Microsoft.AspNet.Server.KestrelTests/UrlPathDecoder.cs index 02769fd01..7e1164034 100644 --- a/test/Microsoft.AspNet.Server.KestrelTests/UrlPathDecoder.cs +++ b/test/Microsoft.AspNet.Server.KestrelTests/UrlPathDecoder.cs @@ -124,7 +124,7 @@ public void DecodeWithBoundary(string raw, int rawLength, string expect, int exp private MemoryPoolIterator2 BuildSample(string data) { var store = data.Select(c => (byte)c).ToArray(); - var mem = MemoryPoolBlock2.Create(new ArraySegment(store), IntPtr.Zero, null, null); + var mem = MemoryPoolBlock2.Create(new ArraySegment(store)); mem.End = store.Length; return mem.GetIterator();