Lower Thread Post contention

Author: benaadams

src/Microsoft.AspNet.Server.Kestrel/Http/SocketOutput.cs

@@ -16,7 +16,7 @@ public class SocketOutput : ISocketOutput
     {
         public const int MaxPooledWriteReqs = 1024;
 
-        private const int _maxPendingWrites = 3;
+        //private const int _maxPendingWrites = 3;
         private const int _maxBytesPreCompleted = 65536;
         private const int _initialTaskQueues = 64;
         private const int _maxPooledWriteContexts = 32;
@@ -41,7 +41,7 @@ public class SocketOutput : ISocketOutput
 
         // The number of write operations that have been scheduled so far
         // but have not completed.
-        private int _writesPending = 0;
+        //private int _writesPending = 0;
         private int _numBytesPreCompleted = 0;
         private Exception _lastWriteError;
         private WriteContext _nextWriteContext;
@@ -50,6 +50,8 @@ public class SocketOutput : ISocketOutput
         private readonly Queue<WriteContext> _writeContextPool;
         private readonly Queue<UvWriteReq> _writeReqPool;
 
+        private int _writePending = 0;
+
         private readonly object _contextLock = new object();
         private readonly object _taskQueueLock = new object();
 
@@ -160,16 +162,16 @@ public unsafe Task WriteAsync(
             if (number > 10)
             {
                 number = 0;
-                ScheduleWrite();
-                Interlocked.Increment(ref _writesPending);
+                scheduleWrite = true;
             }
-            else if (Interlocked.Increment(ref _writesPending) < _maxPendingWrites && immediate)
+            else if (immediate)
             {
-                ScheduleWrite();
+                scheduleWrite = true;
             }
-            else
+
+            if (scheduleWrite)
             {
-                Interlocked.Decrement(ref _writesPending);
+                ScheduleWrite();
             }
 
             // Return TaskCompletionSource's Task if set, otherwise completed Task 
@@ -259,17 +261,20 @@ private static void ReturnBlocks(MemoryPoolBlock2 block)
 
         private void ScheduleWrite()
         {
-            _thread.Post(_this => _this.WriteAllPending(), this);
+            if (Interlocked.CompareExchange(ref _writePending, 1, 0) == 0)
+            {
+                _thread.Post(_this => _this.WriteAllPending(), this);
+            }
         }
 
         // This is called on the libuv event loop
         private void WriteAllPending()
         {
+            Volatile.Write(ref _writePending, 0);
             var writingContext = Interlocked.Exchange(ref _nextWriteContext, null);
 
             if (writingContext == null)
             {
-                Interlocked.Decrement(ref _writesPending);
                 return;
             }
 
@@ -279,8 +284,6 @@ private void WriteAllPending()
             }
             catch
             {
-                Interlocked.Decrement(ref _writesPending);
-
                 throw;
             }
         }
@@ -308,10 +311,6 @@ private void OnWriteCompleted(WriteContext writeContext)
             {
                 scheduleWrite = true;
             }
-            else
-            {
-                Interlocked.Decrement(ref _writesPending);
-            }
 
             // _numBytesPreCompleted can temporarily go negative in the event there are
             // completed writes that we haven't triggered callbacks for yet.
@@ -351,7 +350,7 @@ private void OnWriteCompleted(WriteContext writeContext)
 
             _log.ConnectionWriteCallback(_connectionId, status);
 
-            if (scheduleWrite)
+            if (scheduleWrite || _nextWriteContext != null)
             {
                 ScheduleWrite();
             }
@@ -590,6 +589,7 @@ private static void ReturnWrittenBlocks(MemoryPoolBlock2 block)
 
             private void LockWrite()
             {
+
                 var head = Self._head;
                 var tail = Self._tail;