diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs index b1b45eb6e..389310461 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs @@ -1020,64 +1020,83 @@ public void CopyFrom(byte[] data, int offset, int count) public unsafe void CopyFromAscii(string data) { var block = _block; - if (block == null) + if (block != null) { - return; - } + Debug.Assert(block.Next == null); + Debug.Assert(block.End == _index); - Debug.Assert(block.Next == null); - Debug.Assert(block.End == _index); - - var pool = block.Pool; - var blockIndex = _index; - var length = data.Length; + fixed (char* pData = data) + { + var input = pData; + var blockIndex = _index; + var length = data.Length; - var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex; - var bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3; + var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex; + var count = Math.Min(length, bytesLeftInBlock); + var output = (block.DataFixedPtr + block.End); - fixed (char* pData = data) - { - var input = pData; - var inputEnd = pData + length; - var inputEndMinusSpan = inputEnd - 3; + blockIndex += count; - while (input < inputEnd) - { - if (bytesLeftInBlock == 0) + // Only one path should inline, other branch should be eliminated + if (IntPtr.Size == 8) { - var nextBlock = pool.Lease(); - block.End = blockIndex; - Volatile.Write(ref block.Next, nextBlock); - block = nextBlock; - - blockIndex = block.Data.Offset; - bytesLeftInBlock = block.Data.Count; - bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3; + CopyFromAscii64Bit(input, output, count); + } + else + { + CopyFromAscii32Bit(input, output, count); } - var output = (block.DataFixedPtr + block.End); - var copied = 0; - for (; input < inputEndMinusSpan && copied < bytesLeftInBlockMinusSpan; copied += 4) + block.End = blockIndex; + if (length <= bytesLeftInBlock) { - *(output) = (byte)*(input); - *(output + 1) = (byte)*(input + 1); - *(output + 2) = (byte)*(input + 2); - *(output + 3) = (byte)*(input + 3); - output += 4; - input += 4; + _index = blockIndex; } - for (; input < inputEnd && copied < bytesLeftInBlock; copied++) + else { - *(output++) = (byte)*(input++); + CopyFromAsciiMultiblock(input + count, length - bytesLeftInBlock); } - - blockIndex += copied; - bytesLeftInBlockMinusSpan -= copied; - bytesLeftInBlock -= copied; } } + } + + private unsafe void CopyFromAsciiMultiblock(char* inputStart, int remainingLength) + { + var input = inputStart; + var length = remainingLength; + var block = _block; + var pool = block.Pool; + int blockIndex; + do + { + var nextBlock = pool.Lease(); + Volatile.Write(ref block.Next, nextBlock); + block = nextBlock; + + var bytesLeftInBlock = block.Data.Count; + + var output = (block.DataFixedPtr + block.End); + var toCopy = Math.Min(length, block.Data.Count); + var toCopyULong = toCopy & ~0x3; + + // Only one path should inline, other branch should be eliminated + blockIndex = block.Data.Offset + toCopy; + + if (IntPtr.Size == 8) + { + CopyFromAscii64Bit(input, output, toCopy); + } + else + { + CopyFromAscii32Bit(input, output, toCopy); + } + + block.End = blockIndex; + + length -= toCopy; + input += toCopy; + } while (length > 0); - block.End = blockIndex; _block = block; _index = blockIndex; } @@ -1254,5 +1273,92 @@ public static Vector GetVector(byte vectorByte) return Vector.AsVectorByte(new Vector(vectorByte * 0x01010101u)); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe static void CopyFromAscii64Bit(char* input, byte* output, int count) + { + const int Shift16Shift24 = (1 << 16) | (1 << 24); + const int Shift8Identity = (1 << 8) + 1; + + // Encode as bytes upto the first non-ASCII byte and return count encoded + var i = 0; + if (count >= 4) + { + var unaligned = (unchecked(-(int)input) >> 1) & 0x3; + // Unaligned chars + for (; i < unaligned; i++) + { + var ch = *(input + i); + *(output + i) = (byte)ch; // Cast convert + } + + // Aligned + var ulongDoubleCount = (count - i) & ~0x7; + for (; i < ulongDoubleCount; i += 8) + { + var inputUlong0 = *(ulong*)(input + i); + var inputUlong1 = *(ulong*)(input + i + 4); + // Pack 16 ASCII chars into 16 bytes + *(uint*)(output + i) = + ((uint)((inputUlong0 * Shift16Shift24) >> 24) & 0xffff) | + ((uint)((inputUlong0 * Shift8Identity) >> 24) & 0xffff0000); + *(uint*)(output + i + 4) = + ((uint)((inputUlong1 * Shift16Shift24) >> 24) & 0xffff) | + ((uint)((inputUlong1 * Shift8Identity) >> 24) & 0xffff0000); + } + if (count - 4 > i) + { + var inputUlong = *(ulong*)(input + i); + // Pack 8 ASCII chars into 8 bytes + *(uint*)(output + i) = + ((uint)((inputUlong * Shift16Shift24) >> 24) & 0xffff) | + ((uint)((inputUlong * Shift8Identity) >> 24) & 0xffff0000); + i += 4; + } + } + + for (; i < count; i++) + { + var ch = *(input + i); + *(output + i) = (byte)ch; // Cast convert + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe static void CopyFromAscii32Bit(char* input, byte* output, int count) + { + // Encode as bytes upto the first non-ASCII byte and return count encoded + var i = 0; + // Unaligned chars + if ((unchecked((int)input) & 0x2) != 0) + { + var ch = *input; + i = 1; + *(output) = (byte)ch; // Cast convert + } + + // Aligned + var uintCount = (count - i) & ~0x3; + for (; i < uintCount; i += 4) + { + var inputUint0 = *(uint*)(input + i); + var inputUint1 = *(uint*)(input + i + 2); + // Pack 4 ASCII chars into 4 bytes + *(ushort*)(output + i) = (ushort)(inputUint0 | (inputUint0 >> 8)); + *(ushort*)(output + i + 2) = (ushort)(inputUint1 | (inputUint1 >> 8)); + } + if (count - 1 > i) + { + var inputUint = *(uint*)(input + i); + // Pack 2 ASCII chars into 2 bytes + *(ushort*)(output + i) = (ushort)(inputUint | (inputUint >> 8)); + i += 2; + } + + if (i < count) + { + var ch = *(input + i); + *(output + i) = (byte)ch; // Cast convert + } + } } } diff --git a/test/Microsoft.AspNetCore.Server.Kestrel.Performance/Program.cs b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/Program.cs index 3e2d92774..4a7d58eeb 100644 --- a/test/Microsoft.AspNetCore.Server.Kestrel.Performance/Program.cs +++ b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/Program.cs @@ -36,6 +36,10 @@ private static void RunSelectedBenchmarks(BenchmarkType type) { BenchmarkRunner.Run(); } + if (type.HasFlag(BenchmarkType.ResponseHeaders)) + { + BenchmarkRunner.Run(); + } } } @@ -44,6 +48,7 @@ public enum BenchmarkType : uint { RequestParsing = 1, Writing = 2, + ResponseHeaders = 4, // add new ones in powers of two - e.g. 2,4,8,16... All = uint.MaxValue diff --git a/test/Microsoft.AspNetCore.Server.Kestrel.Performance/ResponseHeaders.cs b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/ResponseHeaders.cs new file mode 100644 index 000000000..a8dcf7946 --- /dev/null +++ b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/ResponseHeaders.cs @@ -0,0 +1,209 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using BenchmarkDotNet.Attributes; +using Microsoft.AspNetCore.Server.Kestrel.Internal.Http; +using Microsoft.AspNetCore.Testing; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Http.Internal; +using System.Runtime.CompilerServices; +using System.Text; +using Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure; + +namespace Microsoft.AspNetCore.Server.Kestrel.Performance +{ + [Config(typeof(CoreConfig))] + public class ResponseHeaders + { + private const int InnerLoopCount = 512; + + private static readonly byte[] _bytesServer = Encoding.ASCII.GetBytes("\r\nServer: Kestrel"); + private static readonly DateHeaderValueManager _dateHeaderValueManager = new DateHeaderValueManager(); + private static readonly MemoryPool _memoryPool = new MemoryPool(); + private FrameResponseHeaders _responseHeadersDirect; + private HttpResponse _response; + + [Params("ContentLengthNumeric", "ContentLengthString", "Plaintext", "Common", "Unknown")] + public string Type { get; set; } + + [Benchmark(OperationsPerInvoke = InnerLoopCount)] + public void SetHeaders() + { + switch (Type) + { + case "ContentLengthNumeric": + ContentLengthNumeric(InnerLoopCount); + break; + case "ContentLengthString": + ContentLengthString(InnerLoopCount); + break; + case "Plaintext": + Plaintext(InnerLoopCount); + break; + case "Common": + Common(InnerLoopCount); + break; + case "Unknown": + Unknown(InnerLoopCount); + break; + } + } + + [Benchmark(OperationsPerInvoke = InnerLoopCount)] + public void OutputHeaders() + { + for (var i = 0; i < InnerLoopCount; i++) + { + var block = _memoryPool.Lease(); + var iter = new MemoryPoolIterator(block); + _responseHeadersDirect.CopyTo(ref iter); + + ReturnBlocks(block); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void ContentLengthNumeric(int count) + { + for (var i = 0; i < count; i++) + { + _responseHeadersDirect.Reset(); + + _response.ContentLength = 0; + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void ContentLengthString(int count) + { + for (var i = 0; i < count; i++) + { + _responseHeadersDirect.Reset(); + + _response.Headers["Content-Length"] = "0"; + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void Plaintext(int count) + { + for (var i = 0; i < count; i++) + { + _responseHeadersDirect.Reset(); + + _response.StatusCode = 200; + _response.ContentType = "text/plain"; + _response.ContentLength = 13; + + var dateHeaderValues = _dateHeaderValueManager.GetDateHeaderValues(); + _responseHeadersDirect.SetRawDate(dateHeaderValues.String, dateHeaderValues.Bytes); + _responseHeadersDirect.SetRawServer("Kestrel", _bytesServer); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void Common(int count) + { + for (var i = 0; i < count; i++) + { + _responseHeadersDirect.Reset(); + + _response.StatusCode = 200; + _response.ContentType = "text/css"; + _response.ContentLength = 421; + + var headers = _response.Headers; + + headers["Connection"] = "Close"; + headers["Cache-Control"] = "public, max-age=30672000"; + headers["Vary"] = "Accept-Encoding"; + headers["Content-Encoding"] = "gzip"; + headers["Expires"] = "Fri, 12 Jan 2018 22:01:55 GMT"; + headers["Last-Modified"] = "Wed, 22 Jun 2016 20:08:29 GMT"; + headers["Set-Cookie"] = "prov=20629ccd-8b0f-e8ef-2935-cd26609fc0bc; __qca=P0-1591065732-1479167353442; _ga=GA1.2.1298898376.1479167354; _gat=1; sgt=id=9519gfde_3347_4762_8762_df51458c8ec2; acct=t=why-is-%e0%a5%a7%e0%a5%a8%e0%a5%a9-numeric&s=why-is-%e0%a5%a7%e0%a5%a8%e0%a5%a9-numeric"; + headers["ETag"] = "\"54ef7954-1078\""; + headers["Transfer-Encoding"] = "chunked"; + headers["Content-Language"] = "en-gb"; + headers["Upgrade"] = "websocket"; + headers["Via"] = "1.1 varnish"; + headers["Access-Control-Allow-Origin"] = "*"; + headers["Access-Control-Allow-credentials"] = "true"; + headers["Access-Control-Expose-Headers"] = "Client-Protocol, Content-Length, Content-Type, X-Bandwidth-Est, X-Bandwidth-Est2, X-Bandwidth-Est-Comp, X-Bandwidth-Avg, X-Walltime-Ms, X-Sequence-Num"; + + var dateHeaderValues = _dateHeaderValueManager.GetDateHeaderValues(); + _responseHeadersDirect.SetRawDate(dateHeaderValues.String, dateHeaderValues.Bytes); + _responseHeadersDirect.SetRawServer("Kestrel", _bytesServer); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void Unknown(int count) + { + for (var i = 0; i < count; i++) + { + _responseHeadersDirect.Reset(); + + _response.StatusCode = 200; + _response.ContentType = "text/plain"; + _response.ContentLength = 13; + + var headers = _response.Headers; + + headers["Link"] = "; rel=\"canonical\""; + headers["X-Ua-Compatible"] = "IE=Edge"; + headers["X-Powered-By"] = "ASP.NET"; + headers["X-Content-Type-Options"] = "nosniff"; + headers["X-Xss-Protection"] = "1; mode=block"; + headers["X-Frame-Options"] = "SAMEORIGIN"; + headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains; preload"; + headers["Content-Security-Policy"] = "default-src 'none'; script-src 'self' cdnjs.cloudflare.com code.jquery.com scotthelme.disqus.com a.disquscdn.com www.google-analytics.com go.disqus.com platform.twitter.com cdn.syndication.twimg.com; style-src 'self' a.disquscdn.com fonts.googleapis.com cdnjs.cloudflare.com platform.twitter.com; img-src 'self' data: www.gravatar.com www.google-analytics.com links.services.disqus.com referrer.disqus.com a.disquscdn.com cdn.syndication.twimg.com syndication.twitter.com pbs.twimg.com platform.twitter.com abs.twimg.com; child-src fusiontables.googleusercontent.com fusiontables.google.com www.google.com disqus.com www.youtube.com syndication.twitter.com platform.twitter.com; frame-src fusiontables.googleusercontent.com fusiontables.google.com www.google.com disqus.com www.youtube.com syndication.twitter.com platform.twitter.com; connect-src 'self' links.services.disqus.com; font-src 'self' cdnjs.cloudflare.com fonts.gstatic.com fonts.googleapis.com; form-action 'self'; upgrade-insecure-requests;"; + + var dateHeaderValues = _dateHeaderValueManager.GetDateHeaderValues(); + _responseHeadersDirect.SetRawDate(dateHeaderValues.String, dateHeaderValues.Bytes); + _responseHeadersDirect.SetRawServer("Kestrel", _bytesServer); + } + } + + [Setup] + public void Setup() + { + var connectionContext = new MockConnection(new KestrelServerOptions()); + var frame = new Frame(application: null, context: connectionContext); + frame.Reset(); + frame.InitializeHeaders(); + _responseHeadersDirect = (FrameResponseHeaders)frame.ResponseHeaders; + var context = new DefaultHttpContext(frame); + _response = new DefaultHttpResponse(context); + + switch (Type) + { + case "ContentLengthNumeric": + ContentLengthNumeric(1); + break; + case "ContentLengthString": + ContentLengthString(1); + break; + case "Plaintext": + Plaintext(1); + break; + case "Common": + Common(1); + break; + case "Unknown": + Unknown(1); + break; + } + } + + private static void ReturnBlocks(MemoryPoolBlock block) + { + while (block != null) + { + var returningBlock = block; + block = returningBlock.Next; + + returningBlock.Pool.Return(returningBlock); + } + } + } +} diff --git a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs index 1db21257a..3782df187 100644 --- a/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs +++ b/test/Microsoft.AspNetCore.Server.KestrelTests/MemoryPoolIteratorTests.cs @@ -1214,6 +1214,56 @@ public void TestGetAsciiStringEscaped(string input, string expected, int maxChar } } + [Theory] + [InlineData(0, MemoryPool.MaxPooledBlockLength * 3 + 64)] + public unsafe void TestCopyFromAscii(int start, int end) + { + for (var i = start; i <= end; i++) + { + TestCopyFromAscii(i); + } + } + + private unsafe void TestCopyFromAscii(int size) + { + // Arrange + var block = _pool.Lease(); + try + { + var data = new string('\0', size); + + fixed (char* pData = data) + { + for (var i = 0; i < data.Length; i++) + { + // ascii chars 32 - 126 + pData[i] = (char)((i % (126 - 32)) + 32); + } + } + + // Act + var iter = block.GetIterator(); + + var iterEnd = iter; + + iterEnd.CopyFromAscii(data); + + // Assert + Assert.True(iterEnd.IsEnd); + foreach (var ch in data) + { + Assert.Equal(ch, iter.Take()); + } + Assert.True(iter.IsEnd); + Assert.Equal(iter.Block, iterEnd.Block); + Assert.Equal(iter.Index, iterEnd.Index); + } + finally + { + ReturnBlocks(block); + } + } + private delegate bool GetKnownString(MemoryPoolIterator iter, out string result); private void TestKnownStringsInterning(string input, string expected, GetKnownString action) @@ -1244,6 +1294,17 @@ private void TestKnownStringsInterning(string input, string expected, GetKnownSt Assert.Same(knownString1, knownString2); } + private static void ReturnBlocks(MemoryPoolBlock block) + { + while (block != null) + { + var returningBlock = block; + block = returningBlock.Next; + + returningBlock.Pool.Return(returningBlock); + } + } + public static IEnumerable SeekByteLimitData { get