diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/FrameResponseHeaders.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/FrameResponseHeaders.cs index 2a3f1979c..439a76bd6 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/FrameResponseHeaders.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/FrameResponseHeaders.cs @@ -49,7 +49,7 @@ public void CopyTo(ref WritableBuffer output) output.Write(_CrLf); output.WriteAscii(kv.Key); output.Write(_colonSpace); - output.Write(value); + output.WriteAscii(value); } } } diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/PipelineExtensions.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/PipelineExtensions.cs index a53b5d445..0ab6f20a6 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/PipelineExtensions.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/PipelineExtensions.cs @@ -13,6 +13,11 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http { public static class PipelineExtensions { + private const int _maxULongByteLength = 20; + + [ThreadStatic] + private static byte[] _numericBytesScratch; + public static ValueTask> PeekAsync(this IPipeReader pipelineReader) { var input = pipelineReader.ReadAsync(); @@ -90,18 +95,237 @@ public static ArraySegment GetArray(this Memory memory) return result; } - public static void WriteAscii(this WritableBuffer buffer, string data) + public unsafe static void WriteAscii(this WritableBuffer buffer, string data) + { + if (!string.IsNullOrEmpty(data)) + { + if (buffer.Memory.IsEmpty) + { + buffer.Ensure(); + } + + // Fast path, try copying to the available memory directly + if (data.Length <= buffer.Memory.Length) + { + fixed (char* input = data) + fixed (byte* output = &buffer.Memory.Span.DangerousGetPinnableReference()) + { + EncodeAsciiCharsToBytes(input, output, data.Length); + } + + buffer.Advance(data.Length); + } + else + { + buffer.WriteAsciiMultiWrite(data); + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public unsafe static void WriteNumeric(this WritableBuffer buffer, ulong number) + { + const byte AsciiDigitStart = (byte)'0'; + + if (buffer.Memory.IsEmpty) + { + buffer.Ensure(); + } + + // Fast path, try copying to the available memory directly + var bytesLeftInBlock = buffer.Memory.Length; + var simpleWrite = true; + fixed (byte* output = &buffer.Memory.Span.DangerousGetPinnableReference()) + { + var start = output; + if (number < 10 && bytesLeftInBlock >= 1) + { + *(start) = (byte)(((uint)number) + AsciiDigitStart); + buffer.Advance(1); + } + else if (number < 100 && bytesLeftInBlock >= 2) + { + var val = (uint)number; + var tens = (byte)((val * 205u) >> 11); // div10, valid to 1028 + + *(start) = (byte)(tens + AsciiDigitStart); + *(start + 1) = (byte)(val - (tens * 10) + AsciiDigitStart); + buffer.Advance(2); + } + else if (number < 1000 && bytesLeftInBlock >= 3) + { + var val = (uint)number; + var digit0 = (byte)((val * 41u) >> 12); // div100, valid to 1098 + var digits01 = (byte)((val * 205u) >> 11); // div10, valid to 1028 + + *(start) = (byte)(digit0 + AsciiDigitStart); + *(start + 1) = (byte)(digits01 - (digit0 * 10) + AsciiDigitStart); + *(start + 2) = (byte)(val - (digits01 * 10) + AsciiDigitStart); + buffer.Advance(3); + } + else + { + simpleWrite = false; + } + } + + if (!simpleWrite) + { + buffer.WriteNumericMultiWrite(number); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static unsafe void WriteNumericMultiWrite(this WritableBuffer buffer, ulong number) + { + const byte AsciiDigitStart = (byte)'0'; + + var value = number; + var position = _maxULongByteLength; + var byteBuffer = NumericBytesScratch; + do + { + // Consider using Math.DivRem() if available + var quotient = value / 10; + byteBuffer[--position] = (byte)(AsciiDigitStart + (value - quotient * 10)); // 0x30 = '0' + value = quotient; + } + while (value != 0); + + var length = _maxULongByteLength - position; + buffer.Write(new ReadOnlySpan(byteBuffer, position, length)); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private unsafe static void WriteAsciiMultiWrite(this WritableBuffer buffer, string data) { - buffer.Write(Encoding.ASCII.GetBytes(data)); + var remaining = data.Length; + + fixed (char* input = data) + { + var inputSlice = input; + + while (remaining > 0) + { + var writable = Math.Min(remaining, buffer.Memory.Length); + + buffer.Ensure(writable); + + if (writable == 0) + { + continue; + } + + fixed (byte* output = &buffer.Memory.Span.DangerousGetPinnableReference()) + { + EncodeAsciiCharsToBytes(inputSlice, output, writable); + } + + inputSlice += writable; + remaining -= writable; + + buffer.Advance(writable); + } + } } - public static void Write(this WritableBuffer buffer, string data) + + private unsafe static void EncodeAsciiCharsToBytes(char* input, byte* output, int length) { - buffer.Write(Encoding.UTF8.GetBytes(data)); + // Note: Not BIGENDIAN or check for non-ascii + const int Shift16Shift24 = (1 << 16) | (1 << 24); + const int Shift8Identity = (1 << 8) | (1); + + // Encode as bytes upto the first non-ASCII byte and return count encoded + int i = 0; + // Use Intrinsic switch + if (IntPtr.Size == 8) // 64 bit + { + if (length < 4) goto trailing; + + int unaligned = (int)(((ulong)input) & 0x7) >> 1; + // Unaligned chars + for (; i < unaligned; i++) + { + char ch = *(input + i); + *(output + i) = (byte)ch; // Cast convert + } + + // Aligned + int ulongDoubleCount = (length - i) & ~0x7; + for (; i < ulongDoubleCount; i += 8) + { + ulong inputUlong0 = *(ulong*)(input + i); + ulong inputUlong1 = *(ulong*)(input + i + 4); + // Pack 16 ASCII chars into 16 bytes + *(uint*)(output + i) = + ((uint)((inputUlong0 * Shift16Shift24) >> 24) & 0xffff) | + ((uint)((inputUlong0 * Shift8Identity) >> 24) & 0xffff0000); + *(uint*)(output + i + 4) = + ((uint)((inputUlong1 * Shift16Shift24) >> 24) & 0xffff) | + ((uint)((inputUlong1 * Shift8Identity) >> 24) & 0xffff0000); + } + if (length - 4 > i) + { + ulong inputUlong = *(ulong*)(input + i); + // Pack 8 ASCII chars into 8 bytes + *(uint*)(output + i) = + ((uint)((inputUlong * Shift16Shift24) >> 24) & 0xffff) | + ((uint)((inputUlong * Shift8Identity) >> 24) & 0xffff0000); + i += 4; + } + + trailing: + for (; i < length; i++) + { + char ch = *(input + i); + *(output + i) = (byte)ch; // Cast convert + } + } + else // 32 bit + { + // Unaligned chars + if ((unchecked((int)input) & 0x2) != 0) + { + char ch = *input; + i = 1; + *(output) = (byte)ch; // Cast convert + } + + // Aligned + int uintCount = (length - i) & ~0x3; + for (; i < uintCount; i += 4) + { + uint inputUint0 = *(uint*)(input + i); + uint inputUint1 = *(uint*)(input + i + 2); + // Pack 4 ASCII chars into 4 bytes + *(ushort*)(output + i) = (ushort)(inputUint0 | (inputUint0 >> 8)); + *(ushort*)(output + i + 2) = (ushort)(inputUint1 | (inputUint1 >> 8)); + } + if (length - 1 > i) + { + uint inputUint = *(uint*)(input + i); + // Pack 2 ASCII chars into 2 bytes + *(ushort*)(output + i) = (ushort)(inputUint | (inputUint >> 8)); + i += 2; + } + + if (i < length) + { + char ch = *(input + i); + *(output + i) = (byte)ch; // Cast convert + i = length; + } + } } - public static void WriteNumeric(this WritableBuffer buffer, ulong number) + private static byte[] NumericBytesScratch => _numericBytesScratch ?? CreateNumericBytesScratch(); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static byte[] CreateNumericBytesScratch() { - buffer.Write(number.ToString()); + var bytes = new byte[_maxULongByteLength]; + _numericBytesScratch = bytes; + return bytes; } } } \ No newline at end of file diff --git a/test/Microsoft.AspNetCore.Server.Kestrel.Performance/ResponseHeadersBenchmark.cs b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/ResponseHeaderCollectionBenchmark.cs similarity index 88% rename from test/Microsoft.AspNetCore.Server.Kestrel.Performance/ResponseHeadersBenchmark.cs rename to test/Microsoft.AspNetCore.Server.Kestrel.Performance/ResponseHeaderCollectionBenchmark.cs index f5dab5e3e..691b0da43 100644 --- a/test/Microsoft.AspNetCore.Server.Kestrel.Performance/ResponseHeadersBenchmark.cs +++ b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/ResponseHeaderCollectionBenchmark.cs @@ -3,16 +3,16 @@ using System.Runtime.CompilerServices; using System.Text; -using BenchmarkDotNet.Attributes; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Http.Internal; using Microsoft.AspNetCore.Server.Kestrel.Internal.Http; using Microsoft.AspNetCore.Testing; +using BenchmarkDotNet.Attributes; namespace Microsoft.AspNetCore.Server.Kestrel.Performance { [Config(typeof(CoreConfig))] - public class ResponseHeadersBenchmark + public class ResponseHeaderCollectionBenchmark { private const int InnerLoopCount = 512; @@ -21,27 +21,42 @@ public class ResponseHeadersBenchmark private FrameResponseHeaders _responseHeadersDirect; private HttpResponse _response; - [Params("ContentLengthNumeric", "ContentLengthString", "Plaintext", "Common", "Unknown")] - public string Type { get; set; } + public enum BenchmarkTypes + { + ContentLengthNumeric, + ContentLengthString, + Plaintext, + Common, + Unknown + } + + [Params( + BenchmarkTypes.ContentLengthNumeric, + BenchmarkTypes.ContentLengthString, + BenchmarkTypes.Plaintext, + BenchmarkTypes.Common, + BenchmarkTypes.Unknown + )] + public BenchmarkTypes Type { get; set; } [Benchmark(OperationsPerInvoke = InnerLoopCount)] public void SetHeaders() { switch (Type) { - case "ContentLengthNumeric": + case BenchmarkTypes.ContentLengthNumeric: ContentLengthNumeric(InnerLoopCount); break; - case "ContentLengthString": + case BenchmarkTypes.ContentLengthString: ContentLengthString(InnerLoopCount); break; - case "Plaintext": + case BenchmarkTypes.Plaintext: Plaintext(InnerLoopCount); break; - case "Common": + case BenchmarkTypes.Common: Common(InnerLoopCount); break; - case "Unknown": + case BenchmarkTypes.Unknown: Unknown(InnerLoopCount); break; } @@ -163,19 +178,19 @@ public void Setup() switch (Type) { - case "ContentLengthNumeric": + case BenchmarkTypes.ContentLengthNumeric: ContentLengthNumeric(1); break; - case "ContentLengthString": + case BenchmarkTypes.ContentLengthString: ContentLengthString(1); break; - case "Plaintext": + case BenchmarkTypes.Plaintext: Plaintext(1); break; - case "Common": + case BenchmarkTypes.Common: Common(1); break; - case "Unknown": + case BenchmarkTypes.Unknown: Unknown(1); break; } diff --git a/test/Microsoft.AspNetCore.Server.Kestrel.Performance/ResponseHeadersWritingBenchmark.cs b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/ResponseHeadersWritingBenchmark.cs new file mode 100644 index 000000000..dd89ce254 --- /dev/null +++ b/test/Microsoft.AspNetCore.Server.Kestrel.Performance/ResponseHeadersWritingBenchmark.cs @@ -0,0 +1,151 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO.Pipelines; +using System.Net; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Server.Kestrel.Internal; +using Microsoft.AspNetCore.Server.Kestrel.Internal.Http; +using Microsoft.AspNetCore.Server.Kestrel.Internal.Infrastructure; +using Microsoft.AspNetCore.Testing; +using BenchmarkDotNet.Attributes; +using Moq; + +namespace Microsoft.AspNetCore.Server.Kestrel.Performance +{ + [Config(typeof(CoreConfig))] + public class ResponseHeadersWritingBenchmark + { + private static readonly byte[] _helloWorldPayload = Encoding.ASCII.GetBytes("Hello, World!"); + + private readonly TestFrame _frame; + + public ResponseHeadersWritingBenchmark() + { + _frame = MakeFrame(); + } + + public enum BenchmarkTypes + { + TechEmpowerPlaintext, + PlaintextChunked, + PlaintextWithCookie, + PlaintextChunkedWithCookie, + LiveAspNet + } + + [Params( + BenchmarkTypes.TechEmpowerPlaintext, + BenchmarkTypes.PlaintextChunked, + BenchmarkTypes.PlaintextWithCookie, + BenchmarkTypes.PlaintextChunkedWithCookie, + BenchmarkTypes.LiveAspNet + )] + public BenchmarkTypes Type { get; set; } + + [Benchmark] + public async Task Output() + { + _frame.Reset(); + _frame.StatusCode = 200; + + Task writeTask = Task.CompletedTask; + switch (Type) + { + case BenchmarkTypes.TechEmpowerPlaintext: + writeTask = TechEmpowerPlaintext(); + break; + case BenchmarkTypes.PlaintextChunked: + writeTask = PlaintextChunked(); + break; + case BenchmarkTypes.PlaintextWithCookie: + writeTask = PlaintextWithCookie(); + break; + case BenchmarkTypes.PlaintextChunkedWithCookie: + writeTask = PlaintextChunkedWithCookie(); + break; + case BenchmarkTypes.LiveAspNet: + writeTask = LiveAspNet(); + break; + } + + await writeTask; + await _frame.ProduceEndAsync(); + } + + private Task TechEmpowerPlaintext() + { + var responseHeaders = _frame.ResponseHeaders; + responseHeaders["Content-Type"] = "text/plain"; + responseHeaders.ContentLength = _helloWorldPayload.Length; + return _frame.WriteAsync(new ArraySegment(_helloWorldPayload), default(CancellationToken)); + } + + private Task PlaintextChunked() + { + var responseHeaders = _frame.ResponseHeaders; + responseHeaders["Content-Type"] = "text/plain"; + return _frame.WriteAsync(new ArraySegment(_helloWorldPayload), default(CancellationToken)); + } + + private Task LiveAspNet() + { + var responseHeaders = _frame.ResponseHeaders; + responseHeaders["Content-Encoding"] = "gzip"; + responseHeaders["Content-Type"] = "text/html; charset=utf-8"; + responseHeaders["Strict-Transport-Security"] = "max-age=31536000; includeSubdomains"; + responseHeaders["Vary"] = "Accept-Encoding"; + responseHeaders["X-Powered-By"] = "ASP.NET"; + return _frame.WriteAsync(new ArraySegment(_helloWorldPayload), default(CancellationToken)); + } + + private Task PlaintextWithCookie() + { + var responseHeaders = _frame.ResponseHeaders; + responseHeaders["Content-Type"] = "text/plain"; + responseHeaders["Set-Cookie"] = "prov=20629ccd-8b0f-e8ef-2935-cd26609fc0bc; __qca=P0-1591065732-1479167353442; _ga=GA1.2.1298898376.1479167354; _gat=1; sgt=id=9519gfde_3347_4762_8762_df51458c8ec2; acct=t=why-is-%e0%a5%a7%e0%a5%a8%e0%a5%a9-numeric&s=why-is-%e0%a5%a7%e0%a5%a8%e0%a5%a9-numeric"; + responseHeaders.ContentLength = _helloWorldPayload.Length; + return _frame.WriteAsync(new ArraySegment(_helloWorldPayload), default(CancellationToken)); + } + + private Task PlaintextChunkedWithCookie() + { + var responseHeaders = _frame.ResponseHeaders; + responseHeaders["Content-Type"] = "text/plain"; + responseHeaders["Set-Cookie"] = "prov=20629ccd-8b0f-e8ef-2935-cd26609fc0bc; __qca=P0-1591065732-1479167353442; _ga=GA1.2.1298898376.1479167354; _gat=1; sgt=id=9519gfde_3347_4762_8762_df51458c8ec2; acct=t=why-is-%e0%a5%a7%e0%a5%a8%e0%a5%a9-numeric&s=why-is-%e0%a5%a7%e0%a5%a8%e0%a5%a9-numeric"; + return _frame.WriteAsync(new ArraySegment(_helloWorldPayload), default(CancellationToken)); + } + + private TestFrame MakeFrame() + { + var socketInput = new PipeFactory().Create(); + + var serviceContext = new ServiceContext + { + DateHeaderValueManager = new DateHeaderValueManager(), + ServerOptions = new KestrelServerOptions(), + Log = Mock.Of() + }; + var listenerContext = new ListenerContext(serviceContext) + { + ListenOptions = new ListenOptions(new IPEndPoint(IPAddress.Loopback, 5000)) + }; + var connectionContext = new ConnectionContext(listenerContext) + { + Input = socketInput, + Output = new MockSocketOutput(), + ConnectionControl = Mock.Of() + }; + connectionContext.ListenerContext.ServiceContext.HttpParserFactory = f => new Internal.Http.KestrelHttpParser(log: null); + + var frame = new TestFrame(application: null, context: connectionContext); + frame.Reset(); + frame.InitializeHeaders(); + + return frame; + } + } +}