Skip to content
This repository was archived by the owner on Dec 18, 2018. It is now read-only.

Speed up WritableBuffer.WriteXxx #1504

Merged
merged 6 commits into from
Mar 18, 2017
Merged

Speed up WritableBuffer.WriteXxx #1504

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4511b3a
Speed up WritableBuffer.WriteAscii
benaadams Mar 18, 2017
54421c4
Add Benchmarks
benaadams Mar 18, 2017
31258df
Non-standard header vals are still ascii
benaadams Mar 18, 2017
2d77c4f
Speedup WriteNumeric
benaadams Mar 18, 2017
a220436
Don't advance for write
benaadams Mar 18, 2017
b84fe47
Remove cruft
benaadams Mar 18, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/FrameResponseHeaders.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public void CopyTo(ref WritableBuffer output)
output.Write(_CrLf);
output.WriteAscii(kv.Key);
output.Write(_colonSpace);
output.Write(value);
output.WriteAscii(value);
}
}
}
Expand Down
236 changes: 230 additions & 6 deletions src/Microsoft.AspNetCore.Server.Kestrel/Internal/Http/PipelineExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Internal.Http
{
public static class PipelineExtensions
{
private const int _maxULongByteLength = 20;

[ThreadStatic]
private static byte[] _numericBytesScratch;

public static ValueTask<ArraySegment<byte>> PeekAsync(this IPipeReader pipelineReader)
{
var input = pipelineReader.ReadAsync();
Expand Down Expand Up @@ -90,18 +95,237 @@ public static ArraySegment<byte> GetArray(this Memory<byte> memory)
return result;
}

public static void WriteAscii(this WritableBuffer buffer, string data)
public unsafe static void WriteAscii(this WritableBuffer buffer, string data)
{
if (!string.IsNullOrEmpty(data))
{
if (buffer.Memory.IsEmpty)
{
buffer.Ensure();
}

// Fast path, try copying to the available memory directly
if (data.Length <= buffer.Memory.Length)
{
fixed (char* input = data)
fixed (byte* output = &buffer.Memory.Span.DangerousGetPinnableReference())
{
EncodeAsciiCharsToBytes(input, output, data.Length);
}

buffer.Advance(data.Length);
}
else
{
buffer.WriteAsciiMultiWrite(data);
}
}
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
public unsafe static void WriteNumeric(this WritableBuffer buffer, ulong number)
{
const byte AsciiDigitStart = (byte)'0';

if (buffer.Memory.IsEmpty)
{
buffer.Ensure();
}

// Fast path, try copying to the available memory directly
var bytesLeftInBlock = buffer.Memory.Length;
var simpleWrite = true;
fixed (byte* output = &buffer.Memory.Span.DangerousGetPinnableReference())
{
var start = output;
if (number < 10 && bytesLeftInBlock >= 1)
{
*(start) = (byte)(((uint)number) + AsciiDigitStart);
buffer.Advance(1);
}
else if (number < 100 && bytesLeftInBlock >= 2)
{
var val = (uint)number;
var tens = (byte)((val * 205u) >> 11); // div10, valid to 1028

*(start) = (byte)(tens + AsciiDigitStart);
*(start + 1) = (byte)(val - (tens * 10) + AsciiDigitStart);
buffer.Advance(2);
}
else if (number < 1000 && bytesLeftInBlock >= 3)
{
var val = (uint)number;
var digit0 = (byte)((val * 41u) >> 12); // div100, valid to 1098
var digits01 = (byte)((val * 205u) >> 11); // div10, valid to 1028

*(start) = (byte)(digit0 + AsciiDigitStart);
*(start + 1) = (byte)(digits01 - (digit0 * 10) + AsciiDigitStart);
*(start + 2) = (byte)(val - (digits01 * 10) + AsciiDigitStart);
buffer.Advance(3);
}
else
{
simpleWrite = false;
}
}

if (!simpleWrite)
{
buffer.WriteNumericMultiWrite(number);
}
}

[MethodImpl(MethodImplOptions.NoInlining)]
private static unsafe void WriteNumericMultiWrite(this WritableBuffer buffer, ulong number)
{
const byte AsciiDigitStart = (byte)'0';

var value = number;
var position = _maxULongByteLength;
var byteBuffer = NumericBytesScratch;
do
{
// Consider using Math.DivRem() if available
var quotient = value / 10;
byteBuffer[--position] = (byte)(AsciiDigitStart + (value - quotient * 10)); // 0x30 = '0'
value = quotient;
}
while (value != 0);

var length = _maxULongByteLength - position;
buffer.Write(new ReadOnlySpan<byte>(byteBuffer, position, length));
}

[MethodImpl(MethodImplOptions.NoInlining)]
private unsafe static void WriteAsciiMultiWrite(this WritableBuffer buffer, string data)
{
buffer.Write(Encoding.ASCII.GetBytes(data));
var remaining = data.Length;

fixed (char* input = data)
{
var inputSlice = input;

while (remaining > 0)
{
var writable = Math.Min(remaining, buffer.Memory.Length);

buffer.Ensure(writable);

if (writable == 0)
{
continue;
}

fixed (byte* output = &buffer.Memory.Span.DangerousGetPinnableReference())
{
EncodeAsciiCharsToBytes(inputSlice, output, writable);
}

inputSlice += writable;
remaining -= writable;

buffer.Advance(writable);
}
}
}
public static void Write(this WritableBuffer buffer, string data)

private unsafe static void EncodeAsciiCharsToBytes(char* input, byte* output, int length)
{
buffer.Write(Encoding.UTF8.GetBytes(data));
// Note: Not BIGENDIAN or check for non-ascii
const int Shift16Shift24 = (1 << 16) | (1 << 24);
const int Shift8Identity = (1 << 8) | (1);

// Encode as bytes upto the first non-ASCII byte and return count encoded
int i = 0;
// Use Intrinsic switch
if (IntPtr.Size == 8) // 64 bit
{
if (length < 4) goto trailing;

int unaligned = (int)(((ulong)input) & 0x7) >> 1;
// Unaligned chars
for (; i < unaligned; i++)
{
char ch = *(input + i);
*(output + i) = (byte)ch; // Cast convert
}

// Aligned
int ulongDoubleCount = (length - i) & ~0x7;
for (; i < ulongDoubleCount; i += 8)
{
ulong inputUlong0 = *(ulong*)(input + i);
ulong inputUlong1 = *(ulong*)(input + i + 4);
// Pack 16 ASCII chars into 16 bytes
*(uint*)(output + i) =
((uint)((inputUlong0 * Shift16Shift24) >> 24) & 0xffff) |
((uint)((inputUlong0 * Shift8Identity) >> 24) & 0xffff0000);
*(uint*)(output + i + 4) =
((uint)((inputUlong1 * Shift16Shift24) >> 24) & 0xffff) |
((uint)((inputUlong1 * Shift8Identity) >> 24) & 0xffff0000);
}
if (length - 4 > i)
{
ulong inputUlong = *(ulong*)(input + i);
// Pack 8 ASCII chars into 8 bytes
*(uint*)(output + i) =
((uint)((inputUlong * Shift16Shift24) >> 24) & 0xffff) |
((uint)((inputUlong * Shift8Identity) >> 24) & 0xffff0000);
i += 4;
}

trailing:
for (; i < length; i++)
{
char ch = *(input + i);
*(output + i) = (byte)ch; // Cast convert
}
}
else // 32 bit
{
// Unaligned chars
if ((unchecked((int)input) & 0x2) != 0)
{
char ch = *input;
i = 1;
*(output) = (byte)ch; // Cast convert
}

// Aligned
int uintCount = (length - i) & ~0x3;
for (; i < uintCount; i += 4)
{
uint inputUint0 = *(uint*)(input + i);
uint inputUint1 = *(uint*)(input + i + 2);
// Pack 4 ASCII chars into 4 bytes
*(ushort*)(output + i) = (ushort)(inputUint0 | (inputUint0 >> 8));
*(ushort*)(output + i + 2) = (ushort)(inputUint1 | (inputUint1 >> 8));
}
if (length - 1 > i)
{
uint inputUint = *(uint*)(input + i);
// Pack 2 ASCII chars into 2 bytes
*(ushort*)(output + i) = (ushort)(inputUint | (inputUint >> 8));
i += 2;
}

if (i < length)
{
char ch = *(input + i);
*(output + i) = (byte)ch; // Cast convert
i = length;
}
}
}

public static void WriteNumeric(this WritableBuffer buffer, ulong number)
private static byte[] NumericBytesScratch => _numericBytesScratch ?? CreateNumericBytesScratch();

[MethodImpl(MethodImplOptions.NoInlining)]
private static byte[] CreateNumericBytesScratch()
{
buffer.Write(number.ToString());
var bytes = new byte[_maxULongByteLength];
_numericBytesScratch = bytes;
return bytes;
}
}
}
43 changes: 29 additions & 14 deletions ...l.Performance/ResponseHeadersBenchmark.cs → ...ance/ResponseHeaderCollectionBenchmark.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,16 @@

using System.Runtime.CompilerServices;
using System.Text;
using BenchmarkDotNet.Attributes;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Http.Internal;
using Microsoft.AspNetCore.Server.Kestrel.Internal.Http;
using Microsoft.AspNetCore.Testing;
using BenchmarkDotNet.Attributes;

namespace Microsoft.AspNetCore.Server.Kestrel.Performance
{
[Config(typeof(CoreConfig))]
public class ResponseHeadersBenchmark
public class ResponseHeaderCollectionBenchmark
{
private const int InnerLoopCount = 512;

Expand All @@ -21,27 +21,42 @@ public class ResponseHeadersBenchmark
private FrameResponseHeaders _responseHeadersDirect;
private HttpResponse _response;

[Params("ContentLengthNumeric", "ContentLengthString", "Plaintext", "Common", "Unknown")]
public string Type { get; set; }
public enum BenchmarkTypes
{
ContentLengthNumeric,
ContentLengthString,
Plaintext,
Common,
Unknown
}

[Params(
BenchmarkTypes.ContentLengthNumeric,
BenchmarkTypes.ContentLengthString,
BenchmarkTypes.Plaintext,
BenchmarkTypes.Common,
BenchmarkTypes.Unknown
)]
public BenchmarkTypes Type { get; set; }

[Benchmark(OperationsPerInvoke = InnerLoopCount)]
public void SetHeaders()
{
switch (Type)
{
case "ContentLengthNumeric":
case BenchmarkTypes.ContentLengthNumeric:
ContentLengthNumeric(InnerLoopCount);
break;
case "ContentLengthString":
case BenchmarkTypes.ContentLengthString:
ContentLengthString(InnerLoopCount);
break;
case "Plaintext":
case BenchmarkTypes.Plaintext:
Plaintext(InnerLoopCount);
break;
case "Common":
case BenchmarkTypes.Common:
Common(InnerLoopCount);
break;
case "Unknown":
case BenchmarkTypes.Unknown:
Unknown(InnerLoopCount);
break;
}
Expand Down Expand Up @@ -163,19 +178,19 @@ public void Setup()

switch (Type)
{
case "ContentLengthNumeric":
case BenchmarkTypes.ContentLengthNumeric:
ContentLengthNumeric(1);
break;
case "ContentLengthString":
case BenchmarkTypes.ContentLengthString:
ContentLengthString(1);
break;
case "Plaintext":
case BenchmarkTypes.Plaintext:
Plaintext(1);
break;
case "Common":
case BenchmarkTypes.Common:
Common(1);
break;
case "Unknown":
case BenchmarkTypes.Unknown:
Unknown(1);
break;
}
Expand Down
Loading