Skip to content
This repository was archived by the owner on Dec 18, 2018. It is now read-only.

Faster CopyFromAscii #1308

Closed
wants to merge 6 commits into from
Closed

Faster CopyFromAscii #1308

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7ae0e0d
Faster CopyFromAscii
benaadams Jan 14, 2017
1d14ae1
Share Single and Multi impl; add 32bit
benaadams Jan 20, 2017
55e5a56
Increase test range
benaadams Jan 20, 2017
66f7549
Add ResponseHeaders benchmark
benaadams Jan 22, 2017
ed66e0e
Use real world headers
benaadams Jan 22, 2017
06d1437
coreclr feedback
benaadams Jan 23, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
192 changes: 149 additions & 43 deletions src/Microsoft.AspNetCore.Server.Kestrel/Internal/Infrastructure/MemoryPoolIterator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1020,64 +1020,83 @@ public void CopyFrom(byte[] data, int offset, int count)
public unsafe void CopyFromAscii(string data)
{
var block = _block;
if (block == null)
if (block != null)
{
return;
}
Debug.Assert(block.Next == null);
Debug.Assert(block.End == _index);

Debug.Assert(block.Next == null);
Debug.Assert(block.End == _index);

var pool = block.Pool;
var blockIndex = _index;
var length = data.Length;
fixed (char* pData = data)
{
var input = pData;
var blockIndex = _index;
var length = data.Length;

var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex;
var bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3;
var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex;
var count = Math.Min(length, bytesLeftInBlock);
var output = (block.DataFixedPtr + block.End);

fixed (char* pData = data)
{
var input = pData;
var inputEnd = pData + length;
var inputEndMinusSpan = inputEnd - 3;
blockIndex += count;

while (input < inputEnd)
{
if (bytesLeftInBlock == 0)
// Only one path should inline, other branch should be eliminated
if (IntPtr.Size == 8)
{
var nextBlock = pool.Lease();
block.End = blockIndex;
Volatile.Write(ref block.Next, nextBlock);
block = nextBlock;

blockIndex = block.Data.Offset;
bytesLeftInBlock = block.Data.Count;
bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3;
CopyFromAscii64Bit(input, output, count);
}
else
{
CopyFromAscii32Bit(input, output, count);
}

var output = (block.DataFixedPtr + block.End);
var copied = 0;
for (; input < inputEndMinusSpan && copied < bytesLeftInBlockMinusSpan; copied += 4)
block.End = blockIndex;
if (length <= bytesLeftInBlock)
{
*(output) = (byte)*(input);
*(output + 1) = (byte)*(input + 1);
*(output + 2) = (byte)*(input + 2);
*(output + 3) = (byte)*(input + 3);
output += 4;
input += 4;
_index = blockIndex;
}
for (; input < inputEnd && copied < bytesLeftInBlock; copied++)
else
{
*(output++) = (byte)*(input++);
CopyFromAsciiMultiblock(input + count, length - bytesLeftInBlock);
}

blockIndex += copied;
bytesLeftInBlockMinusSpan -= copied;
bytesLeftInBlock -= copied;
}
}
}

private unsafe void CopyFromAsciiMultiblock(char* inputStart, int remainingLength)
{
var input = inputStart;
var length = remainingLength;
var block = _block;
var pool = block.Pool;
int blockIndex;
do
{
var nextBlock = pool.Lease();
Volatile.Write(ref block.Next, nextBlock);
block = nextBlock;

var bytesLeftInBlock = block.Data.Count;

var output = (block.DataFixedPtr + block.End);
var toCopy = Math.Min(length, block.Data.Count);
var toCopyULong = toCopy & ~0x3;

// Only one path should inline, other branch should be eliminated
blockIndex = block.Data.Offset + toCopy;

if (IntPtr.Size == 8)
{
CopyFromAscii64Bit(input, output, toCopy);
}
else
{
CopyFromAscii32Bit(input, output, toCopy);
}

block.End = blockIndex;

length -= toCopy;
input += toCopy;
} while (length > 0);

block.End = blockIndex;
_block = block;
_index = blockIndex;
}
Expand Down Expand Up @@ -1254,5 +1273,92 @@ public static Vector<byte> GetVector(byte vectorByte)
return Vector.AsVectorByte(new Vector<uint>(vectorByte * 0x01010101u));
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe static void CopyFromAscii64Bit(char* input, byte* output, int count)
{
const int Shift16Shift24 = (1 << 16) | (1 << 24);
const int Shift8Identity = (1 << 8) + 1;

// Encode as bytes upto the first non-ASCII byte and return count encoded
var i = 0;
if (count >= 4)
{
var unaligned = (unchecked(-(int)input) >> 1) & 0x3;
// Unaligned chars
for (; i < unaligned; i++)
{
var ch = *(input + i);
*(output + i) = (byte)ch; // Cast convert
}

// Aligned
var ulongDoubleCount = (count - i) & ~0x7;
for (; i < ulongDoubleCount; i += 8)
{
var inputUlong0 = *(ulong*)(input + i);
var inputUlong1 = *(ulong*)(input + i + 4);
// Pack 16 ASCII chars into 16 bytes
*(uint*)(output + i) =
((uint)((inputUlong0 * Shift16Shift24) >> 24) & 0xffff) |
((uint)((inputUlong0 * Shift8Identity) >> 24) & 0xffff0000);
*(uint*)(output + i + 4) =
((uint)((inputUlong1 * Shift16Shift24) >> 24) & 0xffff) |
((uint)((inputUlong1 * Shift8Identity) >> 24) & 0xffff0000);
}
if (count - 4 > i)
{
var inputUlong = *(ulong*)(input + i);
// Pack 8 ASCII chars into 8 bytes
*(uint*)(output + i) =
((uint)((inputUlong * Shift16Shift24) >> 24) & 0xffff) |
((uint)((inputUlong * Shift8Identity) >> 24) & 0xffff0000);
i += 4;
}
}

for (; i < count; i++)
{
var ch = *(input + i);
*(output + i) = (byte)ch; // Cast convert
}
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
private unsafe static void CopyFromAscii32Bit(char* input, byte* output, int count)
{
// Encode as bytes upto the first non-ASCII byte and return count encoded
var i = 0;
// Unaligned chars
if ((unchecked((int)input) & 0x2) != 0)
{
var ch = *input;
i = 1;
*(output) = (byte)ch; // Cast convert
}

// Aligned
var uintCount = (count - i) & ~0x3;
for (; i < uintCount; i += 4)
{
var inputUint0 = *(uint*)(input + i);
var inputUint1 = *(uint*)(input + i + 2);
// Pack 4 ASCII chars into 4 bytes
*(ushort*)(output + i) = (ushort)(inputUint0 | (inputUint0 >> 8));
*(ushort*)(output + i + 2) = (ushort)(inputUint1 | (inputUint1 >> 8));
}
if (count - 1 > i)
{
var inputUint = *(uint*)(input + i);
// Pack 2 ASCII chars into 2 bytes
*(ushort*)(output + i) = (ushort)(inputUint | (inputUint >> 8));
i += 2;
}

if (i < count)
{
var ch = *(input + i);
*(output + i) = (byte)ch; // Cast convert
}
}
}
}
5 changes: 5 additions & 0 deletions test/Microsoft.AspNetCore.Server.Kestrel.Performance/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ private static void RunSelectedBenchmarks(BenchmarkType type)
{
BenchmarkRunner.Run<Writing>();
}
if (type.HasFlag(BenchmarkType.ResponseHeaders))
{
BenchmarkRunner.Run<ResponseHeaders>();
}
}
}

Expand All @@ -44,6 +48,7 @@ public enum BenchmarkType : uint
{
RequestParsing = 1,
Writing = 2,
ResponseHeaders = 4,
// add new ones in powers of two - e.g. 2,4,8,16...

All = uint.MaxValue
Expand Down
Loading