Skip to content
This repository was archived by the owner on Dec 18, 2018. It is now read-only.

Commit af593d2

Browse files
benaadamsbenaadams
committed
Faster CopyFrom
1 parent 7cc5195 commit af593d2

File tree

7 files changed

+136
-67
lines changed

7 files changed

+136
-67
lines changed

‎src/Microsoft.AspNet.Server.Kestrel/Http/SocketInput.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,10 @@ public IncomingBuffer IncomingStart(int minimumSize)
6262
{
6363
lock (_sync)
6464
{
65-
if (_tail != null && minimumSize <= _tail.Data.Offset + _tail.Data.Count - _tail.End)
65+
if (_tail != null && minimumSize <= _tail.BlockEndOffset - _tail.End)
6666
{
6767
_pinned = _tail;
68-
var data = new ArraySegment<byte>(_pinned.Data.Array, _pinned.End, _pinned.Data.Offset + _pinned.Data.Count - _pinned.End);
68+
var data = new ArraySegment<byte>(_pinned.Data.Array, _pinned.End, _pinned.BlockEndOffset - _pinned.End);
6969
var dataPtr = _pinned.Pin() + _pinned.End;
7070
return new IncomingBuffer
7171
{

‎src/Microsoft.AspNet.Server.Kestrel/Http/SocketOutput.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ public Task WriteAsync(
8787
if (buffer.Count > 0)
8888
{
8989
var tail = ProducingStart();
90-
tail.CopyFrom(buffer);
90+
tail.CopyFrom(buffer.Array, buffer.Offset, buffer.Count);
9191
// We do our own accounting below
9292
ProducingCompleteNoPreComplete(tail);
9393
}
@@ -423,7 +423,7 @@ private static void BytesBetween(MemoryPoolIterator2 start, MemoryPoolIterator2
423423
return;
424424
}
425425

426-
bytes = start.Block.Data.Offset + start.Block.Data.Count - start.Index;
426+
bytes = start.Block.BlockEndOffset - start.Index;
427427
buffers = 1;
428428

429429
for (var block = start.Block.Next; block != end.Block; block = block.Next)

‎src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolBlock2.cs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,11 @@ protected MemoryPoolBlock2()
5252
/// </summary>
5353
public byte[] Array => Data.Array;
5454

55+
/// <summary>
56+
/// Fixed end offset of the block
57+
/// </summary>
58+
public int BlockEndOffset { get; private set; }
59+
5560
/// <summary>
5661
/// The Start represents the offset into Array where the range of "active" bytes begins. At the point when the block is leased
5762
/// the Start is guaranteed to be equal to Array.Offset. The value of Start may be assigned anywhere between Data.Offset and
@@ -144,6 +149,7 @@ public static MemoryPoolBlock2 Create(
144149
Slab = slab,
145150
Start = data.Offset,
146151
End = data.Offset,
152+
BlockEndOffset = data.Offset + data.Count
147153
};
148154
}
149155

‎src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2.cs

Lines changed: 122 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -632,47 +632,53 @@ public void CopyFrom(ArraySegment<byte> buffer)
632632
CopyFrom(buffer.Array, buffer.Offset, buffer.Count);
633633
}
634634

635-
public void CopyFrom(byte[] data, int offset, int count)
635+
public unsafe void CopyFrom(byte[] data, int offset, int count)
636636
{
637637
Debug.Assert(_block != null);
638638
Debug.Assert(_block.Pool != null);
639639
Debug.Assert(_block.Next == null);
640640
Debug.Assert(_block.End == _index);
641-
642-
var pool = _block.Pool;
641+
643642
var block = _block;
644643
var blockIndex = _index;
644+
var bytesLeftInBlock = block.BlockEndOffset - blockIndex;
645645

646-
var bufferIndex = offset;
647-
var remaining = count;
648-
var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex;
646+
if (bytesLeftInBlock >= count)
647+
{
648+
_index = blockIndex + count;
649+
Buffer.BlockCopy(data, offset, block.Array, blockIndex, count);
650+
block.End = _index;
651+
return;
652+
}
649653

650-
while (remaining > 0)
654+
do
651655
{
652656
if (bytesLeftInBlock == 0)
653657
{
654-
var nextBlock = pool.Lease();
655-
block.End = blockIndex;
658+
var nextBlock = block.Pool.Lease();
659+
blockIndex = nextBlock.Data.Offset;
660+
bytesLeftInBlock = nextBlock.Data.Count;
656661
block.Next = nextBlock;
657662
block = nextBlock;
658-
659-
blockIndex = block.Data.Offset;
660-
bytesLeftInBlock = block.Data.Count;
661663
}
662664

663-
var bytesToCopy = remaining < bytesLeftInBlock ? remaining : bytesLeftInBlock;
664-
665-
Buffer.BlockCopy(data, bufferIndex, block.Array, blockIndex, bytesToCopy);
666-
667-
blockIndex += bytesToCopy;
668-
bufferIndex += bytesToCopy;
669-
remaining -= bytesToCopy;
670-
bytesLeftInBlock -= bytesToCopy;
671-
}
672-
673-
block.End = blockIndex;
674-
_block = block;
675-
_index = blockIndex;
665+
if (count > bytesLeftInBlock)
666+
{
667+
count -= bytesLeftInBlock;
668+
Buffer.BlockCopy(data, offset, block.Array, blockIndex, bytesLeftInBlock);
669+
offset += bytesLeftInBlock;
670+
block.End = blockIndex + bytesLeftInBlock;
671+
bytesLeftInBlock = 0;
672+
}
673+
else
674+
{
675+
_index = blockIndex + count;
676+
Buffer.BlockCopy(data, offset, block.Array, blockIndex, count);
677+
block.End = _index;
678+
_block = block;
679+
return;
680+
}
681+
} while (true);
676682
}
677683

678684
public unsafe void CopyFromAscii(string data)
@@ -682,63 +688,120 @@ public unsafe void CopyFromAscii(string data)
682688
Debug.Assert(_block.Next == null);
683689
Debug.Assert(_block.End == _index);
684690

685-
var pool = _block.Pool;
686691
var block = _block;
687692
var blockIndex = _index;
688-
var length = data.Length;
693+
var count = data.Length;
689694

690-
var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex;
691-
var bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3;
695+
var blockRemaining = block.BlockEndOffset - blockIndex;
692696

693697
fixed (char* pData = data)
694698
{
695-
var input = pData;
696-
var inputEnd = pData + length;
697-
var inputEndMinusSpan = inputEnd - 3;
699+
if (blockRemaining >= count)
700+
{
701+
_index = blockIndex + count;
702+
703+
fixed (byte* output = block.Array)
704+
{
705+
CopyFromAscii(pData, output + blockIndex, count);
706+
}
698707

699-
while (input < inputEnd)
708+
block.End = _index;
709+
return;
710+
}
711+
712+
var input = pData;
713+
do
700714
{
701-
if (bytesLeftInBlock == 0)
715+
if (blockRemaining == 0)
702716
{
703-
var nextBlock = pool.Lease();
704-
block.End = blockIndex;
717+
var nextBlock = block.Pool.Lease();
718+
blockIndex = nextBlock.Data.Offset;
719+
blockRemaining = nextBlock.Data.Count;
705720
block.Next = nextBlock;
706721
block = nextBlock;
707-
708-
blockIndex = block.Data.Offset;
709-
bytesLeftInBlock = block.Data.Count;
710-
bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3;
711722
}
712723

713-
fixed (byte* pOutput = block.Data.Array)
724+
if (count > blockRemaining)
714725
{
715-
var output = pOutput + block.End;
726+
count -= blockRemaining;
716727

717-
var copied = 0;
718-
for (; input < inputEndMinusSpan && copied < bytesLeftInBlockMinusSpan; copied += 4)
728+
fixed (byte* output = block.Array)
719729
{
720-
*(output) = (byte)*(input);
721-
*(output + 1) = (byte)*(input + 1);
722-
*(output + 2) = (byte)*(input + 2);
723-
*(output + 3) = (byte)*(input + 3);
724-
output += 4;
725-
input += 4;
730+
CopyFromAscii(input, output + blockIndex, blockRemaining);
726731
}
727-
for (; input < inputEnd && copied < bytesLeftInBlock; copied++)
732+
733+
block.End = blockIndex + blockRemaining;
734+
input += blockRemaining;
735+
blockRemaining = 0;
736+
}
737+
else
738+
{
739+
_index = blockIndex + count;
740+
741+
fixed (byte* output = block.Array)
728742
{
729-
*(output++) = (byte)*(input++);
743+
CopyFromAscii(input, output + blockIndex, count);
730744
}
731745

732-
blockIndex += copied;
733-
bytesLeftInBlockMinusSpan -= copied;
734-
bytesLeftInBlock -= copied;
746+
block.End = _index;
747+
_block = block;
748+
return;
735749
}
736-
}
750+
} while (true);
737751
}
752+
}
738753

739-
block.End = blockIndex;
740-
_block = block;
741-
_index = blockIndex;
754+
private unsafe static void CopyFromAscii(char* input, byte* output, int count)
755+
{
756+
var i = 0;
757+
758+
while (i < count - 11)
759+
{
760+
i += 12;
761+
*(output) = (byte)*(input);
762+
*(output + 1) = (byte)*(input + 1);
763+
*(output + 2) = (byte)*(input + 2);
764+
*(output + 3) = (byte)*(input + 3);
765+
*(output + 4) = (byte)*(input + 4);
766+
*(output + 5) = (byte)*(input + 5);
767+
*(output + 6) = (byte)*(input + 6);
768+
*(output + 7) = (byte)*(input + 7);
769+
*(output + 8) = (byte)*(input + 8);
770+
*(output + 9) = (byte)*(input + 9);
771+
*(output + 10) = (byte)*(input + 10);
772+
*(output + 11) = (byte)*(input + 11);
773+
output += 12;
774+
input += 12;
775+
}
776+
if (i < count - 5)
777+
{
778+
i += 6;
779+
*(output) = (byte)*(input);
780+
*(output + 1) = (byte)*(input + 1);
781+
*(output + 2) = (byte)*(input + 2);
782+
*(output + 3) = (byte)*(input + 3);
783+
*(output + 4) = (byte)*(input + 4);
784+
*(output + 5) = (byte)*(input + 5);
785+
output += 6;
786+
input += 6;
787+
}
788+
if (i < count - 3)
789+
{
790+
i += 4;
791+
*(output) = (byte)*(input);
792+
*(output + 1) = (byte)*(input + 1);
793+
*(output + 2) = (byte)*(input + 2);
794+
*(output + 3) = (byte)*(input + 3);
795+
output += 4;
796+
input += 4;
797+
}
798+
while (i < count)
799+
{
800+
i++;
801+
*output = (byte)*input;
802+
output++;
803+
input++;
804+
}
742805
}
743806
}
744807
}

‎src/Microsoft.AspNet.Server.Kestrel/Networking/UvWriteReq.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ public unsafe void Write(
6666
for (var index = 0; index < nBuffers; index++)
6767
{
6868
var blockStart = block == start.Block ? start.Index : block.Data.Offset;
69-
var blockEnd = block == end.Block ? end.Index : block.Data.Offset + block.Data.Count;
69+
var blockEnd = block == end.Block ? end.Index : block.BlockEndOffset;
7070

7171
// create and pin each segment being written
7272
pBuffers[index] = Libuv.buf_init(

‎test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolBlock2Tests.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,7 @@ public void CopyFromCorrectlyTraversesBlocks()
245245

246246
Assert.Null(block1.Next);
247247

248-
end.CopyFrom(new ArraySegment<byte>(buffer));
248+
end.CopyFrom(buffer, 0, buffer.Length);
249249

250250
Assert.NotNull(block1.Next);
251251

‎test/Microsoft.AspNet.Server.KestrelTests/SocketOutputTests.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -308,11 +308,11 @@ public void ProducingStartAndProducingCompleteCanBeUsedDirectly()
308308

309309
// block 1
310310
var start = socketOutput.ProducingStart();
311-
start.Block.End = start.Block.Data.Offset + start.Block.Data.Count;
311+
start.Block.End = start.Block.BlockEndOffset;
312312

313313
// block 2
314314
var block2 = memory.Lease();
315-
block2.End = block2.Data.Offset + block2.Data.Count;
315+
block2.End = block2.BlockEndOffset;
316316
start.Block.Next = block2;
317317

318318
var end = new MemoryPoolIterator2(block2, block2.End);

0 commit comments

Comments
 (0)