Skip to content
This repository was archived by the owner on Dec 18, 2018. It is now read-only.

Commit 3c38f48

Browse files
benaadamsbenaadams
committed
Faster CopyFrom
1 parent 0f389f0 commit 3c38f48

File tree

7 files changed

+153
-89
lines changed

7 files changed

+153
-89
lines changed

‎src/Microsoft.AspNet.Server.Kestrel/Http/SocketInput.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,10 @@ public IncomingBuffer IncomingStart(int minimumSize)
6262
{
6363
lock (_sync)
6464
{
65-
if (_tail != null && minimumSize <= _tail.Data.Offset + _tail.Data.Count - _tail.End)
65+
if (_tail != null && minimumSize <= _tail.BlockEndOffset - _tail.End)
6666
{
6767
_pinned = _tail;
68-
var data = new ArraySegment<byte>(_pinned.Data.Array, _pinned.End, _pinned.Data.Offset + _pinned.Data.Count - _pinned.End);
68+
var data = new ArraySegment<byte>(_pinned.Data.Array, _pinned.End, _pinned.BlockEndOffset - _pinned.End);
6969
var dataPtr = _pinned.Pin() + _pinned.End;
7070
return new IncomingBuffer
7171
{

‎src/Microsoft.AspNet.Server.Kestrel/Http/SocketOutput.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ public Task WriteAsync(
8787
if (buffer.Count > 0)
8888
{
8989
var tail = ProducingStart();
90-
tail.CopyFrom(buffer);
90+
tail.CopyFrom(buffer.Array, buffer.Offset, buffer.Count);
9191
// We do our own accounting below
9292
ProducingCompleteNoPreComplete(tail);
9393
}
@@ -423,7 +423,7 @@ private static void BytesBetween(MemoryPoolIterator2 start, MemoryPoolIterator2
423423
return;
424424
}
425425

426-
bytes = start.Block.Data.Offset + start.Block.Data.Count - start.Index;
426+
bytes = start.Block.BlockEndOffset - start.Index;
427427
buffers = 1;
428428

429429
for (var block = start.Block.Next; block != end.Block; block = block.Next)

‎src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolBlock2.cs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,11 @@ protected MemoryPoolBlock2()
5252
/// </summary>
5353
public byte[] Array => Data.Array;
5454

55+
/// <summary>
56+
/// Fixed end offset of the block
57+
/// </summary>
58+
public int BlockEndOffset { get; private set; }
59+
5560
/// <summary>
5661
/// The Start represents the offset into Array where the range of "active" bytes begins. At the point when the block is leased
5762
/// the Start is guaranteed to be equal to Array.Offset. The value of Start may be assigned anywhere between Data.Offset and
@@ -144,6 +149,7 @@ public static MemoryPoolBlock2 Create(
144149
Slab = slab,
145150
Start = data.Offset,
146151
End = data.Offset,
152+
BlockEndOffset = data.Offset + data.Count
147153
};
148154
}
149155

‎src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2.cs

Lines changed: 139 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -513,21 +513,24 @@ public int GetLength(MemoryPoolIterator2 end)
513513
var block = _block;
514514
var index = _index;
515515
var length = 0;
516-
while (true)
516+
checked
517517
{
518-
if (block == end._block)
519-
{
520-
return length + end._index - index;
521-
}
522-
else if (block.Next == null)
518+
while (true)
523519
{
524-
throw new InvalidOperationException("end did not follow iterator");
525-
}
526-
else
527-
{
528-
length += block.End - index;
529-
block = block.Next;
530-
index = block.Start;
520+
if (block == end._block)
521+
{
522+
return length + end._index - index;
523+
}
524+
else if (block.Next == null)
525+
{
526+
throw new InvalidOperationException("end did not follow iterator");
527+
}
528+
else
529+
{
530+
length += block.End - index;
531+
block = block.Next;
532+
index = block.Start;
533+
}
531534
}
532535
}
533536
}
@@ -588,113 +591,168 @@ public void CopyFrom(ArraySegment<byte> buffer)
588591
CopyFrom(buffer.Array, buffer.Offset, buffer.Count);
589592
}
590593

591-
public void CopyFrom(byte[] data, int offset, int count)
594+
public unsafe void CopyFrom(byte[] data, int offset, int count)
592595
{
593-
Debug.Assert(_block != null);
594-
Debug.Assert(_block.Pool != null);
595-
Debug.Assert(_block.Next == null);
596-
Debug.Assert(_block.End == _index);
597-
598-
var pool = _block.Pool;
599596
var block = _block;
600597
var blockIndex = _index;
598+
var bytesLeftInBlock = block.BlockEndOffset - blockIndex;
601599

602-
var bufferIndex = offset;
603-
var remaining = count;
604-
var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex;
600+
if (bytesLeftInBlock >= count)
601+
{
602+
_index = blockIndex + count;
603+
Buffer.BlockCopy(data, offset, block.Array, blockIndex, count);
604+
block.End = _index;
605+
return;
606+
}
605607

606-
while (remaining > 0)
608+
do
607609
{
608610
if (bytesLeftInBlock == 0)
609611
{
610-
var nextBlock = pool.Lease();
611-
block.End = blockIndex;
612+
var nextBlock = block.Pool.Lease();
613+
blockIndex = nextBlock.Data.Offset;
614+
bytesLeftInBlock = nextBlock.Data.Count;
612615
block.Next = nextBlock;
613616
block = nextBlock;
614-
615-
blockIndex = block.Data.Offset;
616-
bytesLeftInBlock = block.Data.Count;
617617
}
618618

619-
var bytesToCopy = remaining < bytesLeftInBlock ? remaining : bytesLeftInBlock;
620-
621-
Buffer.BlockCopy(data, bufferIndex, block.Array, blockIndex, bytesToCopy);
622-
623-
blockIndex += bytesToCopy;
624-
bufferIndex += bytesToCopy;
625-
remaining -= bytesToCopy;
626-
bytesLeftInBlock -= bytesToCopy;
627-
}
628-
629-
block.End = blockIndex;
630-
_block = block;
631-
_index = blockIndex;
619+
if (count > bytesLeftInBlock)
620+
{
621+
count -= bytesLeftInBlock;
622+
Buffer.BlockCopy(data, offset, block.Array, blockIndex, bytesLeftInBlock);
623+
offset += bytesLeftInBlock;
624+
block.End = blockIndex + bytesLeftInBlock;
625+
bytesLeftInBlock = 0;
626+
continue;
627+
}
628+
else
629+
{
630+
_index = blockIndex + count;
631+
Buffer.BlockCopy(data, offset, block.Array, blockIndex, count);
632+
block.End = _index;
633+
_block = block;
634+
return;
635+
}
636+
} while (true);
632637
}
633638

634639
public unsafe void CopyFromAscii(string data)
635640
{
636-
Debug.Assert(_block != null);
637-
Debug.Assert(_block.Pool != null);
638-
Debug.Assert(_block.Next == null);
639-
Debug.Assert(_block.End == _index);
640-
641-
var pool = _block.Pool;
642641
var block = _block;
643642
var blockIndex = _index;
644-
var length = data.Length;
643+
var count = data.Length;
645644

646-
var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex;
647-
var bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3;
645+
var blockRemaining = block.BlockEndOffset - blockIndex;
648646

649647
fixed (char* pData = data)
650648
{
651-
var input = pData;
652-
var inputEnd = pData + length;
653-
var inputEndMinusSpan = inputEnd - 3;
649+
if (blockRemaining >= count)
650+
{
651+
_index = blockIndex + count;
652+
653+
fixed (byte* output = block.Array)
654+
{
655+
CopyFromAscii(pData, output + blockIndex, count);
656+
}
657+
658+
block.End = _index;
659+
return;
660+
}
654661

655-
while (input < inputEnd)
662+
var input = pData;
663+
do
656664
{
657-
if (bytesLeftInBlock == 0)
665+
if (blockRemaining == 0)
658666
{
659-
var nextBlock = pool.Lease();
660-
block.End = blockIndex;
667+
var nextBlock = block.Pool.Lease();
668+
blockIndex = nextBlock.Data.Offset;
669+
blockRemaining = nextBlock.Data.Count;
661670
block.Next = nextBlock;
662671
block = nextBlock;
663-
664-
blockIndex = block.Data.Offset;
665-
bytesLeftInBlock = block.Data.Count;
666-
bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3;
667672
}
668673

669-
fixed (byte* pOutput = block.Data.Array)
674+
if (count > blockRemaining)
670675
{
671-
var output = pOutput + block.End;
676+
count -= blockRemaining;
672677

673-
var copied = 0;
674-
for (; input < inputEndMinusSpan && copied < bytesLeftInBlockMinusSpan; copied += 4)
678+
fixed (byte* output = block.Array)
675679
{
676-
*(output) = (byte)*(input);
677-
*(output + 1) = (byte)*(input + 1);
678-
*(output + 2) = (byte)*(input + 2);
679-
*(output + 3) = (byte)*(input + 3);
680-
output += 4;
681-
input += 4;
680+
CopyFromAscii(input, output + blockIndex, blockRemaining);
682681
}
683-
for (; input < inputEnd && copied < bytesLeftInBlock; copied++)
682+
683+
block.End = blockIndex + blockRemaining;
684+
input += blockRemaining;
685+
blockRemaining = 0;
686+
continue;
687+
}
688+
else
689+
{
690+
_index = blockIndex + count;
691+
692+
fixed (byte* output = block.Array)
684693
{
685-
*(output++) = (byte)*(input++);
694+
CopyFromAscii(input, output + blockIndex, count);
686695
}
687696

688-
blockIndex += copied;
689-
bytesLeftInBlockMinusSpan -= copied;
690-
bytesLeftInBlock -= copied;
697+
block.End = _index;
698+
_block = block;
699+
return;
691700
}
692-
}
701+
} while (true);
693702
}
703+
}
694704

695-
block.End = blockIndex;
696-
_block = block;
697-
_index = blockIndex;
705+
private unsafe static void CopyFromAscii(char* input, byte* output, int count)
706+
{
707+
var i = 0;
708+
709+
while (i < count - 11)
710+
{
711+
i += 12;
712+
*(output) = (byte)*(input);
713+
*(output + 1) = (byte)*(input + 1);
714+
*(output + 2) = (byte)*(input + 2);
715+
*(output + 3) = (byte)*(input + 3);
716+
*(output + 4) = (byte)*(input + 4);
717+
*(output + 5) = (byte)*(input + 5);
718+
*(output + 6) = (byte)*(input + 6);
719+
*(output + 7) = (byte)*(input + 7);
720+
*(output + 8) = (byte)*(input + 8);
721+
*(output + 9) = (byte)*(input + 9);
722+
*(output + 10) = (byte)*(input + 10);
723+
*(output + 11) = (byte)*(input + 11);
724+
output += 12;
725+
input += 12;
726+
}
727+
if (i < count - 5)
728+
{
729+
i += 6;
730+
*(output) = (byte)*(input);
731+
*(output + 1) = (byte)*(input + 1);
732+
*(output + 2) = (byte)*(input + 2);
733+
*(output + 3) = (byte)*(input + 3);
734+
*(output + 4) = (byte)*(input + 4);
735+
*(output + 5) = (byte)*(input + 5);
736+
output += 6;
737+
input += 6;
738+
}
739+
if (i < count - 3)
740+
{
741+
i += 4;
742+
*(output) = (byte)*(input);
743+
*(output + 1) = (byte)*(input + 1);
744+
*(output + 2) = (byte)*(input + 2);
745+
*(output + 3) = (byte)*(input + 3);
746+
output += 4;
747+
input += 4;
748+
}
749+
while (i < count)
750+
{
751+
i++;
752+
*output = (byte)*input;
753+
output++;
754+
input++;
755+
}
698756
}
699757
}
700758
}

‎src/Microsoft.AspNet.Server.Kestrel/Networking/UvWriteReq.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ public unsafe void Write(
6666
for (var index = 0; index < nBuffers; index++)
6767
{
6868
var blockStart = block == start.Block ? start.Index : block.Data.Offset;
69-
var blockEnd = block == end.Block ? end.Index : block.Data.Offset + block.Data.Count;
69+
var blockEnd = block == end.Block ? end.Index : block.BlockEndOffset;
7070

7171
// create and pin each segment being written
7272
pBuffers[index] = Libuv.buf_init(

‎test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolBlock2Tests.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,7 @@ public void CopyFromCorrectlyTraversesBlocks()
245245

246246
Assert.Null(block1.Next);
247247

248-
end.CopyFrom(new ArraySegment<byte>(buffer));
248+
end.CopyFrom(buffer, 0, buffer.Length);
249249

250250
Assert.NotNull(block1.Next);
251251

‎test/Microsoft.AspNet.Server.KestrelTests/SocketOutputTests.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -308,11 +308,11 @@ public void ProducingStartAndProducingCompleteCanBeUsedDirectly()
308308

309309
// block 1
310310
var start = socketOutput.ProducingStart();
311-
start.Block.End = start.Block.Data.Offset + start.Block.Data.Count;
311+
start.Block.End = start.Block.BlockEndOffset;
312312

313313
// block 2
314314
var block2 = memory.Lease();
315-
block2.End = block2.Data.Offset + block2.Data.Count;
315+
block2.End = block2.BlockEndOffset;
316316
start.Block.Next = block2;
317317

318318
var end = new MemoryPoolIterator2(block2, block2.End);

0 commit comments

Comments
 (0)