Skip to content
This repository was archived by the owner on Dec 18, 2018. It is now read-only.

Commit caaf9d4

Browse files
benaadamsbenaadams
committed
Faster CopyFrom
1 parent d9f6ac7 commit caaf9d4

File tree

8 files changed

+139
-68
lines changed

8 files changed

+139
-68
lines changed

‎src/Microsoft.AspNet.Server.Kestrel/Http/Connection.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ private Libuv.uv_buf_t OnAlloc(UvStreamHandle handle, int suggestedSize)
144144

145145
return handle.Libuv.buf_init(
146146
result.Pin() + result.End,
147-
result.Data.Offset + result.Data.Count - result.End);
147+
result.BlockEndOffset - result.End);
148148
}
149149

150150
private static void ReadCallback(UvStreamHandle handle, int status, object state)

‎src/Microsoft.AspNet.Server.Kestrel/Http/SocketInput.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ public MemoryPoolBlock2 IncomingStart()
4242
{
4343
const int minimumSize = 2048;
4444

45-
if (_tail != null && minimumSize <= _tail.Data.Offset + _tail.Data.Count - _tail.End)
45+
if (_tail != null && minimumSize <= _tail.BlockEndOffset - _tail.End)
4646
{
4747
_pinned = _tail;
4848
}

‎src/Microsoft.AspNet.Server.Kestrel/Http/SocketOutput.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -400,7 +400,7 @@ private static void BytesBetween(MemoryPoolIterator2 start, MemoryPoolIterator2
400400
return;
401401
}
402402

403-
bytes = start.Block.Data.Offset + start.Block.Data.Count - start.Index;
403+
bytes = start.Block.BlockEndOffset - start.Index;
404404
buffers = 1;
405405

406406
for (var block = start.Block.Next; block != end.Block; block = block.Next)

‎src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolBlock2.cs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,11 @@ protected MemoryPoolBlock2()
5252
/// </summary>
5353
public byte[] Array => Data.Array;
5454

55+
/// <summary>
56+
/// Fixed end offset of the block
57+
/// </summary>
58+
public int BlockEndOffset { get; private set; }
59+
5560
/// <summary>
5661
/// The Start represents the offset into Array where the range of "active" bytes begins. At the point when the block is leased
5762
/// the Start is guaranteed to be equal to Array.Offset. The value of Start may be assigned anywhere between Data.Offset and
@@ -144,6 +149,7 @@ public static MemoryPoolBlock2 Create(
144149
Slab = slab,
145150
Start = data.Offset,
146151
End = data.Offset,
152+
BlockEndOffset = data.Offset + data.Count
147153
};
148154
}
149155

‎src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2.cs

Lines changed: 126 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -692,46 +692,52 @@ public void CopyFrom(ArraySegment<byte> buffer)
692692
CopyFrom(buffer.Array, buffer.Offset, buffer.Count);
693693
}
694694

695-
public void CopyFrom(byte[] data, int offset, int count)
695+
public unsafe void CopyFrom(byte[] data, int offset, int count)
696696
{
697697
Debug.Assert(_block != null);
698698
Debug.Assert(_block.Next == null);
699699
Debug.Assert(_block.End == _index);
700-
701-
var pool = _block.Pool;
700+
702701
var block = _block;
703702
var blockIndex = _index;
703+
var bytesLeftInBlock = block.BlockEndOffset - blockIndex;
704704

705-
var bufferIndex = offset;
706-
var remaining = count;
707-
var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex;
705+
if (bytesLeftInBlock >= count)
706+
{
707+
_index = blockIndex + count;
708+
Buffer.BlockCopy(data, offset, block.Array, blockIndex, count);
709+
block.End = _index;
710+
return;
711+
}
708712

709-
while (remaining > 0)
713+
do
710714
{
711715
if (bytesLeftInBlock == 0)
712716
{
713-
var nextBlock = pool.Lease();
714-
block.End = blockIndex;
717+
var nextBlock = block.Pool.Lease();
718+
blockIndex = nextBlock.Data.Offset;
719+
bytesLeftInBlock = nextBlock.Data.Count;
715720
block.Next = nextBlock;
716721
block = nextBlock;
717-
718-
blockIndex = block.Data.Offset;
719-
bytesLeftInBlock = block.Data.Count;
720722
}
721723

722-
var bytesToCopy = remaining < bytesLeftInBlock ? remaining : bytesLeftInBlock;
723-
724-
Buffer.BlockCopy(data, bufferIndex, block.Array, blockIndex, bytesToCopy);
725-
726-
blockIndex += bytesToCopy;
727-
bufferIndex += bytesToCopy;
728-
remaining -= bytesToCopy;
729-
bytesLeftInBlock -= bytesToCopy;
730-
}
731-
732-
block.End = blockIndex;
733-
_block = block;
734-
_index = blockIndex;
724+
if (count > bytesLeftInBlock)
725+
{
726+
count -= bytesLeftInBlock;
727+
Buffer.BlockCopy(data, offset, block.Array, blockIndex, bytesLeftInBlock);
728+
offset += bytesLeftInBlock;
729+
block.End = blockIndex + bytesLeftInBlock;
730+
bytesLeftInBlock = 0;
731+
}
732+
else
733+
{
734+
_index = blockIndex + count;
735+
Buffer.BlockCopy(data, offset, block.Array, blockIndex, count);
736+
block.End = _index;
737+
_block = block;
738+
return;
739+
}
740+
} while (true);
735741
}
736742

737743
public unsafe void CopyFromAscii(string data)
@@ -740,64 +746,123 @@ public unsafe void CopyFromAscii(string data)
740746
Debug.Assert(_block.Next == null);
741747
Debug.Assert(_block.End == _index);
742748

743-
var pool = _block.Pool;
744749
var block = _block;
745750
var blockIndex = _index;
746-
var length = data.Length;
751+
var count = data.Length;
747752

748-
var bytesLeftInBlock = block.Data.Offset + block.Data.Count - blockIndex;
749-
var bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3;
753+
var blockRemaining = block.BlockEndOffset - blockIndex;
750754

751-
fixed (char* pData = data)
755+
fixed (char* pInput = data)
752756
{
753-
var input = pData;
754-
var inputEnd = pData + length;
755-
var inputEndMinusSpan = inputEnd - 3;
757+
if (blockRemaining >= count)
758+
{
759+
_index = blockIndex + count;
760+
761+
fixed (byte* pOutput = &block.Data.Array[blockIndex])
762+
{
763+
CopyFromAscii(pInput, pOutput, count);
764+
}
756765

757-
while (input < inputEnd)
766+
block.End = _index;
767+
return;
768+
}
769+
770+
var input = pInput;
771+
do
758772
{
759-
if (bytesLeftInBlock == 0)
773+
if (blockRemaining == 0)
760774
{
761-
var nextBlock = pool.Lease();
762-
block.End = blockIndex;
775+
var nextBlock = block.Pool.Lease();
776+
blockIndex = nextBlock.Data.Offset;
777+
blockRemaining = nextBlock.Data.Count;
763778
block.Next = nextBlock;
764779
block = nextBlock;
765-
766-
blockIndex = block.Data.Offset;
767-
bytesLeftInBlock = block.Data.Count;
768-
bytesLeftInBlockMinusSpan = bytesLeftInBlock - 3;
769780
}
770781

771-
fixed (byte* pOutput = &block.Data.Array[block.End])
782+
if (count > blockRemaining)
772783
{
773-
//this line is needed to allow output be an register var
774-
var output = pOutput;
784+
count -= blockRemaining;
775785

776-
var copied = 0;
777-
for (; input < inputEndMinusSpan && copied < bytesLeftInBlockMinusSpan; copied += 4)
786+
fixed (byte* pOutput = &block.Data.Array[blockIndex])
778787
{
779-
*(output) = (byte)*(input);
780-
*(output + 1) = (byte)*(input + 1);
781-
*(output + 2) = (byte)*(input + 2);
782-
*(output + 3) = (byte)*(input + 3);
783-
output += 4;
784-
input += 4;
788+
CopyFromAscii(input, pOutput, blockRemaining);
785789
}
786-
for (; input < inputEnd && copied < bytesLeftInBlock; copied++)
790+
791+
block.End = blockIndex + blockRemaining;
792+
input += blockRemaining;
793+
blockRemaining = 0;
794+
}
795+
else
796+
{
797+
_index = blockIndex + count;
798+
799+
fixed (byte* pOutput = &block.Data.Array[blockIndex])
787800
{
788-
*(output++) = (byte)*(input++);
801+
CopyFromAscii(input, pOutput, count);
789802
}
790803

791-
blockIndex += copied;
792-
bytesLeftInBlockMinusSpan -= copied;
793-
bytesLeftInBlock -= copied;
804+
block.End = _index;
805+
_block = block;
806+
return;
794807
}
795-
}
808+
} while (true);
796809
}
810+
}
797811

798-
block.End = blockIndex;
799-
_block = block;
800-
_index = blockIndex;
812+
private unsafe static void CopyFromAscii(char* pInput, byte* pOutput, int count)
813+
{
814+
var i = 0;
815+
//these line is needed to allow input/output be an register var
816+
var input = pInput;
817+
var output = pOutput;
818+
819+
while (i < count - 11)
820+
{
821+
i += 12;
822+
*(output) = (byte)*(input);
823+
*(output + 1) = (byte)*(input + 1);
824+
*(output + 2) = (byte)*(input + 2);
825+
*(output + 3) = (byte)*(input + 3);
826+
*(output + 4) = (byte)*(input + 4);
827+
*(output + 5) = (byte)*(input + 5);
828+
*(output + 6) = (byte)*(input + 6);
829+
*(output + 7) = (byte)*(input + 7);
830+
*(output + 8) = (byte)*(input + 8);
831+
*(output + 9) = (byte)*(input + 9);
832+
*(output + 10) = (byte)*(input + 10);
833+
*(output + 11) = (byte)*(input + 11);
834+
output += 12;
835+
input += 12;
836+
}
837+
if (i < count - 5)
838+
{
839+
i += 6;
840+
*(output) = (byte)*(input);
841+
*(output + 1) = (byte)*(input + 1);
842+
*(output + 2) = (byte)*(input + 2);
843+
*(output + 3) = (byte)*(input + 3);
844+
*(output + 4) = (byte)*(input + 4);
845+
*(output + 5) = (byte)*(input + 5);
846+
output += 6;
847+
input += 6;
848+
}
849+
if (i < count - 3)
850+
{
851+
i += 4;
852+
*(output) = (byte)*(input);
853+
*(output + 1) = (byte)*(input + 1);
854+
*(output + 2) = (byte)*(input + 2);
855+
*(output + 3) = (byte)*(input + 3);
856+
output += 4;
857+
input += 4;
858+
}
859+
while (i < count)
860+
{
861+
i++;
862+
*output = (byte)*input;
863+
output++;
864+
input++;
865+
}
801866
}
802867
}
803868
}

‎src/Microsoft.AspNet.Server.Kestrel/Networking/UvWriteReq.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ public unsafe void Write(
6666
for (var index = 0; index < nBuffers; index++)
6767
{
6868
var blockStart = block == start.Block ? start.Index : block.Data.Offset;
69-
var blockEnd = block == end.Block ? end.Index : block.Data.Offset + block.Data.Count;
69+
var blockEnd = block == end.Block ? end.Index : block.BlockEndOffset;
7070

7171
// create and pin each segment being written
7272
pBuffers[index] = Libuv.buf_init(

‎test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolBlock2Tests.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,7 @@ public void CopyFromCorrectlyTraversesBlocks()
248248

249249
Assert.Null(block1.Next);
250250

251-
end.CopyFrom(new ArraySegment<byte>(buffer));
251+
end.CopyFrom(buffer, 0, buffer.Length);
252252

253253
Assert.NotNull(block1.Next);
254254

‎test/Microsoft.AspNet.Server.KestrelTests/SocketOutputTests.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -308,11 +308,11 @@ public void ProducingStartAndProducingCompleteCanBeUsedDirectly()
308308

309309
// block 1
310310
var start = socketOutput.ProducingStart();
311-
start.Block.End = start.Block.Data.Offset + start.Block.Data.Count;
311+
start.Block.End = start.Block.BlockEndOffset;
312312

313313
// block 2
314314
var block2 = memory.Lease();
315-
block2.End = block2.Data.Offset + block2.Data.Count;
315+
block2.End = block2.BlockEndOffset;
316316
start.Block.Next = block2;
317317

318318
var end = new MemoryPoolIterator2(block2, block2.End);

0 commit comments

Comments
 (0)