Skip to content
This repository was archived by the owner on Dec 18, 2018. It is now read-only.

Commit 4460cd3

Browse files
benaadamsbenaadams
committed
Don't use Vector<byte>.Dot
1 parent dc9f5c4 commit 4460cd3

File tree

2 files changed

+163
-91
lines changed

2 files changed

+163
-91
lines changed

‎src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2.cs

Lines changed: 99 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -3,25 +3,12 @@
33

44
using System;
55
using System.Diagnostics;
6-
using System.Linq;
76
using System.Numerics;
87

98
namespace Microsoft.AspNet.Server.Kestrel.Infrastructure
109
{
1110
public struct MemoryPoolIterator2
1211
{
13-
/// <summary>
14-
/// Array of "minus one" bytes of the length of SIMD operations on the current hardware. Used as an argument in the
15-
/// vector dot product that counts matching character occurrence.
16-
/// </summary>
17-
private static Vector<byte> _dotCount = new Vector<byte>(Byte.MaxValue);
18-
19-
/// <summary>
20-
/// Array of negative numbers starting at 0 and continuing for the length of SIMD operations on the current hardware.
21-
/// Used as an argument in the vector dot product that determines matching character index.
22-
/// </summary>
23-
private static Vector<byte> _dotIndex = new Vector<byte>(Enumerable.Range(0, Vector<byte>.Count).Select(x => (byte)-x).ToArray());
24-
2512
private MemoryPoolBlock2 _block;
2613
private int _index;
2714

@@ -146,7 +133,6 @@ public int Seek(int char0)
146133
}
147134

148135
var byte0 = (byte)char0;
149-
var vectorStride = Vector<byte>.Count;
150136
var ch0Vector = new Vector<byte>(byte0);
151137

152138
var block = _block;
@@ -169,27 +155,20 @@ public int Seek(int char0)
169155
while (block.End != index)
170156
{
171157
var following = block.End - index;
172-
if (following >= vectorStride)
158+
if (following >= Vector<byte>.Count)
173159
{
174160
var data = new Vector<byte>(array, index);
175161
var ch0Equals = Vector.Equals(data, ch0Vector);
176-
var ch0Count = Vector.Dot(ch0Equals, _dotCount);
177162

178-
if (ch0Count == 0)
163+
if (ch0Equals.Equals(Vector<byte>.Zero))
179164
{
180-
index += vectorStride;
165+
index += Vector<byte>.Count;
181166
continue;
182167
}
183-
else if (ch0Count == 1)
184-
{
185-
_block = block;
186-
_index = index + Vector.Dot(ch0Equals, _dotIndex);
187-
return char0;
188-
}
189-
else
190-
{
191-
following = vectorStride;
192-
}
168+
169+
_block = block;
170+
_index = index + FindFirstEqualByte(ch0Equals);
171+
return char0;
193172
}
194173
while (following > 0)
195174
{
@@ -215,7 +194,6 @@ public int Seek(int char0, int char1)
215194

216195
var byte0 = (byte)char0;
217196
var byte1 = (byte)char1;
218-
var vectorStride = Vector<byte>.Count;
219197
var ch0Vector = new Vector<byte>(byte0);
220198
var ch1Vector = new Vector<byte>(byte1);
221199

@@ -239,40 +217,39 @@ public int Seek(int char0, int char1)
239217
while (block.End != index)
240218
{
241219
var following = block.End - index;
242-
if (following >= vectorStride)
220+
if (following >= Vector<byte>.Count)
243221
{
244222
var data = new Vector<byte>(array, index);
245223
var ch0Equals = Vector.Equals(data, ch0Vector);
246-
var ch0Count = Vector.Dot(ch0Equals, _dotCount);
247224
var ch1Equals = Vector.Equals(data, ch1Vector);
248-
var ch1Count = Vector.Dot(ch1Equals, _dotCount);
225+
int ch0Index = int.MaxValue;
226+
int ch1Index = int.MaxValue;
249227

250-
if (ch0Count == 0 && ch1Count == 0)
228+
if (!ch0Equals.Equals(Vector<byte>.Zero))
251229
{
252-
index += vectorStride;
253-
continue;
230+
ch0Index = FindFirstEqualByte(ch0Equals);
254231
}
255-
else if (ch0Count < 2 && ch1Count < 2)
232+
if (!ch1Equals.Equals(Vector<byte>.Zero))
256233
{
257-
var ch0Index = ch0Count == 1 ? Vector.Dot(ch0Equals, _dotIndex) : byte.MaxValue;
258-
var ch1Index = ch1Count == 1 ? Vector.Dot(ch1Equals, _dotIndex) : byte.MaxValue;
259-
if (ch0Index < ch1Index)
260-
{
261-
_block = block;
262-
_index = index + ch0Index;
263-
return char0;
264-
}
265-
else
266-
{
267-
_block = block;
268-
_index = index + ch1Index;
269-
return char1;
270-
}
234+
ch1Index = FindFirstEqualByte(ch1Equals);
271235
}
272-
else
236+
237+
if (ch0Index == int.MaxValue && ch1Index == int.MaxValue)
238+
{
239+
index += Vector<byte>.Count;
240+
continue;
241+
}
242+
243+
_block = block;
244+
245+
if (ch0Index < ch1Index)
273246
{
274-
following = vectorStride;
247+
_index = index + ch0Index;
248+
return char0;
275249
}
250+
251+
_index = index + ch1Index;
252+
return char1;
276253
}
277254
while (following > 0)
278255
{
@@ -306,7 +283,6 @@ public int Seek(int char0, int char1, int char2)
306283
var byte0 = (byte)char0;
307284
var byte1 = (byte)char1;
308285
var byte2 = (byte)char2;
309-
var vectorStride = Vector<byte>.Count;
310286
var ch0Vector = new Vector<byte>(byte0);
311287
var ch1Vector = new Vector<byte>(byte1);
312288
var ch2Vector = new Vector<byte>(byte2);
@@ -331,64 +307,68 @@ public int Seek(int char0, int char1, int char2)
331307
while (block.End != index)
332308
{
333309
var following = block.End - index;
334-
if (following >= vectorStride)
310+
if (following >= Vector<byte>.Count)
335311
{
336312
var data = new Vector<byte>(array, index);
337313
var ch0Equals = Vector.Equals(data, ch0Vector);
338-
var ch0Count = Vector.Dot(ch0Equals, _dotCount);
339314
var ch1Equals = Vector.Equals(data, ch1Vector);
340-
var ch1Count = Vector.Dot(ch1Equals, _dotCount);
341315
var ch2Equals = Vector.Equals(data, ch2Vector);
342-
var ch2Count = Vector.Dot(ch2Equals, _dotCount);
316+
int ch0Index = int.MaxValue;
317+
int ch1Index = int.MaxValue;
318+
int ch2Index = int.MaxValue;
343319

344-
if (ch0Count == 0 && ch1Count == 0 && ch2Count == 0)
320+
if (!ch0Equals.Equals(Vector<byte>.Zero))
345321
{
346-
index += vectorStride;
347-
continue;
322+
ch0Index = FindFirstEqualByte(ch0Equals);
323+
}
324+
if (!ch1Equals.Equals(Vector<byte>.Zero))
325+
{
326+
ch1Index = FindFirstEqualByte(ch1Equals);
348327
}
349-
else if (ch0Count < 2 && ch1Count < 2 && ch2Count < 2)
328+
if (!ch2Equals.Equals(Vector<byte>.Zero))
350329
{
351-
var ch0Index = ch0Count == 1 ? Vector.Dot(ch0Equals, _dotIndex) : byte.MaxValue;
352-
var ch1Index = ch1Count == 1 ? Vector.Dot(ch1Equals, _dotIndex) : byte.MaxValue;
353-
var ch2Index = ch2Count == 1 ? Vector.Dot(ch2Equals, _dotIndex) : byte.MaxValue;
330+
ch2Index = FindFirstEqualByte(ch2Equals);
331+
}
354332

355-
int toReturn, toMove;
356-
if (ch0Index < ch1Index)
333+
if (ch0Index == int.MaxValue && ch1Index == int.MaxValue && ch2Index == int.MaxValue)
334+
{
335+
index += Vector<byte>.Count;
336+
continue;
337+
}
338+
339+
int toReturn, toMove;
340+
if (ch0Index < ch1Index)
341+
{
342+
if (ch0Index < ch2Index)
357343
{
358-
if (ch0Index < ch2Index)
359-
{
360-
toReturn = char0;
361-
toMove = ch0Index;
362-
}
363-
else
364-
{
365-
toReturn = char2;
366-
toMove = ch2Index;
367-
}
344+
toReturn = char0;
345+
toMove = ch0Index;
368346
}
369347
else
370348
{
371-
if (ch1Index < ch2Index)
372-
{
373-
toReturn = char1;
374-
toMove = ch1Index;
375-
}
376-
else
377-
{
378-
toReturn = char2;
379-
toMove = ch2Index;
380-
}
349+
toReturn = char2;
350+
toMove = ch2Index;
381351
}
382-
383-
_block = block;
384-
_index = index + toMove;
385-
return toReturn;
386352
}
387353
else
388354
{
389-
following = vectorStride;
355+
if (ch1Index < ch2Index)
356+
{
357+
toReturn = char1;
358+
toMove = ch1Index;
359+
}
360+
else
361+
{
362+
toReturn = char2;
363+
toMove = ch2Index;
364+
}
390365
}
366+
367+
_block = block;
368+
_index = index + toMove;
369+
return toReturn;
391370
}
371+
392372
while (following > 0)
393373
{
394374
var byteIndex = block.Array[index];
@@ -417,6 +397,34 @@ public int Seek(int char0, int char1, int char2)
417397
}
418398
}
419399

400+
private static int FindFirstEqualByte(Vector<byte> chEquals)
401+
{
402+
// Quasi-tree search
403+
var vector64 = Vector.AsVectorInt64(chEquals);
404+
for (var i = 0; i < Vector<long>.Count; i++)
405+
{
406+
var longValue = vector64[i];
407+
if (longValue == 0) continue;
408+
409+
var shift = i << 1;
410+
var offset = shift << 2;
411+
var vector32 = Vector.AsVectorInt32(chEquals);
412+
if (vector32[shift] != 0)
413+
{
414+
if (chEquals[offset] != 0) return offset;
415+
if (chEquals[++offset] != 0) return offset;
416+
if (chEquals[++offset] != 0) return offset;
417+
return ++offset;
418+
}
419+
offset += 4;
420+
if (chEquals[offset] != 0) return offset;
421+
if (chEquals[++offset] != 0) return offset;
422+
if (chEquals[++offset] != 0) return offset;
423+
return ++offset;
424+
}
425+
throw new InvalidOperationException();
426+
}
427+
420428
/// <summary>
421429
/// Save the data at the current location then move to the next available space.
422430
/// </summary>

‎test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolBlock2Tests.cs

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,70 @@ public void SeekWorks()
3131
hit = iterator;
3232
hit.Seek(byte.MaxValue, ch);
3333
Assert.Equal(ch, iterator.GetLength(hit));
34+
35+
hit = iterator;
36+
hit.Seek(ch, byte.MaxValue, byte.MaxValue);
37+
Assert.Equal(ch, iterator.GetLength(hit));
38+
39+
hit = iterator;
40+
hit.Seek(byte.MaxValue, ch, byte.MaxValue);
41+
Assert.Equal(ch, iterator.GetLength(hit));
42+
43+
hit = iterator;
44+
hit.Seek(ch, byte.MaxValue, byte.MaxValue);
45+
Assert.Equal(ch, iterator.GetLength(hit));
46+
}
47+
}
48+
}
49+
50+
[Fact]
51+
public void SeekWorksAcrossBlocks()
52+
{
53+
using (var pool = new MemoryPool2())
54+
{
55+
var block1 = pool.Lease(256);
56+
var block2 = block1.Next = pool.Lease(256);
57+
var block3 = block2.Next = pool.Lease(256);
58+
59+
foreach (var ch in Enumerable.Range(0, 34).Select(x => (byte)x))
60+
{
61+
block1.Array[block1.End++] = ch;
62+
}
63+
foreach (var ch in Enumerable.Range(34, 25).Select(x => (byte)x))
64+
{
65+
block2.Array[block2.End++] = ch;
66+
}
67+
foreach (var ch in Enumerable.Range(59, 197).Select(x => (byte)x))
68+
{
69+
block3.Array[block3.End++] = ch;
70+
}
71+
72+
var iterator = block1.GetIterator();
73+
foreach (var ch in Enumerable.Range(0, 256).Select(x => (char)x))
74+
{
75+
var hit = iterator;
76+
hit.Seek(ch);
77+
Assert.Equal(ch, iterator.GetLength(hit));
78+
79+
hit = iterator;
80+
hit.Seek(ch, byte.MaxValue);
81+
Assert.Equal(ch, iterator.GetLength(hit));
82+
83+
hit = iterator;
84+
hit.Seek(byte.MaxValue, ch);
85+
Assert.Equal(ch, iterator.GetLength(hit));
86+
87+
hit = iterator;
88+
hit.Seek(ch, byte.MaxValue, byte.MaxValue);
89+
Assert.Equal(ch, iterator.GetLength(hit));
90+
91+
hit = iterator;
92+
hit.Seek(byte.MaxValue, ch, byte.MaxValue);
93+
Assert.Equal(ch, iterator.GetLength(hit));
94+
95+
hit = iterator;
96+
hit.Seek(ch, byte.MaxValue, byte.MaxValue);
97+
Assert.Equal(ch, iterator.GetLength(hit));
3498
}
3599
}
36100
}

0 commit comments

Comments
 (0)