Skip to content
This repository was archived by the owner on Dec 18, 2018. It is now read-only.

Commit 5d6fdc5

Browse files
halter73halter73
committed
Merge branch 'benaadams/vector.dot' into dev
2 parents 67a0165 + 4460cd3 commit 5d6fdc5

File tree

2 files changed

+163
-91
lines changed

2 files changed

+163
-91
lines changed

‎src/Microsoft.AspNet.Server.Kestrel/Infrastructure/MemoryPoolIterator2.cs

Lines changed: 99 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -3,25 +3,12 @@
33

44
using System;
55
using System.Diagnostics;
6-
using System.Linq;
76
using System.Numerics;
87

98
namespace Microsoft.AspNet.Server.Kestrel.Infrastructure
109
{
1110
public struct MemoryPoolIterator2
1211
{
13-
/// <summary>
14-
/// Array of "minus one" bytes of the length of SIMD operations on the current hardware. Used as an argument in the
15-
/// vector dot product that counts matching character occurrence.
16-
/// </summary>
17-
private static Vector<byte> _dotCount = new Vector<byte>(Byte.MaxValue);
18-
19-
/// <summary>
20-
/// Array of negative numbers starting at 0 and continuing for the length of SIMD operations on the current hardware.
21-
/// Used as an argument in the vector dot product that determines matching character index.
22-
/// </summary>
23-
private static Vector<byte> _dotIndex = new Vector<byte>(Enumerable.Range(0, Vector<byte>.Count).Select(x => (byte)-x).ToArray());
24-
2512
private MemoryPoolBlock2 _block;
2613
private int _index;
2714

@@ -189,7 +176,6 @@ public int Seek(int char0)
189176
}
190177

191178
var byte0 = (byte)char0;
192-
var vectorStride = Vector<byte>.Count;
193179
var ch0Vector = new Vector<byte>(byte0);
194180

195181
var block = _block;
@@ -212,27 +198,20 @@ public int Seek(int char0)
212198
while (block.End != index)
213199
{
214200
var following = block.End - index;
215-
if (following >= vectorStride)
201+
if (following >= Vector<byte>.Count)
216202
{
217203
var data = new Vector<byte>(array, index);
218204
var ch0Equals = Vector.Equals(data, ch0Vector);
219-
var ch0Count = Vector.Dot(ch0Equals, _dotCount);
220205

221-
if (ch0Count == 0)
206+
if (ch0Equals.Equals(Vector<byte>.Zero))
222207
{
223-
index += vectorStride;
208+
index += Vector<byte>.Count;
224209
continue;
225210
}
226-
else if (ch0Count == 1)
227-
{
228-
_block = block;
229-
_index = index + Vector.Dot(ch0Equals, _dotIndex);
230-
return char0;
231-
}
232-
else
233-
{
234-
following = vectorStride;
235-
}
211+
212+
_block = block;
213+
_index = index + FindFirstEqualByte(ch0Equals);
214+
return char0;
236215
}
237216
while (following > 0)
238217
{
@@ -258,7 +237,6 @@ public int Seek(int char0, int char1)
258237

259238
var byte0 = (byte)char0;
260239
var byte1 = (byte)char1;
261-
var vectorStride = Vector<byte>.Count;
262240
var ch0Vector = new Vector<byte>(byte0);
263241
var ch1Vector = new Vector<byte>(byte1);
264242

@@ -282,40 +260,39 @@ public int Seek(int char0, int char1)
282260
while (block.End != index)
283261
{
284262
var following = block.End - index;
285-
if (following >= vectorStride)
263+
if (following >= Vector<byte>.Count)
286264
{
287265
var data = new Vector<byte>(array, index);
288266
var ch0Equals = Vector.Equals(data, ch0Vector);
289-
var ch0Count = Vector.Dot(ch0Equals, _dotCount);
290267
var ch1Equals = Vector.Equals(data, ch1Vector);
291-
var ch1Count = Vector.Dot(ch1Equals, _dotCount);
268+
int ch0Index = int.MaxValue;
269+
int ch1Index = int.MaxValue;
292270

293-
if (ch0Count == 0 && ch1Count == 0)
271+
if (!ch0Equals.Equals(Vector<byte>.Zero))
294272
{
295-
index += vectorStride;
296-
continue;
273+
ch0Index = FindFirstEqualByte(ch0Equals);
297274
}
298-
else if (ch0Count < 2 && ch1Count < 2)
275+
if (!ch1Equals.Equals(Vector<byte>.Zero))
299276
{
300-
var ch0Index = ch0Count == 1 ? Vector.Dot(ch0Equals, _dotIndex) : byte.MaxValue;
301-
var ch1Index = ch1Count == 1 ? Vector.Dot(ch1Equals, _dotIndex) : byte.MaxValue;
302-
if (ch0Index < ch1Index)
303-
{
304-
_block = block;
305-
_index = index + ch0Index;
306-
return char0;
307-
}
308-
else
309-
{
310-
_block = block;
311-
_index = index + ch1Index;
312-
return char1;
313-
}
277+
ch1Index = FindFirstEqualByte(ch1Equals);
314278
}
315-
else
279+
280+
if (ch0Index == int.MaxValue && ch1Index == int.MaxValue)
281+
{
282+
index += Vector<byte>.Count;
283+
continue;
284+
}
285+
286+
_block = block;
287+
288+
if (ch0Index < ch1Index)
316289
{
317-
following = vectorStride;
290+
_index = index + ch0Index;
291+
return char0;
318292
}
293+
294+
_index = index + ch1Index;
295+
return char1;
319296
}
320297
while (following > 0)
321298
{
@@ -349,7 +326,6 @@ public int Seek(int char0, int char1, int char2)
349326
var byte0 = (byte)char0;
350327
var byte1 = (byte)char1;
351328
var byte2 = (byte)char2;
352-
var vectorStride = Vector<byte>.Count;
353329
var ch0Vector = new Vector<byte>(byte0);
354330
var ch1Vector = new Vector<byte>(byte1);
355331
var ch2Vector = new Vector<byte>(byte2);
@@ -374,64 +350,68 @@ public int Seek(int char0, int char1, int char2)
374350
while (block.End != index)
375351
{
376352
var following = block.End - index;
377-
if (following >= vectorStride)
353+
if (following >= Vector<byte>.Count)
378354
{
379355
var data = new Vector<byte>(array, index);
380356
var ch0Equals = Vector.Equals(data, ch0Vector);
381-
var ch0Count = Vector.Dot(ch0Equals, _dotCount);
382357
var ch1Equals = Vector.Equals(data, ch1Vector);
383-
var ch1Count = Vector.Dot(ch1Equals, _dotCount);
384358
var ch2Equals = Vector.Equals(data, ch2Vector);
385-
var ch2Count = Vector.Dot(ch2Equals, _dotCount);
359+
int ch0Index = int.MaxValue;
360+
int ch1Index = int.MaxValue;
361+
int ch2Index = int.MaxValue;
386362

387-
if (ch0Count == 0 && ch1Count == 0 && ch2Count == 0)
363+
if (!ch0Equals.Equals(Vector<byte>.Zero))
388364
{
389-
index += vectorStride;
390-
continue;
365+
ch0Index = FindFirstEqualByte(ch0Equals);
366+
}
367+
if (!ch1Equals.Equals(Vector<byte>.Zero))
368+
{
369+
ch1Index = FindFirstEqualByte(ch1Equals);
391370
}
392-
else if (ch0Count < 2 && ch1Count < 2 && ch2Count < 2)
371+
if (!ch2Equals.Equals(Vector<byte>.Zero))
393372
{
394-
var ch0Index = ch0Count == 1 ? Vector.Dot(ch0Equals, _dotIndex) : byte.MaxValue;
395-
var ch1Index = ch1Count == 1 ? Vector.Dot(ch1Equals, _dotIndex) : byte.MaxValue;
396-
var ch2Index = ch2Count == 1 ? Vector.Dot(ch2Equals, _dotIndex) : byte.MaxValue;
373+
ch2Index = FindFirstEqualByte(ch2Equals);
374+
}
397375

398-
int toReturn, toMove;
399-
if (ch0Index < ch1Index)
376+
if (ch0Index == int.MaxValue && ch1Index == int.MaxValue && ch2Index == int.MaxValue)
377+
{
378+
index += Vector<byte>.Count;
379+
continue;
380+
}
381+
382+
int toReturn, toMove;
383+
if (ch0Index < ch1Index)
384+
{
385+
if (ch0Index < ch2Index)
400386
{
401-
if (ch0Index < ch2Index)
402-
{
403-
toReturn = char0;
404-
toMove = ch0Index;
405-
}
406-
else
407-
{
408-
toReturn = char2;
409-
toMove = ch2Index;
410-
}
387+
toReturn = char0;
388+
toMove = ch0Index;
411389
}
412390
else
413391
{
414-
if (ch1Index < ch2Index)
415-
{
416-
toReturn = char1;
417-
toMove = ch1Index;
418-
}
419-
else
420-
{
421-
toReturn = char2;
422-
toMove = ch2Index;
423-
}
392+
toReturn = char2;
393+
toMove = ch2Index;
424394
}
425-
426-
_block = block;
427-
_index = index + toMove;
428-
return toReturn;
429395
}
430396
else
431397
{
432-
following = vectorStride;
398+
if (ch1Index < ch2Index)
399+
{
400+
toReturn = char1;
401+
toMove = ch1Index;
402+
}
403+
else
404+
{
405+
toReturn = char2;
406+
toMove = ch2Index;
407+
}
433408
}
409+
410+
_block = block;
411+
_index = index + toMove;
412+
return toReturn;
434413
}
414+
435415
while (following > 0)
436416
{
437417
var byteIndex = block.Array[index];
@@ -460,6 +440,34 @@ public int Seek(int char0, int char1, int char2)
460440
}
461441
}
462442

443+
private static int FindFirstEqualByte(Vector<byte> chEquals)
444+
{
445+
// Quasi-tree search
446+
var vector64 = Vector.AsVectorInt64(chEquals);
447+
for (var i = 0; i < Vector<long>.Count; i++)
448+
{
449+
var longValue = vector64[i];
450+
if (longValue == 0) continue;
451+
452+
var shift = i << 1;
453+
var offset = shift << 2;
454+
var vector32 = Vector.AsVectorInt32(chEquals);
455+
if (vector32[shift] != 0)
456+
{
457+
if (chEquals[offset] != 0) return offset;
458+
if (chEquals[++offset] != 0) return offset;
459+
if (chEquals[++offset] != 0) return offset;
460+
return ++offset;
461+
}
462+
offset += 4;
463+
if (chEquals[offset] != 0) return offset;
464+
if (chEquals[++offset] != 0) return offset;
465+
if (chEquals[++offset] != 0) return offset;
466+
return ++offset;
467+
}
468+
throw new InvalidOperationException();
469+
}
470+
463471
/// <summary>
464472
/// Save the data at the current location then move to the next available space.
465473
/// </summary>

‎test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolBlock2Tests.cs

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,70 @@ public void SeekWorks()
3131
hit = iterator;
3232
hit.Seek(byte.MaxValue, ch);
3333
Assert.Equal(ch, iterator.GetLength(hit));
34+
35+
hit = iterator;
36+
hit.Seek(ch, byte.MaxValue, byte.MaxValue);
37+
Assert.Equal(ch, iterator.GetLength(hit));
38+
39+
hit = iterator;
40+
hit.Seek(byte.MaxValue, ch, byte.MaxValue);
41+
Assert.Equal(ch, iterator.GetLength(hit));
42+
43+
hit = iterator;
44+
hit.Seek(ch, byte.MaxValue, byte.MaxValue);
45+
Assert.Equal(ch, iterator.GetLength(hit));
46+
}
47+
}
48+
}
49+
50+
[Fact]
51+
public void SeekWorksAcrossBlocks()
52+
{
53+
using (var pool = new MemoryPool2())
54+
{
55+
var block1 = pool.Lease(256);
56+
var block2 = block1.Next = pool.Lease(256);
57+
var block3 = block2.Next = pool.Lease(256);
58+
59+
foreach (var ch in Enumerable.Range(0, 34).Select(x => (byte)x))
60+
{
61+
block1.Array[block1.End++] = ch;
62+
}
63+
foreach (var ch in Enumerable.Range(34, 25).Select(x => (byte)x))
64+
{
65+
block2.Array[block2.End++] = ch;
66+
}
67+
foreach (var ch in Enumerable.Range(59, 197).Select(x => (byte)x))
68+
{
69+
block3.Array[block3.End++] = ch;
70+
}
71+
72+
var iterator = block1.GetIterator();
73+
foreach (var ch in Enumerable.Range(0, 256).Select(x => (char)x))
74+
{
75+
var hit = iterator;
76+
hit.Seek(ch);
77+
Assert.Equal(ch, iterator.GetLength(hit));
78+
79+
hit = iterator;
80+
hit.Seek(ch, byte.MaxValue);
81+
Assert.Equal(ch, iterator.GetLength(hit));
82+
83+
hit = iterator;
84+
hit.Seek(byte.MaxValue, ch);
85+
Assert.Equal(ch, iterator.GetLength(hit));
86+
87+
hit = iterator;
88+
hit.Seek(ch, byte.MaxValue, byte.MaxValue);
89+
Assert.Equal(ch, iterator.GetLength(hit));
90+
91+
hit = iterator;
92+
hit.Seek(byte.MaxValue, ch, byte.MaxValue);
93+
Assert.Equal(ch, iterator.GetLength(hit));
94+
95+
hit = iterator;
96+
hit.Seek(ch, byte.MaxValue, byte.MaxValue);
97+
Assert.Equal(ch, iterator.GetLength(hit));
3498
}
3599
}
36100
}

0 commit comments

Comments
 (0)