From 9d01665afcde3a70a3be8b622f841af264c457e9 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Thu, 11 Aug 2022 12:32:38 +0200
Subject: [PATCH 01/34] Vectorize LastIndexOf

---
 .../src/System/SpanHelpers.Byte.cs            | 31 +-----
 .../src/System/SpanHelpers.T.cs               | 94 +++++++++++++++++++
 2 files changed, 97 insertions(+), 28 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
index 5029e01a3161d7..c04b483a7e4a65 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
@@ -675,11 +675,11 @@ public static int LastIndexOf(ref byte searchSpace, byte value, int length)
             nuint offset = (nuint)(uint)length; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations
             nuint lengthToExamine = (nuint)(uint)length;
 
-            if (Vector.IsHardwareAccelerated && length >= Vector<byte>.Count * 2)
+            if (Vector128.IsHardwareAccelerated && length >= Vector128<byte>.Count)
             {
-                lengthToExamine = UnalignedCountVectorFromEnd(ref searchSpace, length);
+                return LastIndexOfValueType(ref searchSpace, value, length);
             }
-        SequentialScan:
+
             while (lengthToExamine >= 8)
             {
                 lengthToExamine -= 8;
@@ -727,31 +727,6 @@ public static int LastIndexOf(ref byte searchSpace, byte value, int length)
                     goto Found;
             }
 
-            if (Vector.IsHardwareAccelerated && (offset > 0))
-            {
-                lengthToExamine = (offset & (nuint)~(Vector<byte>.Count - 1));
-
-                Vector<byte> values = new Vector<byte>(value);
-
-                while (lengthToExamine > (nuint)(Vector<byte>.Count - 1))
-                {
-                    var matches = Vector.Equals(values, LoadVector(ref searchSpace, offset - (nuint)Vector<byte>.Count));
-                    if (Vector<byte>.Zero.Equals(matches))
-                    {
-                        offset -= (nuint)Vector<byte>.Count;
-                        lengthToExamine -= (nuint)Vector<byte>.Count;
-                        continue;
-                    }
-
-                    // Find offset of first match and add to current offset
-                    return (int)(offset) - Vector<byte>.Count + LocateLastFoundByte(matches);
-                }
-                if (offset > 0)
-                {
-                    lengthToExamine = offset;
-                    goto SequentialScan;
-                }
-            }
             return -1;
         Found: // Workaround for https://github.com/dotnet/runtime/issues/8795
             return (int)offset;
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
index ed26ed9d04958b..b8a9db0756bde4 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
@@ -1342,5 +1342,99 @@ public static int SequenceCompareTo<T>(ref T first, int firstLength, ref T secon
             }
             return firstLength.CompareTo(secondLength);
         }
+
+        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
+        internal static int LastIndexOfValueType<T>(ref T searchSpace, T value, int length) where T : struct, IEquatable<T>
+        {
+            Debug.Assert(length >= 0, "Expected non-negative length");
+            Debug.Assert(value is byte or short or int or long, "Expected caller to normalize to one of these types");
+
+            if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
+            {
+                for (int i = length - 1; i >= 0; i--)
+                {
+                    if (Unsafe.Add(ref searchSpace, i).Equals(value))
+                    {
+                        return i;
+                    }
+                }
+            }
+            else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
+            {
+                Vector256<T> equals, values = Vector256.Create(value);
+                ref T currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256<T>.Count);
+
+                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
+                do
+                {
+                    equals = Vector256.Equals(values, Vector256.LoadUnsafe(ref currentSearchSpace));
+                    if (equals == Vector256<T>.Zero)
+                    {
+                        currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<T>.Count);
+                        continue;
+                    }
+
+                    return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals);
+                }
+                while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace));
+
+                // If any elements remain, process the first vector in the search space.
+                if ((uint)length % Vector256<T>.Count != 0)
+                {
+                    equals = Vector256.Equals(values, Vector256.LoadUnsafe(ref searchSpace));
+                    if (equals != Vector256<T>.Zero)
+                    {
+                        return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
+                    }
+                }
+            }
+            else
+            {
+                Vector128<T> equals, values = Vector128.Create(value);
+                ref T currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128<T>.Count);
+
+                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
+                do
+                {
+                    equals = Vector128.Equals(values, Vector128.LoadUnsafe(ref currentSearchSpace));
+                    if (equals == Vector128<T>.Zero)
+                    {
+                        currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<T>.Count);
+                        continue;
+                    }
+
+                    return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals);
+                }
+                while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace));
+
+                // If any elements remain, process the first vector in the search space.
+                if ((uint)length % Vector128<T>.Count != 0)
+                {
+                    equals = Vector128.Equals(values, Vector128.LoadUnsafe(ref searchSpace));
+                    if (equals != Vector128<T>.Zero)
+                    {
+                        return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
+                    }
+                }
+            }
+
+            return -1;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static int ComputeLastIndex<T>(ref T searchSpace, ref T current, Vector128<T> equals) where T : struct, IEquatable<T>
+        {
+            uint notEqualsElements = equals.ExtractMostSignificantBits();
+            int index = 31 - BitOperations.LeadingZeroCount(notEqualsElements); // 31 = 32 (bits in Int32) - 1 (indexing from zero)
+            return (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / Unsafe.SizeOf<T>()) + index;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static int ComputeLastIndex<T>(ref T searchSpace, ref T current, Vector256<T> equals) where T : struct, IEquatable<T>
+        {
+            uint notEqualsElements = equals.ExtractMostSignificantBits();
+            int index = 31 - BitOperations.LeadingZeroCount(notEqualsElements); // 31 = 32 (bits in Int32) - 1 (indexing from zero)
+            return (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / Unsafe.SizeOf<T>()) + index;
+        }
     }
 }

From 14224a3c59d71c9caed39a617b1bb90ad23a77dd Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Thu, 11 Aug 2022 13:48:10 +0200
Subject: [PATCH 02/34] use structs to get performant code without code
 duplication

---
 .../src/System/MemoryExtensions.cs            | 32 ++++++++++++++++
 .../src/System/SpanHelpers.Byte.cs            |  2 +-
 .../src/System/SpanHelpers.T.cs               | 37 ++++++++++++++++---
 3 files changed, 64 insertions(+), 7 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
index ce3f24f7d6513d..07e7a67d99cf28 100644
--- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
@@ -689,6 +689,38 @@ public static int LastIndexOfAnyExcept<T>(this Span<T> span, ReadOnlySpan<T> val
         /// </returns>
         public static int LastIndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value) where T : IEquatable<T>?
         {
+            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            {
+                if (Unsafe.SizeOf<T>() == sizeof(byte))
+                {
+                    return SpanHelpers.LastIndexOfValueType<byte, SpanHelpers.ExceptEqualityComparer<byte>>(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, byte>(ref value),
+                        span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
+                {
+                    return SpanHelpers.LastIndexOfValueType<short, SpanHelpers.ExceptEqualityComparer<short>>(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value),
+                        span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(int))
+                {
+                    return SpanHelpers.LastIndexOfValueType<int, SpanHelpers.ExceptEqualityComparer<int>>(
+                        ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, int>(ref value),
+                        span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(long))
+                {
+                    return SpanHelpers.LastIndexOfValueType<long, SpanHelpers.ExceptEqualityComparer<long>>(
+                        ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, long>(ref value),
+                        span.Length);
+                }
+            }
+
             for (int i = span.Length - 1; i >= 0; i--)
             {
                 if (!EqualityComparer<T>.Default.Equals(span[i], value))
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
index c04b483a7e4a65..5ac2ad0cc6be87 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
@@ -677,7 +677,7 @@ public static int LastIndexOf(ref byte searchSpace, byte value, int length)
 
             if (Vector128.IsHardwareAccelerated && length >= Vector128<byte>.Count)
             {
-                return LastIndexOfValueType(ref searchSpace, value, length);
+                return LastIndexOfValueType<byte, DefaultEqualityComparer<byte>>(ref searchSpace, value, length);
             }
 
             while (lengthToExamine >= 8)
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
index b8a9db0756bde4..d9301b0a336e13 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
@@ -1344,16 +1344,20 @@ public static int SequenceCompareTo<T>(ref T first, int firstLength, ref T secon
         }
 
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        internal static int LastIndexOfValueType<T>(ref T searchSpace, T value, int length) where T : struct, IEquatable<T>
+        internal static int LastIndexOfValueType<T, C>(ref T searchSpace, T value, int length)
+            where T : struct, IEquatable<T>
+            where C : struct, IVectorEqualityComparer<T>
         {
             Debug.Assert(length >= 0, "Expected non-negative length");
             Debug.Assert(value is byte or short or int or long, "Expected caller to normalize to one of these types");
 
+            C comparer = default;
+
             if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
             {
                 for (int i = length - 1; i >= 0; i--)
                 {
-                    if (Unsafe.Add(ref searchSpace, i).Equals(value))
+                    if (comparer.Equals(Unsafe.Add(ref searchSpace, i), value))
                     {
                         return i;
                     }
@@ -1367,7 +1371,7 @@ internal static int LastIndexOfValueType<T>(ref T searchSpace, T value, int leng
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
-                    equals = Vector256.Equals(values, Vector256.LoadUnsafe(ref currentSearchSpace));
+                    equals = comparer.Equals256(values, Vector256.LoadUnsafe(ref currentSearchSpace));
                     if (equals == Vector256<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<T>.Count);
@@ -1381,7 +1385,7 @@ internal static int LastIndexOfValueType<T>(ref T searchSpace, T value, int leng
                 // If any elements remain, process the first vector in the search space.
                 if ((uint)length % Vector256<T>.Count != 0)
                 {
-                    equals = Vector256.Equals(values, Vector256.LoadUnsafe(ref searchSpace));
+                    equals = comparer.Equals256(values, Vector256.LoadUnsafe(ref searchSpace));
                     if (equals != Vector256<T>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
@@ -1396,7 +1400,7 @@ internal static int LastIndexOfValueType<T>(ref T searchSpace, T value, int leng
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
-                    equals = Vector128.Equals(values, Vector128.LoadUnsafe(ref currentSearchSpace));
+                    equals = comparer.Equals128(values, Vector128.LoadUnsafe(ref currentSearchSpace));
                     if (equals == Vector128<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<T>.Count);
@@ -1410,7 +1414,7 @@ internal static int LastIndexOfValueType<T>(ref T searchSpace, T value, int leng
                 // If any elements remain, process the first vector in the search space.
                 if ((uint)length % Vector128<T>.Count != 0)
                 {
-                    equals = Vector128.Equals(values, Vector128.LoadUnsafe(ref searchSpace));
+                    equals = comparer.Equals128(values, Vector128.LoadUnsafe(ref searchSpace));
                     if (equals != Vector128<T>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
@@ -1436,5 +1440,26 @@ private static int ComputeLastIndex<T>(ref T searchSpace, ref T current, Vector2
             int index = 31 - BitOperations.LeadingZeroCount(notEqualsElements); // 31 = 32 (bits in Int32) - 1 (indexing from zero)
             return (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / Unsafe.SizeOf<T>()) + index;
         }
+
+        internal interface IVectorEqualityComparer<T> where T : struct, IEquatable<T>
+        {
+            bool Equals(T left, T right);
+            Vector128<T> Equals128(Vector128<T> left, Vector128<T> right);
+            Vector256<T> Equals256(Vector256<T> left, Vector256<T> right);
+        }
+
+        internal readonly struct DefaultEqualityComparer<T> : IVectorEqualityComparer<T> where T : struct, IEquatable<T>
+        {
+            public bool Equals(T left, T right) => left.Equals(right);
+            public Vector128<T> Equals128(Vector128<T> left, Vector128<T> right) => Vector128.Equals(left, right);
+            public Vector256<T> Equals256(Vector256<T> left, Vector256<T> right) => Vector256.Equals(left, right);
+        }
+
+        internal readonly struct ExceptEqualityComparer<T> : IVectorEqualityComparer<T> where T : struct, IEquatable<T>
+        {
+            public bool Equals(T left, T right) => !left.Equals(right);
+            public Vector128<T> Equals128(Vector128<T> left, Vector128<T> right) => ~Vector128.Equals(left, right);
+            public Vector256<T> Equals256(Vector256<T> left, Vector256<T> right) => ~Vector256.Equals(left, right);
+        }
     }
 }

From d05cf56d36c949209f3f767c787934429b3a6181 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Thu, 11 Aug 2022 15:36:16 +0200
Subject: [PATCH 03/34] use it in all possible places

---
 .../src/System/Array.cs                       | 14 +--
 .../src/System/MemoryExtensions.cs            | 48 +++++-----
 .../src/System/SpanHelpers.Byte.cs            | 84 +---------------
 .../src/System/SpanHelpers.Char.cs            | 96 +------------------
 .../src/System/String.Searching.cs            |  7 +-
 5 files changed, 42 insertions(+), 207 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/Array.cs b/src/libraries/System.Private.CoreLib/src/System/Array.cs
index 32435ec4367ec8..a7d2ff01e9d23d 100644
--- a/src/libraries/System.Private.CoreLib/src/System/Array.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/Array.cs
@@ -1586,19 +1586,19 @@ public static int LastIndexOf<T>(T[] array, T value, int startIndex, int count)
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
                     int endIndex = startIndex - count + 1;
-                    int result = SpanHelpers.LastIndexOf(
+                    int result = SpanHelpers.LastIndexOfValueType<byte, SpanHelpers.DefaultEqualityComparer<byte>>(
                         ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<byte[]>(array)), endIndex),
                         Unsafe.As<T, byte>(ref value),
                         count);
 
                     return (result >= 0 ? endIndex : 0) + result;
                 }
-                else if (Unsafe.SizeOf<T>() == sizeof(char))
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
                     int endIndex = startIndex - count + 1;
-                    int result = SpanHelpers.LastIndexOf(
-                        ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<char[]>(array)), endIndex),
-                        Unsafe.As<T, char>(ref value),
+                    int result = SpanHelpers.LastIndexOfValueType<short, SpanHelpers.DefaultEqualityComparer<short>>(
+                        ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<short[]>(array)), endIndex),
+                        Unsafe.As<T, short>(ref value),
                         count);
 
                     return (result >= 0 ? endIndex : 0) + result;
@@ -1606,7 +1606,7 @@ ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<char[]>(array))
                 else if (Unsafe.SizeOf<T>() == sizeof(int))
                 {
                     int endIndex = startIndex - count + 1;
-                    int result = SpanHelpers.LastIndexOf(
+                    int result = SpanHelpers.LastIndexOfValueType<int, SpanHelpers.DefaultEqualityComparer<int>>(
                         ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<int[]>(array)), endIndex),
                         Unsafe.As<T, int>(ref value),
                         count);
@@ -1616,7 +1616,7 @@ ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<int[]>(array)),
                 else if (Unsafe.SizeOf<T>() == sizeof(long))
                 {
                     int endIndex = startIndex - count + 1;
-                    int result = SpanHelpers.LastIndexOf(
+                    int result = SpanHelpers.LastIndexOfValueType<long, SpanHelpers.DefaultEqualityComparer<long>>(
                         ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<long[]>(array)), endIndex),
                         Unsafe.As<T, long>(ref value),
                         count);
diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
index 07e7a67d99cf28..3eec999bed4bba 100644
--- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
@@ -408,24 +408,7 @@ ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(value)),
         /// <param name="value">The value to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOf<T>(this Span<T> span, T value) where T : IEquatable<T>?
-        {
-            if (RuntimeHelpers.IsBitwiseEquatable<T>())
-            {
-                if (Unsafe.SizeOf<T>() == sizeof(byte))
-                    return SpanHelpers.LastIndexOf(
-                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, byte>(ref value),
-                        span.Length);
-
-                if (Unsafe.SizeOf<T>() == sizeof(char))
-                    return SpanHelpers.LastIndexOf(
-                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, char>(ref value),
-                        span.Length);
-            }
-
-            return SpanHelpers.LastIndexOf<T>(ref MemoryMarshal.GetReference(span), value, span.Length);
-        }
+            => LastIndexOf((ReadOnlySpan<T>)span, value);
 
         /// <summary>
         /// Searches for the specified sequence and returns the index of its last occurrence. If not found, returns -1. Values are compared using IEquatable{T}.Equals(T).
@@ -934,16 +917,33 @@ public static int LastIndexOf<T>(this ReadOnlySpan<T> span, T value) where T : I
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
-                    return SpanHelpers.LastIndexOf(
+                {
+                    return SpanHelpers.LastIndexOfValueType<byte, SpanHelpers.DefaultEqualityComparer<byte>>(
                             ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                             Unsafe.As<T, byte>(ref value),
                             span.Length);
-
-                if (Unsafe.SizeOf<T>() == sizeof(char))
-                    return SpanHelpers.LastIndexOf(
-                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, char>(ref value),
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
+                {
+                    return SpanHelpers.LastIndexOfValueType<short, SpanHelpers.DefaultEqualityComparer<short>>(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value),
                         span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(int))
+                {
+                    return SpanHelpers.LastIndexOfValueType<int, SpanHelpers.DefaultEqualityComparer<int>>(
+                        ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, int>(ref value),
+                        span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(long))
+                {
+                    return SpanHelpers.LastIndexOfValueType<long, SpanHelpers.DefaultEqualityComparer<long>>(
+                        ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, long>(ref value),
+                        span.Length);
+                }
             }
 
             return SpanHelpers.LastIndexOf<T>(ref MemoryMarshal.GetReference(span), value, span.Length);
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
index 5ac2ad0cc6be87..f267f195a00580 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
@@ -197,7 +197,7 @@ public static int LastIndexOf(ref byte searchSpace, int searchSpaceLength, ref b
 
             int valueTailLength = valueLength - 1;
             if (valueTailLength == 0)
-                return LastIndexOf(ref searchSpace, value, searchSpaceLength); // for single-byte values use plain LastIndexOf
+                return LastIndexOfValueType<byte, DefaultEqualityComparer<byte>>(ref searchSpace, value, searchSpaceLength); // for single-byte values use plain LastIndexOf
 
             int offset = 0;
             byte valueHead = value;
@@ -217,7 +217,7 @@ public static int LastIndexOf(ref byte searchSpace, int searchSpaceLength, ref b
                     break;  // The unsearched portion is now shorter than the sequence we're looking for. So it can't be there.
 
                 // Do a quick search for the first element of "value".
-                int relativeIndex = LastIndexOf(ref searchSpace, valueHead, remainingSearchSpaceLength);
+                int relativeIndex = LastIndexOfValueType<byte, DefaultEqualityComparer<byte>>(ref searchSpace, valueHead, remainingSearchSpaceLength);
                 if (relativeIndex < 0)
                     break;
 
@@ -666,86 +666,6 @@ public static unsafe int IndexOf(ref byte searchSpace, byte value, int length)
             return (int)(offset + 7);
         }
 
-        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        public static int LastIndexOf(ref byte searchSpace, byte value, int length)
-        {
-            Debug.Assert(length >= 0);
-
-            uint uValue = value; // Use uint for comparisons to avoid unnecessary 8->32 extensions
-            nuint offset = (nuint)(uint)length; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations
-            nuint lengthToExamine = (nuint)(uint)length;
-
-            if (Vector128.IsHardwareAccelerated && length >= Vector128<byte>.Count)
-            {
-                return LastIndexOfValueType<byte, DefaultEqualityComparer<byte>>(ref searchSpace, value, length);
-            }
-
-            while (lengthToExamine >= 8)
-            {
-                lengthToExamine -= 8;
-                offset -= 8;
-
-                if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 7))
-                    goto Found7;
-                if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 6))
-                    goto Found6;
-                if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 5))
-                    goto Found5;
-                if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 4))
-                    goto Found4;
-                if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 3))
-                    goto Found3;
-                if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 2))
-                    goto Found2;
-                if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 1))
-                    goto Found1;
-                if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset))
-                    goto Found;
-            }
-
-            if (lengthToExamine >= 4)
-            {
-                lengthToExamine -= 4;
-                offset -= 4;
-
-                if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 3))
-                    goto Found3;
-                if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 2))
-                    goto Found2;
-                if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset + 1))
-                    goto Found1;
-                if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset))
-                    goto Found;
-            }
-
-            while (lengthToExamine > 0)
-            {
-                lengthToExamine -= 1;
-                offset -= 1;
-
-                if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset))
-                    goto Found;
-            }
-
-            return -1;
-        Found: // Workaround for https://github.com/dotnet/runtime/issues/8795
-            return (int)offset;
-        Found1:
-            return (int)(offset + 1);
-        Found2:
-            return (int)(offset + 2);
-        Found3:
-            return (int)(offset + 3);
-        Found4:
-            return (int)(offset + 4);
-        Found5:
-            return (int)(offset + 5);
-        Found6:
-            return (int)(offset + 6);
-        Found7:
-            return (int)(offset + 7);
-        }
-
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
         public static int IndexOfAny(ref byte searchSpace, byte value0, byte value1, int length)
         {
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
index 8eedd2955eb55c..c2fd20a9742c6b 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
@@ -214,7 +214,8 @@ public static int LastIndexOf(ref char searchSpace, int searchSpaceLength, ref c
 
             int valueTailLength = valueLength - 1;
             if (valueTailLength == 0)
-                return LastIndexOf(ref searchSpace, value, searchSpaceLength); // for single-char values use plain LastIndexOf
+                return LastIndexOfValueType<short, DefaultEqualityComparer<short>>(
+                    ref Unsafe.As<char, short>(ref searchSpace), (short)value, searchSpaceLength); // for single-char values use plain LastIndexOf
 
             int offset = 0;
             char valueHead = value;
@@ -234,7 +235,8 @@ public static int LastIndexOf(ref char searchSpace, int searchSpaceLength, ref c
                     break;  // The unsearched portion is now shorter than the sequence we're looking for. So it can't be there.
 
                 // Do a quick search for the first element of "value".
-                int relativeIndex = LastIndexOf(ref searchSpace, valueHead, remainingSearchSpaceLength);
+                int relativeIndex = LastIndexOfValueType<short, DefaultEqualityComparer<short>>(
+                    ref Unsafe.As<char, short>(ref searchSpace), (short)valueHead, remainingSearchSpaceLength);
                 if (relativeIndex == -1)
                     break;
 
@@ -1761,96 +1763,6 @@ public static unsafe int IndexOfAny(ref char searchStart, char value0, char valu
             goto NotFound;
         }
 
-        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        public static unsafe int LastIndexOf(ref char searchSpace, char value, int length)
-        {
-            Debug.Assert(length >= 0);
-
-            fixed (char* pChars = &searchSpace)
-            {
-                char* pCh = pChars + length;
-                char* pEndCh = pChars;
-
-                if (Vector.IsHardwareAccelerated && length >= Vector<ushort>.Count * 2)
-                {
-                    // Figure out how many characters to read sequentially from the end until we are vector aligned
-                    // This is equivalent to: length = ((int)pCh % Unsafe.SizeOf<Vector<ushort>>()) / elementsPerByte
-                    const int elementsPerByte = sizeof(ushort) / sizeof(byte);
-                    length = ((int)pCh & (Unsafe.SizeOf<Vector<ushort>>() - 1)) / elementsPerByte;
-                }
-
-            SequentialScan:
-                while (length >= 4)
-                {
-                    length -= 4;
-                    pCh -= 4;
-
-                    if (*(pCh + 3) == value)
-                        goto Found3;
-                    if (*(pCh + 2) == value)
-                        goto Found2;
-                    if (*(pCh + 1) == value)
-                        goto Found1;
-                    if (*pCh == value)
-                        goto Found;
-                }
-
-                while (length > 0)
-                {
-                    length--;
-                    pCh--;
-
-                    if (*pCh == value)
-                        goto Found;
-                }
-
-                // We get past SequentialScan only if IsHardwareAccelerated is true. However, we still have the redundant check to allow
-                // the JIT to see that the code is unreachable and eliminate it when the platform does not have hardware accelerated.
-                if (Vector.IsHardwareAccelerated && pCh > pEndCh)
-                {
-                    // Get the highest multiple of Vector<ushort>.Count that is within the search space.
-                    // That will be how many times we iterate in the loop below.
-                    // This is equivalent to: length = Vector<ushort>.Count * ((int)(pCh - pEndCh) / Vector<ushort>.Count)
-                    length = (int)((pCh - pEndCh) & ~(Vector<ushort>.Count - 1));
-
-                    // Get comparison Vector
-                    Vector<ushort> vComparison = new Vector<ushort>(value);
-
-                    while (length > 0)
-                    {
-                        char* pStart = pCh - Vector<ushort>.Count;
-                        // Using Unsafe.Read instead of ReadUnaligned since the search space is pinned and pCh (and hence pSart) is always vector aligned
-                        Debug.Assert(((int)pStart & (Unsafe.SizeOf<Vector<ushort>>() - 1)) == 0);
-                        Vector<ushort> vMatches = Vector.Equals(vComparison, Unsafe.Read<Vector<ushort>>(pStart));
-                        if (Vector<ushort>.Zero.Equals(vMatches))
-                        {
-                            pCh -= Vector<ushort>.Count;
-                            length -= Vector<ushort>.Count;
-                            continue;
-                        }
-                        // Find offset of last match
-                        return (int)(pStart - pEndCh) + LocateLastFoundChar(vMatches);
-                    }
-
-                    if (pCh > pEndCh)
-                    {
-                        length = (int)(pCh - pEndCh);
-                        goto SequentialScan;
-                    }
-                }
-
-                return -1;
-            Found:
-                return (int)(pCh - pEndCh);
-            Found1:
-                return (int)(pCh - pEndCh) + 1;
-            Found2:
-                return (int)(pCh - pEndCh) + 2;
-            Found3:
-                return (int)(pCh - pEndCh) + 3;
-            }
-        }
-
         // Vector sub-search adapted from https://github.com/aspnet/KestrelHttpServer/pull/1138
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static int LocateFirstFoundChar(Vector<ushort> match)
diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs
index 945959f104a7e3..f9c06bc6b9a508 100644
--- a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs
@@ -281,7 +281,9 @@ public int IndexOf(string value, int startIndex, int count, StringComparison com
         // The character at position startIndex is included in the search.  startIndex is the larger
         // index within the string.
         //
-        public int LastIndexOf(char value) => SpanHelpers.LastIndexOf(ref _firstChar, value, Length);
+        public int LastIndexOf(char value)
+            => SpanHelpers.LastIndexOfValueType<short, SpanHelpers.DefaultEqualityComparer<short>>(
+                    ref Unsafe.As<char, short>(ref _firstChar), (short)value, Length);
 
         public int LastIndexOf(char value, int startIndex)
         {
@@ -306,7 +308,8 @@ public unsafe int LastIndexOf(char value, int startIndex, int count)
             }
 
             int startSearchAt = startIndex + 1 - count;
-            int result = SpanHelpers.LastIndexOf(ref Unsafe.Add(ref _firstChar, startSearchAt), value, count);
+            int result = SpanHelpers.LastIndexOfValueType<short, SpanHelpers.DefaultEqualityComparer<short>>
+                (ref Unsafe.As<char, short>(ref Unsafe.Add(ref _firstChar, startSearchAt)), (short)value, count);
 
             return result < 0 ? result : result + startSearchAt;
         }

From 22f71ff8b56d3729f467c108b97ab0c7960dd1de Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Thu, 11 Aug 2022 16:01:52 +0200
Subject: [PATCH 04/34] vectorize LastIndexOfAny(value0, value1)

---
 .../src/System/MemoryExtensions.cs            |  52 +++++---
 .../src/System/SpanHelpers.Byte.cs            | 124 ------------------
 .../src/System/SpanHelpers.T.cs               |  87 ++++++++++++
 3 files changed, 123 insertions(+), 140 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
index 3eec999bed4bba..b27306e453bc1a 100644
--- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
@@ -1239,16 +1239,7 @@ private static unsafe int IndexOfAnyProbabilistic(ref char searchSpace, int sear
         /// <param name="value1">One of the values to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOfAny<T>(this Span<T> span, T value0, T value1) where T : IEquatable<T>?
-        {
-            if (Unsafe.SizeOf<T>() == sizeof(byte) && RuntimeHelpers.IsBitwiseEquatable<T>())
-                return SpanHelpers.LastIndexOfAny(
-                    ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
-                    Unsafe.As<T, byte>(ref value0),
-                    Unsafe.As<T, byte>(ref value1),
-                    span.Length);
-
-            return SpanHelpers.LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, span.Length);
-        }
+            => LastIndexOfAny((ReadOnlySpan<T>)span, value0, value1);
 
         /// <summary>
         /// Searches for the last index of any of the specified values similar to calling LastIndexOf several times with the logical OR operator. If not found, returns -1.
@@ -1298,12 +1289,41 @@ ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(values)),
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOfAny<T>(this ReadOnlySpan<T> span, T value0, T value1) where T : IEquatable<T>?
         {
-            if (Unsafe.SizeOf<T>() == sizeof(byte) && RuntimeHelpers.IsBitwiseEquatable<T>())
-                return SpanHelpers.LastIndexOfAny(
-                    ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
-                    Unsafe.As<T, byte>(ref value0),
-                    Unsafe.As<T, byte>(ref value1),
-                    span.Length);
+            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            {
+                if (Unsafe.SizeOf<T>() == sizeof(byte))
+                {
+                    return SpanHelpers.LastIndexOfAnyValueType<byte, SpanHelpers.DefaultEqualityComparer<byte>>(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, byte>(ref value0),
+                        Unsafe.As<T, byte>(ref value1),
+                        span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
+                {
+                    return SpanHelpers.LastIndexOfAnyValueType<short, SpanHelpers.DefaultEqualityComparer<short>>(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value0),
+                        Unsafe.As<T, short>(ref value1),
+                        span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(int))
+                {
+                    return SpanHelpers.LastIndexOfAnyValueType<int, SpanHelpers.DefaultEqualityComparer<int>>(
+                        ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, int>(ref value0),
+                        Unsafe.As<T, int>(ref value1),
+                        span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(long))
+                {
+                    return SpanHelpers.LastIndexOfAnyValueType<long, SpanHelpers.DefaultEqualityComparer<long>>(
+                        ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, long>(ref value0),
+                        Unsafe.As<T, long>(ref value1),
+                        span.Length);
+                }
+            }
 
             return SpanHelpers.LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, span.Length);
         }
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
index f267f195a00580..ea434b84af63ab 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
@@ -1252,130 +1252,6 @@ public static int IndexOfAny(ref byte searchSpace, byte value0, byte value1, byt
             goto NotFound;
         }
 
-        public static int LastIndexOfAny(ref byte searchSpace, byte value0, byte value1, int length)
-        {
-            Debug.Assert(length >= 0);
-
-            uint uValue0 = value0; // Use uint for comparisons to avoid unnecessary 8->32 extensions
-            uint uValue1 = value1;
-            nuint offset = (nuint)(uint)length; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations
-            nuint lengthToExamine = (nuint)(uint)length;
-
-            if (Vector.IsHardwareAccelerated && length >= Vector<byte>.Count * 2)
-            {
-                lengthToExamine = UnalignedCountVectorFromEnd(ref searchSpace, length);
-            }
-        SequentialScan:
-            uint lookUp;
-            while (lengthToExamine >= 8)
-            {
-                lengthToExamine -= 8;
-                offset -= 8;
-
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 7);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found7;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 6);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found6;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 5);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found5;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 4);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found4;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 3);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found3;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 2);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found2;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 1);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found1;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found;
-            }
-
-            if (lengthToExamine >= 4)
-            {
-                lengthToExamine -= 4;
-                offset -= 4;
-
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 3);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found3;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 2);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found2;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 1);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found1;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found;
-            }
-
-            while (lengthToExamine > 0)
-            {
-                lengthToExamine -= 1;
-                offset -= 1;
-
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found;
-            }
-
-            if (Vector.IsHardwareAccelerated && (offset > 0))
-            {
-                lengthToExamine = (offset & (nuint)~(Vector<byte>.Count - 1));
-
-                Vector<byte> values0 = new Vector<byte>(value0);
-                Vector<byte> values1 = new Vector<byte>(value1);
-
-                while (lengthToExamine > (nuint)(Vector<byte>.Count - 1))
-                {
-                    Vector<byte> search = LoadVector(ref searchSpace, offset - (nuint)Vector<byte>.Count);
-                    var matches = Vector.BitwiseOr(
-                                    Vector.Equals(search, values0),
-                                    Vector.Equals(search, values1));
-                    if (Vector<byte>.Zero.Equals(matches))
-                    {
-                        offset -= (nuint)Vector<byte>.Count;
-                        lengthToExamine -= (nuint)Vector<byte>.Count;
-                        continue;
-                    }
-
-                    // Find offset of first match and add to current offset
-                    return (int)(offset) - Vector<byte>.Count + LocateLastFoundByte(matches);
-                }
-
-                if (offset > 0)
-                {
-                    lengthToExamine = offset;
-                    goto SequentialScan;
-                }
-            }
-            return -1;
-        Found: // Workaround for https://github.com/dotnet/runtime/issues/8795
-            return (int)offset;
-        Found1:
-            return (int)(offset + 1);
-        Found2:
-            return (int)(offset + 2);
-        Found3:
-            return (int)(offset + 3);
-        Found4:
-            return (int)(offset + 4);
-        Found5:
-            return (int)(offset + 5);
-        Found6:
-            return (int)(offset + 6);
-        Found7:
-            return (int)(offset + 7);
-        }
-
         public static int LastIndexOfAny(ref byte searchSpace, byte value0, byte value1, byte value2, int length)
         {
             Debug.Assert(length >= 0);
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
index d9301b0a336e13..49872b6ea44bdb 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
@@ -1425,6 +1425,93 @@ internal static int LastIndexOfValueType<T, C>(ref T searchSpace, T value, int l
             return -1;
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
+        internal static int LastIndexOfAnyValueType<T, C>(ref T searchSpace, T value0, T value1, int length)
+            where T : struct, IEquatable<T>
+            where C : struct, IVectorEqualityComparer<T>
+        {
+            Debug.Assert(length >= 0, "Expected non-negative length");
+            Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types");
+
+            C comparer = default;
+
+            if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
+            {
+                for (int i = length - 1; i >= 0; i--)
+                {
+                    T current = Unsafe.Add(ref searchSpace, i);
+                    if (comparer.Equals(current, value0) || comparer.Equals(current, value1))
+                    {
+                        return i;
+                    }
+                }
+            }
+            else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
+            {
+                Vector256<T> equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1);
+                ref T currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256<T>.Count);
+
+                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
+                do
+                {
+                    current = Vector256.LoadUnsafe(ref currentSearchSpace);
+                    equals = comparer.Equals256(values0, current) | comparer.Equals256(values1, current);
+                    if (equals == Vector256<T>.Zero)
+                    {
+                        currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<T>.Count);
+                        continue;
+                    }
+
+                    return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals);
+                }
+                while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace));
+
+                // If any elements remain, process the first vector in the search space.
+                if ((uint)length % Vector256<T>.Count != 0)
+                {
+                    current = Vector256.LoadUnsafe(ref searchSpace);
+                    equals = comparer.Equals256(values0, current) | comparer.Equals256(values1, current);
+                    if (equals != Vector256<T>.Zero)
+                    {
+                        return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
+                    }
+                }
+            }
+            else
+            {
+                Vector128<T> equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1);
+                ref T currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128<T>.Count);
+
+                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
+                do
+                {
+                    current = Vector128.LoadUnsafe(ref currentSearchSpace);
+                    equals = comparer.Equals128(values0, current) | comparer.Equals128(values1, current);
+                    if (equals == Vector128<T>.Zero)
+                    {
+                        currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<T>.Count);
+                        continue;
+                    }
+
+                    return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals);
+                }
+                while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace));
+
+                // If any elements remain, process the first vector in the search space.
+                if ((uint)length % Vector128<T>.Count != 0)
+                {
+                    current = Vector128.LoadUnsafe(ref searchSpace);
+                    equals = comparer.Equals128(values0, current) | comparer.Equals128(values1, current);
+                    if (equals != Vector128<T>.Zero)
+                    {
+                        return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
+                    }
+                }
+            }
+
+            return -1;
+        }
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static int ComputeLastIndex<T>(ref T searchSpace, ref T current, Vector128<T> equals) where T : struct, IEquatable<T>
         {

From bbc64e006a891f3f65036c2c387d6d97d77c46db Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Thu, 11 Aug 2022 16:15:30 +0200
Subject: [PATCH 05/34] vectorize LastIndexOfAnyExcept(value0, value1)

---
 .../src/System/MemoryExtensions.cs            | 36 +++++++++++++++
 .../src/System/SpanHelpers.T.cs               | 46 ++++++++++---------
 2 files changed, 61 insertions(+), 21 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
index b27306e453bc1a..50b6a169d80040 100644
--- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
@@ -726,6 +726,42 @@ ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
         /// </returns>
         public static int LastIndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value0, T value1) where T : IEquatable<T>?
         {
+            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            {
+                if (Unsafe.SizeOf<T>() == sizeof(byte))
+                {
+                    return SpanHelpers.LastIndexOfAnyValueType<byte, SpanHelpers.ExceptEqualityComparer<byte>>(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, byte>(ref value0),
+                        Unsafe.As<T, byte>(ref value1),
+                        span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
+                {
+                    return SpanHelpers.LastIndexOfAnyValueType<short, SpanHelpers.ExceptEqualityComparer<short>>(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value0),
+                        Unsafe.As<T, short>(ref value1),
+                        span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(int))
+                {
+                    return SpanHelpers.LastIndexOfAnyValueType<int, SpanHelpers.ExceptEqualityComparer<int>>(
+                        ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, int>(ref value0),
+                        Unsafe.As<T, int>(ref value1),
+                        span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(long))
+                {
+                    return SpanHelpers.LastIndexOfAnyValueType<long, SpanHelpers.ExceptEqualityComparer<long>>(
+                        ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, long>(ref value0),
+                        Unsafe.As<T, long>(ref value1),
+                        span.Length);
+                }
+            }
+
             for (int i = span.Length - 1; i >= 0; i--)
             {
                 if (!EqualityComparer<T>.Default.Equals(span[i], value0) &&
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
index 49872b6ea44bdb..47c95a4b2dfd83 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
@@ -1439,8 +1439,7 @@ internal static int LastIndexOfAnyValueType<T, C>(ref T searchSpace, T value0, T
             {
                 for (int i = length - 1; i >= 0; i--)
                 {
-                    T current = Unsafe.Add(ref searchSpace, i);
-                    if (comparer.Equals(current, value0) || comparer.Equals(current, value1))
+                    if (comparer.Equals(Unsafe.Add(ref searchSpace, i), value0, value1))
                     {
                         return i;
                     }
@@ -1448,14 +1447,13 @@ internal static int LastIndexOfAnyValueType<T, C>(ref T searchSpace, T value0, T
             }
             else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
             {
-                Vector256<T> equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1);
+                Vector256<T> equals, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1);
                 ref T currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256<T>.Count);
 
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
-                    current = Vector256.LoadUnsafe(ref currentSearchSpace);
-                    equals = comparer.Equals256(values0, current) | comparer.Equals256(values1, current);
+                    equals = comparer.Equals256(Vector256.LoadUnsafe(ref currentSearchSpace), values0, values1);
                     if (equals == Vector256<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<T>.Count);
@@ -1469,8 +1467,7 @@ internal static int LastIndexOfAnyValueType<T, C>(ref T searchSpace, T value0, T
                 // If any elements remain, process the first vector in the search space.
                 if ((uint)length % Vector256<T>.Count != 0)
                 {
-                    current = Vector256.LoadUnsafe(ref searchSpace);
-                    equals = comparer.Equals256(values0, current) | comparer.Equals256(values1, current);
+                    equals = comparer.Equals256(Vector256.LoadUnsafe(ref searchSpace), values0, values1);
                     if (equals != Vector256<T>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
@@ -1479,14 +1476,13 @@ internal static int LastIndexOfAnyValueType<T, C>(ref T searchSpace, T value0, T
             }
             else
             {
-                Vector128<T> equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1);
+                Vector128<T> equals, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1);
                 ref T currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128<T>.Count);
 
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
-                    current = Vector128.LoadUnsafe(ref currentSearchSpace);
-                    equals = comparer.Equals128(values0, current) | comparer.Equals128(values1, current);
+                    equals = comparer.Equals128(Vector128.LoadUnsafe(ref currentSearchSpace), values0, values1);
                     if (equals == Vector128<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<T>.Count);
@@ -1500,8 +1496,7 @@ internal static int LastIndexOfAnyValueType<T, C>(ref T searchSpace, T value0, T
                 // If any elements remain, process the first vector in the search space.
                 if ((uint)length % Vector128<T>.Count != 0)
                 {
-                    current = Vector128.LoadUnsafe(ref searchSpace);
-                    equals = comparer.Equals128(values0, current) | comparer.Equals128(values1, current);
+                    equals = comparer.Equals128(Vector128.LoadUnsafe(ref searchSpace), values0, values1);
                     if (equals != Vector128<T>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
@@ -1530,23 +1525,32 @@ private static int ComputeLastIndex<T>(ref T searchSpace, ref T current, Vector2
 
         internal interface IVectorEqualityComparer<T> where T : struct, IEquatable<T>
         {
-            bool Equals(T left, T right);
-            Vector128<T> Equals128(Vector128<T> left, Vector128<T> right);
-            Vector256<T> Equals256(Vector256<T> left, Vector256<T> right);
+            bool Equals(T current, T value);
+            bool Equals(T current, T value0, T value1);
+            Vector128<T> Equals128(Vector128<T> current, Vector128<T> value);
+            Vector128<T> Equals128(Vector128<T> current, Vector128<T> value0, Vector128<T> value1);
+            Vector256<T> Equals256(Vector256<T> current, Vector256<T> value);
+            Vector256<T> Equals256(Vector256<T> current, Vector256<T> value0, Vector256<T> value1);
         }
 
         internal readonly struct DefaultEqualityComparer<T> : IVectorEqualityComparer<T> where T : struct, IEquatable<T>
         {
-            public bool Equals(T left, T right) => left.Equals(right);
-            public Vector128<T> Equals128(Vector128<T> left, Vector128<T> right) => Vector128.Equals(left, right);
-            public Vector256<T> Equals256(Vector256<T> left, Vector256<T> right) => Vector256.Equals(left, right);
+            public bool Equals(T current, T value) => current.Equals(value);
+            public bool Equals(T current, T value0, T value1) => current.Equals(value0) || current.Equals(value1);
+            public Vector128<T> Equals128(Vector128<T> current, Vector128<T> value) => Vector128.Equals(current, value);
+            public Vector128<T> Equals128(Vector128<T> current, Vector128<T> value0, Vector128<T> value1) => Vector128.Equals(current, value0) | Vector128.Equals(current, value1);
+            public Vector256<T> Equals256(Vector256<T> current, Vector256<T> value) => Vector256.Equals(current, value);
+            public Vector256<T> Equals256(Vector256<T> current, Vector256<T> value0, Vector256<T> value1) => Vector256.Equals(current, value0) | Vector256.Equals(current, value1);
         }
 
         internal readonly struct ExceptEqualityComparer<T> : IVectorEqualityComparer<T> where T : struct, IEquatable<T>
         {
-            public bool Equals(T left, T right) => !left.Equals(right);
-            public Vector128<T> Equals128(Vector128<T> left, Vector128<T> right) => ~Vector128.Equals(left, right);
-            public Vector256<T> Equals256(Vector256<T> left, Vector256<T> right) => ~Vector256.Equals(left, right);
+            public bool Equals(T current, T value) => !current.Equals(value);
+            public bool Equals(T current, T value0, T value1) => !current.Equals(value0) && !current.Equals(value1);
+            public Vector128<T> Equals128(Vector128<T> current, Vector128<T> value) => ~Vector128.Equals(current, value);
+            public Vector128<T> Equals128(Vector128<T> current, Vector128<T> value0, Vector128<T> value1) => ~(Vector128.Equals(current, value0) | Vector128.Equals(current, value1));
+            public Vector256<T> Equals256(Vector256<T> current, Vector256<T> value) => ~Vector256.Equals(current, value);
+            public Vector256<T> Equals256(Vector256<T> current, Vector256<T> value0, Vector256<T> value1) => ~(Vector256.Equals(current, value0) | Vector256.Equals(current, value1));
         }
     }
 }

From f265d285ef385aa8cc03bc4dc463e8dc8e0b5f30 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Thu, 11 Aug 2022 16:40:24 +0200
Subject: [PATCH 06/34] simplify it to make it easier to add 3 and 4 values
 overloads

---
 .../src/System/SpanHelpers.T.cs               | 57 +++++++++----------
 1 file changed, 27 insertions(+), 30 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
index 47c95a4b2dfd83..3839d6363d3d63 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
@@ -1357,7 +1357,7 @@ internal static int LastIndexOfValueType<T, C>(ref T searchSpace, T value, int l
             {
                 for (int i = length - 1; i >= 0; i--)
                 {
-                    if (comparer.Equals(Unsafe.Add(ref searchSpace, i), value))
+                    if (comparer.NegateIfNeeded(Unsafe.Add(ref searchSpace, i).Equals(value)))
                     {
                         return i;
                     }
@@ -1371,7 +1371,7 @@ internal static int LastIndexOfValueType<T, C>(ref T searchSpace, T value, int l
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
-                    equals = comparer.Equals256(values, Vector256.LoadUnsafe(ref currentSearchSpace));
+                    equals = comparer.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref currentSearchSpace)));
                     if (equals == Vector256<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<T>.Count);
@@ -1385,7 +1385,7 @@ internal static int LastIndexOfValueType<T, C>(ref T searchSpace, T value, int l
                 // If any elements remain, process the first vector in the search space.
                 if ((uint)length % Vector256<T>.Count != 0)
                 {
-                    equals = comparer.Equals256(values, Vector256.LoadUnsafe(ref searchSpace));
+                    equals = comparer.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref searchSpace)));
                     if (equals != Vector256<T>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
@@ -1400,7 +1400,7 @@ internal static int LastIndexOfValueType<T, C>(ref T searchSpace, T value, int l
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
-                    equals = comparer.Equals128(values, Vector128.LoadUnsafe(ref currentSearchSpace));
+                    equals = comparer.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref currentSearchSpace)));
                     if (equals == Vector128<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<T>.Count);
@@ -1414,7 +1414,7 @@ internal static int LastIndexOfValueType<T, C>(ref T searchSpace, T value, int l
                 // If any elements remain, process the first vector in the search space.
                 if ((uint)length % Vector128<T>.Count != 0)
                 {
-                    equals = comparer.Equals128(values, Vector128.LoadUnsafe(ref searchSpace));
+                    equals = comparer.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref searchSpace)));
                     if (equals != Vector128<T>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
@@ -1439,7 +1439,8 @@ internal static int LastIndexOfAnyValueType<T, C>(ref T searchSpace, T value0, T
             {
                 for (int i = length - 1; i >= 0; i--)
                 {
-                    if (comparer.Equals(Unsafe.Add(ref searchSpace, i), value0, value1))
+                    T current = Unsafe.Add(ref searchSpace, i);
+                    if (comparer.NegateIfNeeded(current.Equals(value0) || current.Equals(value1)))
                     {
                         return i;
                     }
@@ -1447,13 +1448,14 @@ internal static int LastIndexOfAnyValueType<T, C>(ref T searchSpace, T value0, T
             }
             else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
             {
-                Vector256<T> equals, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1);
+                Vector256<T> equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1);
                 ref T currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256<T>.Count);
 
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
-                    equals = comparer.Equals256(Vector256.LoadUnsafe(ref currentSearchSpace), values0, values1);
+                    current = Vector256.LoadUnsafe(ref currentSearchSpace);
+                    equals = comparer.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1));
                     if (equals == Vector256<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<T>.Count);
@@ -1467,7 +1469,8 @@ internal static int LastIndexOfAnyValueType<T, C>(ref T searchSpace, T value0, T
                 // If any elements remain, process the first vector in the search space.
                 if ((uint)length % Vector256<T>.Count != 0)
                 {
-                    equals = comparer.Equals256(Vector256.LoadUnsafe(ref searchSpace), values0, values1);
+                    current = Vector256.LoadUnsafe(ref searchSpace);
+                    equals = comparer.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1));
                     if (equals != Vector256<T>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
@@ -1476,13 +1479,14 @@ internal static int LastIndexOfAnyValueType<T, C>(ref T searchSpace, T value0, T
             }
             else
             {
-                Vector128<T> equals, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1);
+                Vector128<T> equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1);
                 ref T currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128<T>.Count);
 
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
-                    equals = comparer.Equals128(Vector128.LoadUnsafe(ref currentSearchSpace), values0, values1);
+                    current = Vector128.LoadUnsafe(ref currentSearchSpace);
+                    equals = comparer.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1));
                     if (equals == Vector128<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<T>.Count);
@@ -1496,7 +1500,8 @@ internal static int LastIndexOfAnyValueType<T, C>(ref T searchSpace, T value0, T
                 // If any elements remain, process the first vector in the search space.
                 if ((uint)length % Vector128<T>.Count != 0)
                 {
-                    equals = comparer.Equals128(Vector128.LoadUnsafe(ref searchSpace), values0, values1);
+                    current = Vector128.LoadUnsafe(ref searchSpace);
+                    equals = comparer.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1));
                     if (equals != Vector128<T>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
@@ -1525,32 +1530,24 @@ private static int ComputeLastIndex<T>(ref T searchSpace, ref T current, Vector2
 
         internal interface IVectorEqualityComparer<T> where T : struct, IEquatable<T>
         {
-            bool Equals(T current, T value);
-            bool Equals(T current, T value0, T value1);
-            Vector128<T> Equals128(Vector128<T> current, Vector128<T> value);
-            Vector128<T> Equals128(Vector128<T> current, Vector128<T> value0, Vector128<T> value1);
-            Vector256<T> Equals256(Vector256<T> current, Vector256<T> value);
-            Vector256<T> Equals256(Vector256<T> current, Vector256<T> value0, Vector256<T> value1);
+            bool NegateIfNeeded(bool equals);
+            Vector128<T> NegateIfNeeded(Vector128<T> equals);
+            Vector256<T> NegateIfNeeded(Vector256<T> equals);
         }
 
         internal readonly struct DefaultEqualityComparer<T> : IVectorEqualityComparer<T> where T : struct, IEquatable<T>
         {
-            public bool Equals(T current, T value) => current.Equals(value);
-            public bool Equals(T current, T value0, T value1) => current.Equals(value0) || current.Equals(value1);
-            public Vector128<T> Equals128(Vector128<T> current, Vector128<T> value) => Vector128.Equals(current, value);
-            public Vector128<T> Equals128(Vector128<T> current, Vector128<T> value0, Vector128<T> value1) => Vector128.Equals(current, value0) | Vector128.Equals(current, value1);
-            public Vector256<T> Equals256(Vector256<T> current, Vector256<T> value) => Vector256.Equals(current, value);
-            public Vector256<T> Equals256(Vector256<T> current, Vector256<T> value0, Vector256<T> value1) => Vector256.Equals(current, value0) | Vector256.Equals(current, value1);
+            public bool NegateIfNeeded(bool equals) => equals;
+            public Vector128<T> NegateIfNeeded(Vector128<T> equals) => equals;
+            public Vector256<T> NegateIfNeeded(Vector256<T> equals) => equals;
         }
 
         internal readonly struct ExceptEqualityComparer<T> : IVectorEqualityComparer<T> where T : struct, IEquatable<T>
         {
-            public bool Equals(T current, T value) => !current.Equals(value);
-            public bool Equals(T current, T value0, T value1) => !current.Equals(value0) && !current.Equals(value1);
-            public Vector128<T> Equals128(Vector128<T> current, Vector128<T> value) => ~Vector128.Equals(current, value);
-            public Vector128<T> Equals128(Vector128<T> current, Vector128<T> value0, Vector128<T> value1) => ~(Vector128.Equals(current, value0) | Vector128.Equals(current, value1));
-            public Vector256<T> Equals256(Vector256<T> current, Vector256<T> value) => ~Vector256.Equals(current, value);
-            public Vector256<T> Equals256(Vector256<T> current, Vector256<T> value0, Vector256<T> value1) => ~(Vector256.Equals(current, value0) | Vector256.Equals(current, value1));
+            public bool NegateIfNeeded(bool equals) => !equals;
+            public Vector128<T> NegateIfNeeded(Vector128<T> equals) => ~equals;
+            public Vector256<T> NegateIfNeeded(Vector256<T> equals) => ~equals;
+
         }
     }
 }

From b0f259c77590191d8e788cabeb2c8da86aae846b Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Thu, 11 Aug 2022 17:04:00 +0200
Subject: [PATCH 07/34] vectorize LastIndexOfAny and LastIndexOfAnyExcept for 3
 values

---
 .../src/System/MemoryExtensions.cs            | 64 ++++++++++----
 .../src/System/SpanHelpers.T.cs               | 87 +++++++++++++++++++
 2 files changed, 133 insertions(+), 18 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
index 50b6a169d80040..30510ae44199cd 100644
--- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
@@ -786,6 +786,29 @@ ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
         /// </returns>
         public static int LastIndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value0, T value1, T value2) where T : IEquatable<T>?
         {
+            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            {
+                if (Unsafe.SizeOf<T>() == sizeof(byte))
+                {
+                    return SpanHelpers.LastIndexOfAnyValueType<byte, SpanHelpers.ExceptEqualityComparer<byte>>(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, byte>(ref value0),
+                        Unsafe.As<T, byte>(ref value1),
+                        Unsafe.As<T, byte>(ref value2),
+                        span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
+                {
+                    return SpanHelpers.LastIndexOfAnyValueType<short, SpanHelpers.ExceptEqualityComparer<short>>(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value0),
+                        Unsafe.As<T, short>(ref value1),
+                        Unsafe.As<T, short>(ref value2),
+                        span.Length);
+                }
+                // we can easily add int and long here, but there must be an evidence that shows that it's actually needed
+            }
+
             for (int i = span.Length - 1; i >= 0; i--)
             {
                 if (!EqualityComparer<T>.Default.Equals(span[i], value0) &&
@@ -1286,17 +1309,7 @@ public static int LastIndexOfAny<T>(this Span<T> span, T value0, T value1) where
         /// <param name="value2">One of the values to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOfAny<T>(this Span<T> span, T value0, T value1, T value2) where T : IEquatable<T>?
-        {
-            if (Unsafe.SizeOf<T>() == sizeof(byte) && RuntimeHelpers.IsBitwiseEquatable<T>())
-                return SpanHelpers.LastIndexOfAny(
-                    ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
-                    Unsafe.As<T, byte>(ref value0),
-                    Unsafe.As<T, byte>(ref value1),
-                    Unsafe.As<T, byte>(ref value2),
-                    span.Length);
-
-            return SpanHelpers.LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, value2, span.Length);
-        }
+            => LastIndexOfAny((ReadOnlySpan<T>)span, value0, value1, value2);
 
         /// <summary>
         /// Searches for the last index of any of the specified values similar to calling LastIndexOf several times with the logical OR operator. If not found, returns -1.
@@ -1374,13 +1387,28 @@ ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOfAny<T>(this ReadOnlySpan<T> span, T value0, T value1, T value2) where T : IEquatable<T>?
         {
-            if (Unsafe.SizeOf<T>() == sizeof(byte) && RuntimeHelpers.IsBitwiseEquatable<T>())
-                return SpanHelpers.LastIndexOfAny(
-                    ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
-                    Unsafe.As<T, byte>(ref value0),
-                    Unsafe.As<T, byte>(ref value1),
-                    Unsafe.As<T, byte>(ref value2),
-                    span.Length);
+            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            {
+                if (Unsafe.SizeOf<T>() == sizeof(byte))
+                {
+                    return SpanHelpers.LastIndexOfAnyValueType<byte, SpanHelpers.DefaultEqualityComparer<byte>>(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, byte>(ref value0),
+                        Unsafe.As<T, byte>(ref value1),
+                        Unsafe.As<T, byte>(ref value2),
+                        span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
+                {
+                    return SpanHelpers.LastIndexOfAnyValueType<short, SpanHelpers.DefaultEqualityComparer<short>>(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value0),
+                        Unsafe.As<T, short>(ref value1),
+                        Unsafe.As<T, short>(ref value2),
+                        span.Length);
+                }
+                // we can easily add int and long here, but there must be an evidence that shows that it's actually needed
+            }
 
             return SpanHelpers.LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, value2, span.Length);
         }
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
index 3839d6363d3d63..9976d62b38c9e9 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
@@ -1512,6 +1512,93 @@ internal static int LastIndexOfAnyValueType<T, C>(ref T searchSpace, T value0, T
             return -1;
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
+        internal static int LastIndexOfAnyValueType<T, C>(ref T searchSpace, T value0, T value1, T value2, int length)
+            where T : struct, IEquatable<T>
+            where C : struct, IVectorEqualityComparer<T>
+        {
+            Debug.Assert(length >= 0, "Expected non-negative length");
+            Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types");
+
+            C comparer = default;
+
+            if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
+            {
+                for (int i = length - 1; i >= 0; i--)
+                {
+                    T current = Unsafe.Add(ref searchSpace, i);
+                    if (comparer.NegateIfNeeded(current.Equals(value0) || current.Equals(value1) || current.Equals(value2)))
+                    {
+                        return i;
+                    }
+                }
+            }
+            else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
+            {
+                Vector256<T> equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1), values2 = Vector256.Create(value2);
+                ref T currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256<T>.Count);
+
+                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
+                do
+                {
+                    current = Vector256.LoadUnsafe(ref currentSearchSpace);
+                    equals = comparer.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) | Vector256.Equals(current, values2));
+                    if (equals == Vector256<T>.Zero)
+                    {
+                        currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<T>.Count);
+                        continue;
+                    }
+
+                    return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals);
+                }
+                while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace));
+
+                // If any elements remain, process the first vector in the search space.
+                if ((uint)length % Vector256<T>.Count != 0)
+                {
+                    current = Vector256.LoadUnsafe(ref searchSpace);
+                    equals = comparer.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) | Vector256.Equals(current, values2));
+                    if (equals != Vector256<T>.Zero)
+                    {
+                        return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
+                    }
+                }
+            }
+            else
+            {
+                Vector128<T> equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1), values2 = Vector128.Create(value2);
+                ref T currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128<T>.Count);
+
+                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
+                do
+                {
+                    current = Vector128.LoadUnsafe(ref currentSearchSpace);
+                    equals = comparer.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) | Vector128.Equals(current, values2));
+                    if (equals == Vector128<T>.Zero)
+                    {
+                        currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<T>.Count);
+                        continue;
+                    }
+
+                    return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals);
+                }
+                while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace));
+
+                // If any elements remain, process the first vector in the search space.
+                if ((uint)length % Vector128<T>.Count != 0)
+                {
+                    current = Vector128.LoadUnsafe(ref searchSpace);
+                    equals = comparer.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) | Vector128.Equals(current, values2));
+                    if (equals != Vector128<T>.Zero)
+                    {
+                        return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
+                    }
+                }
+            }
+
+            return -1;
+        }
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static int ComputeLastIndex<T>(ref T searchSpace, ref T current, Vector128<T> equals) where T : struct, IEquatable<T>
         {

From 7dc03c34a8c5948c82896e5b2122e8d3ec98cceb Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Thu, 11 Aug 2022 17:08:23 +0200
Subject: [PATCH 08/34] rename (I am not convinced it's the best name yet)

---
 .../src/System/Array.cs                       |  8 +--
 .../src/System/MemoryExtensions.cs            | 40 +++++++-------
 .../src/System/SpanHelpers.Byte.cs            |  4 +-
 .../src/System/SpanHelpers.Char.cs            |  4 +-
 .../src/System/SpanHelpers.T.cs               | 54 +++++++++----------
 .../src/System/String.Searching.cs            |  4 +-
 6 files changed, 57 insertions(+), 57 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/Array.cs b/src/libraries/System.Private.CoreLib/src/System/Array.cs
index a7d2ff01e9d23d..1406d248f70549 100644
--- a/src/libraries/System.Private.CoreLib/src/System/Array.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/Array.cs
@@ -1586,7 +1586,7 @@ public static int LastIndexOf<T>(T[] array, T value, int startIndex, int count)
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
                     int endIndex = startIndex - count + 1;
-                    int result = SpanHelpers.LastIndexOfValueType<byte, SpanHelpers.DefaultEqualityComparer<byte>>(
+                    int result = SpanHelpers.LastIndexOfValueType<byte, SpanHelpers.DontNegate<byte>>(
                         ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<byte[]>(array)), endIndex),
                         Unsafe.As<T, byte>(ref value),
                         count);
@@ -1596,7 +1596,7 @@ ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<byte[]>(array))
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
                     int endIndex = startIndex - count + 1;
-                    int result = SpanHelpers.LastIndexOfValueType<short, SpanHelpers.DefaultEqualityComparer<short>>(
+                    int result = SpanHelpers.LastIndexOfValueType<short, SpanHelpers.DontNegate<short>>(
                         ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<short[]>(array)), endIndex),
                         Unsafe.As<T, short>(ref value),
                         count);
@@ -1606,7 +1606,7 @@ ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<short[]>(array)
                 else if (Unsafe.SizeOf<T>() == sizeof(int))
                 {
                     int endIndex = startIndex - count + 1;
-                    int result = SpanHelpers.LastIndexOfValueType<int, SpanHelpers.DefaultEqualityComparer<int>>(
+                    int result = SpanHelpers.LastIndexOfValueType<int, SpanHelpers.DontNegate<int>>(
                         ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<int[]>(array)), endIndex),
                         Unsafe.As<T, int>(ref value),
                         count);
@@ -1616,7 +1616,7 @@ ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<int[]>(array)),
                 else if (Unsafe.SizeOf<T>() == sizeof(long))
                 {
                     int endIndex = startIndex - count + 1;
-                    int result = SpanHelpers.LastIndexOfValueType<long, SpanHelpers.DefaultEqualityComparer<long>>(
+                    int result = SpanHelpers.LastIndexOfValueType<long, SpanHelpers.DontNegate<long>>(
                         ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<long[]>(array)), endIndex),
                         Unsafe.As<T, long>(ref value),
                         count);
diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
index 30510ae44199cd..9e2f22cc7d21dc 100644
--- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
@@ -676,28 +676,28 @@ public static int LastIndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value) wh
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.LastIndexOfValueType<byte, SpanHelpers.ExceptEqualityComparer<byte>>(
+                    return SpanHelpers.LastIndexOfValueType<byte, SpanHelpers.Negate<byte>>(
                         ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, byte>(ref value),
                         span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    return SpanHelpers.LastIndexOfValueType<short, SpanHelpers.ExceptEqualityComparer<short>>(
+                    return SpanHelpers.LastIndexOfValueType<short, SpanHelpers.Negate<short>>(
                         ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, short>(ref value),
                         span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(int))
                 {
-                    return SpanHelpers.LastIndexOfValueType<int, SpanHelpers.ExceptEqualityComparer<int>>(
+                    return SpanHelpers.LastIndexOfValueType<int, SpanHelpers.Negate<int>>(
                         ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, int>(ref value),
                         span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(long))
                 {
-                    return SpanHelpers.LastIndexOfValueType<long, SpanHelpers.ExceptEqualityComparer<long>>(
+                    return SpanHelpers.LastIndexOfValueType<long, SpanHelpers.Negate<long>>(
                         ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, long>(ref value),
                         span.Length);
@@ -730,7 +730,7 @@ public static int LastIndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value0, T
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType<byte, SpanHelpers.ExceptEqualityComparer<byte>>(
+                    return SpanHelpers.LastIndexOfAnyValueType<byte, SpanHelpers.Negate<byte>>(
                         ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, byte>(ref value0),
                         Unsafe.As<T, byte>(ref value1),
@@ -738,7 +738,7 @@ ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType<short, SpanHelpers.ExceptEqualityComparer<short>>(
+                    return SpanHelpers.LastIndexOfAnyValueType<short, SpanHelpers.Negate<short>>(
                         ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, short>(ref value0),
                         Unsafe.As<T, short>(ref value1),
@@ -746,7 +746,7 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(int))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType<int, SpanHelpers.ExceptEqualityComparer<int>>(
+                    return SpanHelpers.LastIndexOfAnyValueType<int, SpanHelpers.Negate<int>>(
                         ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, int>(ref value0),
                         Unsafe.As<T, int>(ref value1),
@@ -754,7 +754,7 @@ ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(long))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType<long, SpanHelpers.ExceptEqualityComparer<long>>(
+                    return SpanHelpers.LastIndexOfAnyValueType<long, SpanHelpers.Negate<long>>(
                         ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, long>(ref value0),
                         Unsafe.As<T, long>(ref value1),
@@ -790,7 +790,7 @@ public static int LastIndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value0, T
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType<byte, SpanHelpers.ExceptEqualityComparer<byte>>(
+                    return SpanHelpers.LastIndexOfAnyValueType<byte, SpanHelpers.Negate<byte>>(
                         ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, byte>(ref value0),
                         Unsafe.As<T, byte>(ref value1),
@@ -799,7 +799,7 @@ ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType<short, SpanHelpers.ExceptEqualityComparer<short>>(
+                    return SpanHelpers.LastIndexOfAnyValueType<short, SpanHelpers.Negate<short>>(
                         ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, short>(ref value0),
                         Unsafe.As<T, short>(ref value1),
@@ -977,28 +977,28 @@ public static int LastIndexOf<T>(this ReadOnlySpan<T> span, T value) where T : I
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.LastIndexOfValueType<byte, SpanHelpers.DefaultEqualityComparer<byte>>(
+                    return SpanHelpers.LastIndexOfValueType<byte, SpanHelpers.DontNegate<byte>>(
                             ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                             Unsafe.As<T, byte>(ref value),
                             span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    return SpanHelpers.LastIndexOfValueType<short, SpanHelpers.DefaultEqualityComparer<short>>(
+                    return SpanHelpers.LastIndexOfValueType<short, SpanHelpers.DontNegate<short>>(
                         ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, short>(ref value),
                         span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(int))
                 {
-                    return SpanHelpers.LastIndexOfValueType<int, SpanHelpers.DefaultEqualityComparer<int>>(
+                    return SpanHelpers.LastIndexOfValueType<int, SpanHelpers.DontNegate<int>>(
                         ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, int>(ref value),
                         span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(long))
                 {
-                    return SpanHelpers.LastIndexOfValueType<long, SpanHelpers.DefaultEqualityComparer<long>>(
+                    return SpanHelpers.LastIndexOfValueType<long, SpanHelpers.DontNegate<long>>(
                         ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, long>(ref value),
                         span.Length);
@@ -1342,7 +1342,7 @@ public static int LastIndexOfAny<T>(this ReadOnlySpan<T> span, T value0, T value
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType<byte, SpanHelpers.DefaultEqualityComparer<byte>>(
+                    return SpanHelpers.LastIndexOfAnyValueType<byte, SpanHelpers.DontNegate<byte>>(
                         ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, byte>(ref value0),
                         Unsafe.As<T, byte>(ref value1),
@@ -1350,7 +1350,7 @@ ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType<short, SpanHelpers.DefaultEqualityComparer<short>>(
+                    return SpanHelpers.LastIndexOfAnyValueType<short, SpanHelpers.DontNegate<short>>(
                         ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, short>(ref value0),
                         Unsafe.As<T, short>(ref value1),
@@ -1358,7 +1358,7 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(int))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType<int, SpanHelpers.DefaultEqualityComparer<int>>(
+                    return SpanHelpers.LastIndexOfAnyValueType<int, SpanHelpers.DontNegate<int>>(
                         ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, int>(ref value0),
                         Unsafe.As<T, int>(ref value1),
@@ -1366,7 +1366,7 @@ ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(long))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType<long, SpanHelpers.DefaultEqualityComparer<long>>(
+                    return SpanHelpers.LastIndexOfAnyValueType<long, SpanHelpers.DontNegate<long>>(
                         ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, long>(ref value0),
                         Unsafe.As<T, long>(ref value1),
@@ -1391,7 +1391,7 @@ public static int LastIndexOfAny<T>(this ReadOnlySpan<T> span, T value0, T value
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType<byte, SpanHelpers.DefaultEqualityComparer<byte>>(
+                    return SpanHelpers.LastIndexOfAnyValueType<byte, SpanHelpers.DontNegate<byte>>(
                         ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, byte>(ref value0),
                         Unsafe.As<T, byte>(ref value1),
@@ -1400,7 +1400,7 @@ ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType<short, SpanHelpers.DefaultEqualityComparer<short>>(
+                    return SpanHelpers.LastIndexOfAnyValueType<short, SpanHelpers.DontNegate<short>>(
                         ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, short>(ref value0),
                         Unsafe.As<T, short>(ref value1),
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
index ea434b84af63ab..b66137abee2d4d 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
@@ -197,7 +197,7 @@ public static int LastIndexOf(ref byte searchSpace, int searchSpaceLength, ref b
 
             int valueTailLength = valueLength - 1;
             if (valueTailLength == 0)
-                return LastIndexOfValueType<byte, DefaultEqualityComparer<byte>>(ref searchSpace, value, searchSpaceLength); // for single-byte values use plain LastIndexOf
+                return LastIndexOfValueType<byte, DontNegate<byte>>(ref searchSpace, value, searchSpaceLength); // for single-byte values use plain LastIndexOf
 
             int offset = 0;
             byte valueHead = value;
@@ -217,7 +217,7 @@ public static int LastIndexOf(ref byte searchSpace, int searchSpaceLength, ref b
                     break;  // The unsearched portion is now shorter than the sequence we're looking for. So it can't be there.
 
                 // Do a quick search for the first element of "value".
-                int relativeIndex = LastIndexOfValueType<byte, DefaultEqualityComparer<byte>>(ref searchSpace, valueHead, remainingSearchSpaceLength);
+                int relativeIndex = LastIndexOfValueType<byte, DontNegate<byte>>(ref searchSpace, valueHead, remainingSearchSpaceLength);
                 if (relativeIndex < 0)
                     break;
 
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
index c2fd20a9742c6b..aea2a3d48f32d0 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
@@ -214,7 +214,7 @@ public static int LastIndexOf(ref char searchSpace, int searchSpaceLength, ref c
 
             int valueTailLength = valueLength - 1;
             if (valueTailLength == 0)
-                return LastIndexOfValueType<short, DefaultEqualityComparer<short>>(
+                return LastIndexOfValueType<short, DontNegate<short>>(
                     ref Unsafe.As<char, short>(ref searchSpace), (short)value, searchSpaceLength); // for single-char values use plain LastIndexOf
 
             int offset = 0;
@@ -235,7 +235,7 @@ public static int LastIndexOf(ref char searchSpace, int searchSpaceLength, ref c
                     break;  // The unsearched portion is now shorter than the sequence we're looking for. So it can't be there.
 
                 // Do a quick search for the first element of "value".
-                int relativeIndex = LastIndexOfValueType<short, DefaultEqualityComparer<short>>(
+                int relativeIndex = LastIndexOfValueType<short, DontNegate<short>>(
                     ref Unsafe.As<char, short>(ref searchSpace), (short)valueHead, remainingSearchSpaceLength);
                 if (relativeIndex == -1)
                     break;
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
index 9976d62b38c9e9..9521c836150273 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
@@ -1344,20 +1344,20 @@ public static int SequenceCompareTo<T>(ref T first, int firstLength, ref T secon
         }
 
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        internal static int LastIndexOfValueType<T, C>(ref T searchSpace, T value, int length)
+        internal static int LastIndexOfValueType<T, N>(ref T searchSpace, T value, int length)
             where T : struct, IEquatable<T>
-            where C : struct, IVectorEqualityComparer<T>
+            where N : struct, INegator<T>
         {
             Debug.Assert(length >= 0, "Expected non-negative length");
             Debug.Assert(value is byte or short or int or long, "Expected caller to normalize to one of these types");
 
-            C comparer = default;
+            N negator = default;
 
             if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
             {
                 for (int i = length - 1; i >= 0; i--)
                 {
-                    if (comparer.NegateIfNeeded(Unsafe.Add(ref searchSpace, i).Equals(value)))
+                    if (negator.NegateIfNeeded(Unsafe.Add(ref searchSpace, i).Equals(value)))
                     {
                         return i;
                     }
@@ -1371,7 +1371,7 @@ internal static int LastIndexOfValueType<T, C>(ref T searchSpace, T value, int l
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
-                    equals = comparer.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref currentSearchSpace)));
+                    equals = negator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref currentSearchSpace)));
                     if (equals == Vector256<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<T>.Count);
@@ -1385,7 +1385,7 @@ internal static int LastIndexOfValueType<T, C>(ref T searchSpace, T value, int l
                 // If any elements remain, process the first vector in the search space.
                 if ((uint)length % Vector256<T>.Count != 0)
                 {
-                    equals = comparer.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref searchSpace)));
+                    equals = negator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref searchSpace)));
                     if (equals != Vector256<T>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
@@ -1400,7 +1400,7 @@ internal static int LastIndexOfValueType<T, C>(ref T searchSpace, T value, int l
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
-                    equals = comparer.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref currentSearchSpace)));
+                    equals = negator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref currentSearchSpace)));
                     if (equals == Vector128<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<T>.Count);
@@ -1414,7 +1414,7 @@ internal static int LastIndexOfValueType<T, C>(ref T searchSpace, T value, int l
                 // If any elements remain, process the first vector in the search space.
                 if ((uint)length % Vector128<T>.Count != 0)
                 {
-                    equals = comparer.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref searchSpace)));
+                    equals = negator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref searchSpace)));
                     if (equals != Vector128<T>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
@@ -1426,21 +1426,21 @@ internal static int LastIndexOfValueType<T, C>(ref T searchSpace, T value, int l
         }
 
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        internal static int LastIndexOfAnyValueType<T, C>(ref T searchSpace, T value0, T value1, int length)
+        internal static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T value1, int length)
             where T : struct, IEquatable<T>
-            where C : struct, IVectorEqualityComparer<T>
+            where N : struct, INegator<T>
         {
             Debug.Assert(length >= 0, "Expected non-negative length");
             Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types");
 
-            C comparer = default;
+            N negator = default;
 
             if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
             {
                 for (int i = length - 1; i >= 0; i--)
                 {
                     T current = Unsafe.Add(ref searchSpace, i);
-                    if (comparer.NegateIfNeeded(current.Equals(value0) || current.Equals(value1)))
+                    if (negator.NegateIfNeeded(current.Equals(value0) || current.Equals(value1)))
                     {
                         return i;
                     }
@@ -1455,7 +1455,7 @@ internal static int LastIndexOfAnyValueType<T, C>(ref T searchSpace, T value0, T
                 do
                 {
                     current = Vector256.LoadUnsafe(ref currentSearchSpace);
-                    equals = comparer.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1));
+                    equals = negator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1));
                     if (equals == Vector256<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<T>.Count);
@@ -1470,7 +1470,7 @@ internal static int LastIndexOfAnyValueType<T, C>(ref T searchSpace, T value0, T
                 if ((uint)length % Vector256<T>.Count != 0)
                 {
                     current = Vector256.LoadUnsafe(ref searchSpace);
-                    equals = comparer.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1));
+                    equals = negator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1));
                     if (equals != Vector256<T>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
@@ -1486,7 +1486,7 @@ internal static int LastIndexOfAnyValueType<T, C>(ref T searchSpace, T value0, T
                 do
                 {
                     current = Vector128.LoadUnsafe(ref currentSearchSpace);
-                    equals = comparer.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1));
+                    equals = negator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1));
                     if (equals == Vector128<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<T>.Count);
@@ -1501,7 +1501,7 @@ internal static int LastIndexOfAnyValueType<T, C>(ref T searchSpace, T value0, T
                 if ((uint)length % Vector128<T>.Count != 0)
                 {
                     current = Vector128.LoadUnsafe(ref searchSpace);
-                    equals = comparer.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1));
+                    equals = negator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1));
                     if (equals != Vector128<T>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
@@ -1513,21 +1513,21 @@ internal static int LastIndexOfAnyValueType<T, C>(ref T searchSpace, T value0, T
         }
 
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        internal static int LastIndexOfAnyValueType<T, C>(ref T searchSpace, T value0, T value1, T value2, int length)
+        internal static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T value1, T value2, int length)
             where T : struct, IEquatable<T>
-            where C : struct, IVectorEqualityComparer<T>
+            where N : struct, INegator<T>
         {
             Debug.Assert(length >= 0, "Expected non-negative length");
             Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types");
 
-            C comparer = default;
+            N negator = default;
 
             if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
             {
                 for (int i = length - 1; i >= 0; i--)
                 {
                     T current = Unsafe.Add(ref searchSpace, i);
-                    if (comparer.NegateIfNeeded(current.Equals(value0) || current.Equals(value1) || current.Equals(value2)))
+                    if (negator.NegateIfNeeded(current.Equals(value0) || current.Equals(value1) || current.Equals(value2)))
                     {
                         return i;
                     }
@@ -1542,7 +1542,7 @@ internal static int LastIndexOfAnyValueType<T, C>(ref T searchSpace, T value0, T
                 do
                 {
                     current = Vector256.LoadUnsafe(ref currentSearchSpace);
-                    equals = comparer.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) | Vector256.Equals(current, values2));
+                    equals = negator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) | Vector256.Equals(current, values2));
                     if (equals == Vector256<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<T>.Count);
@@ -1557,7 +1557,7 @@ internal static int LastIndexOfAnyValueType<T, C>(ref T searchSpace, T value0, T
                 if ((uint)length % Vector256<T>.Count != 0)
                 {
                     current = Vector256.LoadUnsafe(ref searchSpace);
-                    equals = comparer.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) | Vector256.Equals(current, values2));
+                    equals = negator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) | Vector256.Equals(current, values2));
                     if (equals != Vector256<T>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
@@ -1573,7 +1573,7 @@ internal static int LastIndexOfAnyValueType<T, C>(ref T searchSpace, T value0, T
                 do
                 {
                     current = Vector128.LoadUnsafe(ref currentSearchSpace);
-                    equals = comparer.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) | Vector128.Equals(current, values2));
+                    equals = negator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) | Vector128.Equals(current, values2));
                     if (equals == Vector128<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<T>.Count);
@@ -1588,7 +1588,7 @@ internal static int LastIndexOfAnyValueType<T, C>(ref T searchSpace, T value0, T
                 if ((uint)length % Vector128<T>.Count != 0)
                 {
                     current = Vector128.LoadUnsafe(ref searchSpace);
-                    equals = comparer.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) | Vector128.Equals(current, values2));
+                    equals = negator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) | Vector128.Equals(current, values2));
                     if (equals != Vector128<T>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
@@ -1615,21 +1615,21 @@ private static int ComputeLastIndex<T>(ref T searchSpace, ref T current, Vector2
             return (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / Unsafe.SizeOf<T>()) + index;
         }
 
-        internal interface IVectorEqualityComparer<T> where T : struct, IEquatable<T>
+        internal interface INegator<T> where T : struct, IEquatable<T>
         {
             bool NegateIfNeeded(bool equals);
             Vector128<T> NegateIfNeeded(Vector128<T> equals);
             Vector256<T> NegateIfNeeded(Vector256<T> equals);
         }
 
-        internal readonly struct DefaultEqualityComparer<T> : IVectorEqualityComparer<T> where T : struct, IEquatable<T>
+        internal readonly struct DontNegate<T> : INegator<T> where T : struct, IEquatable<T>
         {
             public bool NegateIfNeeded(bool equals) => equals;
             public Vector128<T> NegateIfNeeded(Vector128<T> equals) => equals;
             public Vector256<T> NegateIfNeeded(Vector256<T> equals) => equals;
         }
 
-        internal readonly struct ExceptEqualityComparer<T> : IVectorEqualityComparer<T> where T : struct, IEquatable<T>
+        internal readonly struct Negate<T> : INegator<T> where T : struct, IEquatable<T>
         {
             public bool NegateIfNeeded(bool equals) => !equals;
             public Vector128<T> NegateIfNeeded(Vector128<T> equals) => ~equals;
diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs
index f9c06bc6b9a508..709834937011c1 100644
--- a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs
@@ -282,7 +282,7 @@ public int IndexOf(string value, int startIndex, int count, StringComparison com
         // index within the string.
         //
         public int LastIndexOf(char value)
-            => SpanHelpers.LastIndexOfValueType<short, SpanHelpers.DefaultEqualityComparer<short>>(
+            => SpanHelpers.LastIndexOfValueType<short, SpanHelpers.DontNegate<short>>(
                     ref Unsafe.As<char, short>(ref _firstChar), (short)value, Length);
 
         public int LastIndexOf(char value, int startIndex)
@@ -308,7 +308,7 @@ public unsafe int LastIndexOf(char value, int startIndex, int count)
             }
 
             int startSearchAt = startIndex + 1 - count;
-            int result = SpanHelpers.LastIndexOfValueType<short, SpanHelpers.DefaultEqualityComparer<short>>
+            int result = SpanHelpers.LastIndexOfValueType<short, SpanHelpers.DontNegate<short>>
                 (ref Unsafe.As<char, short>(ref Unsafe.Add(ref _firstChar, startSearchAt)), (short)value, count);
 
             return result < 0 ? result : result + startSearchAt;

From 628d4295559da929aadfa6b27e177e0bd1881b33 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Fri, 12 Aug 2022 10:22:13 +0200
Subject: [PATCH 09/34] vectorize Contains

---
 .../src/System/MemoryExtensions.cs            |  60 ++++-------
 .../src/System/SpanHelpers.Byte.cs            | 100 ------------------
 .../src/System/SpanHelpers.Char.cs            |  93 ----------------
 .../src/System/SpanHelpers.T.cs               |  79 ++++++++++++++
 .../src/System/String.Searching.cs            |   3 +-
 5 files changed, 99 insertions(+), 236 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
index 9e2f22cc7d21dc..fc517ac6899b2d 100644
--- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
@@ -265,36 +265,7 @@ public static ReadOnlyMemory<char> AsMemory(this string? text, Range range)
         /// <param name="value">The value to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static bool Contains<T>(this Span<T> span, T value) where T : IEquatable<T>?
-        {
-            if (RuntimeHelpers.IsBitwiseEquatable<T>())
-            {
-                if (Unsafe.SizeOf<T>() == sizeof(byte))
-                    return SpanHelpers.Contains(
-                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, byte>(ref value),
-                        span.Length);
-
-                if (Unsafe.SizeOf<T>() == sizeof(char))
-                    return SpanHelpers.Contains(
-                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, char>(ref value),
-                        span.Length);
-
-                if (Unsafe.SizeOf<T>() == sizeof(int))
-                    return 0 <= SpanHelpers.IndexOfValueType(
-                        ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, int>(ref value),
-                        span.Length);
-
-                if (Unsafe.SizeOf<T>() == sizeof(long))
-                    return 0 <= SpanHelpers.IndexOfValueType(
-                        ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, long>(ref value),
-                        span.Length);
-            }
-
-            return SpanHelpers.Contains(ref MemoryMarshal.GetReference(span), value, span.Length);
-        }
+            => Contains((ReadOnlySpan<T>)span, value);
 
         /// <summary>
         /// Searches for the specified value and returns true if found. If not found, returns false. Values are compared using IEquatable{T}.Equals(T).
@@ -308,28 +279,33 @@ public static bool Contains<T>(this ReadOnlySpan<T> span, T value) where T : IEq
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
-                    return SpanHelpers.Contains(
+                {
+                    return SpanHelpers.ContainsValueType(
                         ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, byte>(ref value),
                         span.Length);
-
-                if (Unsafe.SizeOf<T>() == sizeof(char))
-                    return SpanHelpers.Contains(
-                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, char>(ref value),
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
+                {
+                    return SpanHelpers.ContainsValueType(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value),
                         span.Length);
-
-                if (Unsafe.SizeOf<T>() == sizeof(int))
-                    return 0 <= SpanHelpers.IndexOfValueType(
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(int))
+                {
+                    return SpanHelpers.ContainsValueType(
                         ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, int>(ref value),
                         span.Length);
-
-                if (Unsafe.SizeOf<T>() == sizeof(long))
-                    return 0 <= SpanHelpers.IndexOfValueType(
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(long))
+                {
+                    return SpanHelpers.ContainsValueType(
                         ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, long>(ref value),
                         span.Length);
+                }
             }
 
             return SpanHelpers.Contains(ref MemoryMarshal.GetReference(span), value, span.Length);
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
index b66137abee2d4d..0417b116ea8570 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
@@ -333,106 +333,6 @@ ref Unsafe.Add(ref searchSpace, offset + bitPos),
             }
         }
 
-        // Adapted from IndexOf(...)
-        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        public static bool Contains(ref byte searchSpace, byte value, int length)
-        {
-            Debug.Assert(length >= 0);
-
-            uint uValue = value; // Use uint for comparisons to avoid unnecessary 8->32 extensions
-            nuint offset = 0; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations
-            nuint lengthToExamine = (uint)length;
-
-            if (Vector.IsHardwareAccelerated && length >= Vector<byte>.Count * 2)
-            {
-                lengthToExamine = UnalignedCountVector(ref searchSpace);
-            }
-
-            while (lengthToExamine >= 8)
-            {
-                lengthToExamine -= 8;
-                ref byte start = ref Unsafe.AddByteOffset(ref searchSpace, offset);
-
-                if (uValue == Unsafe.AddByteOffset(ref start, 0) ||
-                    uValue == Unsafe.AddByteOffset(ref start, 1) ||
-                    uValue == Unsafe.AddByteOffset(ref start, 2) ||
-                    uValue == Unsafe.AddByteOffset(ref start, 3) ||
-                    uValue == Unsafe.AddByteOffset(ref start, 4) ||
-                    uValue == Unsafe.AddByteOffset(ref start, 5) ||
-                    uValue == Unsafe.AddByteOffset(ref start, 6) ||
-                    uValue == Unsafe.AddByteOffset(ref start, 7))
-                {
-                    goto Found;
-                }
-
-                offset += 8;
-            }
-
-            if (lengthToExamine >= 4)
-            {
-                lengthToExamine -= 4;
-                ref byte start = ref Unsafe.AddByteOffset(ref searchSpace, offset);
-
-                if (uValue == Unsafe.AddByteOffset(ref start, 0) ||
-                    uValue == Unsafe.AddByteOffset(ref start, 1) ||
-                    uValue == Unsafe.AddByteOffset(ref start, 2) ||
-                    uValue == Unsafe.AddByteOffset(ref start, 3))
-                {
-                    goto Found;
-                }
-
-                offset += 4;
-            }
-
-            while (lengthToExamine > 0)
-            {
-                lengthToExamine--;
-
-                if (uValue == Unsafe.AddByteOffset(ref searchSpace, offset))
-                    goto Found;
-
-                offset++;
-            }
-
-            if (Vector.IsHardwareAccelerated && (offset < (uint)length))
-            {
-                lengthToExamine = ((uint)length - offset) & (nuint)~(Vector<byte>.Count - 1);
-
-                Vector<byte> values = new(value);
-                Vector<byte> matches;
-
-                while (offset < lengthToExamine)
-                {
-                    matches = Vector.Equals(values, LoadVector(ref searchSpace, offset));
-                    if (matches == Vector<byte>.Zero)
-                    {
-                        offset += (nuint)Vector<byte>.Count;
-                        continue;
-                    }
-
-                    goto Found;
-                }
-
-                // The total length is at least Vector<byte>.Count, so instead of falling back to a
-                // sequential scan for the remainder, we check the vector read from the end -- note: unaligned read necessary.
-                // We do this only if at least one element is left.
-                if (offset < (uint)length)
-                {
-                    offset = (uint)(length - Vector<byte>.Count);
-                    matches = Vector.Equals(values, LoadVector(ref searchSpace, offset));
-                    if (matches != Vector<byte>.Zero)
-                    {
-                        goto Found;
-                    }
-                }
-            }
-
-            return false;
-
-        Found:
-            return true;
-        }
-
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
         public static unsafe int IndexOf(ref byte searchSpace, byte value, int length)
         {
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
index aea2a3d48f32d0..7ae06287837b31 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
@@ -422,99 +422,6 @@ public static unsafe int SequenceCompareTo(ref char first, int firstLength, ref
             return lengthDelta;
         }
 
-        // Adapted from IndexOf(...)
-        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        public static unsafe bool Contains(ref char searchSpace, char value, int length)
-        {
-            Debug.Assert(length >= 0);
-
-            fixed (char* pChars = &searchSpace)
-            {
-                nuint offset = 0; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations
-                nuint lengthToExamine = (uint)length;
-
-                if (Vector.IsHardwareAccelerated && length >= Vector<ushort>.Count * 2)
-                {
-                    // Figure out how many characters to read sequentially until we are vector aligned
-                    // This is equivalent to:
-                    //         unaligned = ((int)pCh % Unsafe.SizeOf<Vector<ushort>>()) / ElementsPerByte
-                    //         length = (Vector<ushort>.Count - unaligned) % Vector<ushort>.Count
-                    const int ElementsPerByte = sizeof(ushort) / sizeof(byte);
-                    int unaligned = (int)((uint)((int)pChars & (Unsafe.SizeOf<Vector<ushort>>() - 1)) / ElementsPerByte);
-                    lengthToExamine = (uint)((Vector<ushort>.Count - unaligned) & (Vector<ushort>.Count - 1));
-                }
-
-                while (lengthToExamine >= 4)
-                {
-                    lengthToExamine -= 4;
-                    char* pStart = pChars + offset;
-
-                    if (value == pStart[0] ||
-                        value == pStart[1] ||
-                        value == pStart[2] ||
-                        value == pStart[3])
-                    {
-                        goto Found;
-                    }
-
-                    offset += 4;
-                }
-
-                while (lengthToExamine > 0)
-                {
-                    lengthToExamine--;
-
-                    if (value == pChars[offset])
-                        goto Found;
-
-                    offset++;
-                }
-
-                // We get past SequentialScan only if IsHardwareAccelerated is true. However, we still have the redundant check to allow
-                // the JIT to see that the code is unreachable and eliminate it when the platform does not have hardware acceleration.
-                if (Vector.IsHardwareAccelerated && (offset < (uint)length))
-                {
-                    // Get the highest multiple of Vector<ushort>.Count that is within the search space.
-                    // That will be how many times we iterate in the loop below.
-                    // This is equivalent to: lengthToExamine = Vector<ushort>.Count + ((uint)length - offset) / Vector<ushort>.Count)
-                    lengthToExamine = ((uint)length - offset) & (nuint)~(Vector<ushort>.Count - 1);
-
-                    Vector<ushort> values = new(value);
-                    Vector<ushort> matches;
-
-                    while (offset < lengthToExamine)
-                    {
-                        // Using Unsafe.Read instead of ReadUnaligned since the search space is pinned and pCh is always vector aligned
-                        Debug.Assert(((int)(pChars + offset) % Unsafe.SizeOf<Vector<ushort>>()) == 0);
-                        matches = Vector.Equals(values, Unsafe.Read<Vector<ushort>>(pChars + offset));
-                        if (matches == Vector<ushort>.Zero)
-                        {
-                            offset += (nuint)Vector<ushort>.Count;
-                            continue;
-                        }
-
-                        goto Found;
-                    }
-
-                    // The total length is at least Vector<ushort>.Count, so instead of falling back to a
-                    // sequential scan for the remainder, we check the vector read from the end -- note: unaligned read necessary.
-                    // We do this only if at least one element is left.
-                    if (offset < (uint)length)
-                    {
-                        matches = Vector.Equals(values, Unsafe.ReadUnaligned<Vector<ushort>>(pChars + (uint)length - (uint)Vector<ushort>.Count));
-                        if (matches != Vector<ushort>.Zero)
-                        {
-                            goto Found;
-                        }
-                    }
-                }
-
-                return false;
-
-            Found:
-                return true;
-            }
-        }
 
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
         public static unsafe int IndexOf(ref char searchSpace, char value, int length)
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
index 9521c836150273..85bfffa1c51ebb 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
@@ -1343,6 +1343,85 @@ public static int SequenceCompareTo<T>(ref T first, int firstLength, ref T secon
             return firstLength.CompareTo(secondLength);
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
+        internal static bool ContainsValueType<T>(ref T searchSpace, T value, int length) where T : struct, IEquatable<T>
+        {
+            Debug.Assert(length >= 0, "Expected non-negative length");
+            Debug.Assert(value is byte or short or int or long, "Expected caller to normalize to one of these types");
+
+            if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
+            {
+                for (int i = 0; i < length; i++)
+                {
+                    if (Unsafe.Add(ref searchSpace, i).Equals(value))
+                    {
+                        return true;
+                    }
+                }
+            }
+            else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
+            {
+                Vector256<T> equals, values = Vector256.Create(value);
+                ref T currentSearchSpace = ref searchSpace;
+                ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256<T>.Count);
+
+                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
+                do
+                {
+                    equals = Vector256.Equals(values, Vector256.LoadUnsafe(ref currentSearchSpace));
+                    if (equals == Vector256<T>.Zero)
+                    {
+                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector256<T>.Count);
+                        continue;
+                    }
+
+                    return true;
+                }
+                while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd));
+
+                // If any elements remain, process the last vector in the search space.
+                if ((uint)length % Vector256<T>.Count != 0)
+                {
+                    equals = Vector256.Equals(values, Vector256.LoadUnsafe(ref oneVectorAwayFromEnd));
+                    if (equals != Vector256<T>.Zero)
+                    {
+                        return true;
+                    }
+                }
+            }
+            else
+            {
+                Vector128<T> equals, values = Vector128.Create(value);
+                ref T currentSearchSpace = ref searchSpace;
+                ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128<T>.Count);
+
+                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
+                do
+                {
+                    equals = Vector128.Equals(values, Vector128.LoadUnsafe(ref currentSearchSpace));
+                    if (equals == Vector128<T>.Zero)
+                    {
+                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128<T>.Count);
+                        continue;
+                    }
+
+                    return true;
+                }
+                while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd));
+
+                // If any elements remain, process the first vector in the search space.
+                if ((uint)length % Vector128<T>.Count != 0)
+                {
+                    equals = Vector128.Equals(values, Vector128.LoadUnsafe(ref oneVectorAwayFromEnd));
+                    if (equals != Vector128<T>.Zero)
+                    {
+                        return true;
+                    }
+                }
+            }
+
+            return false;
+        }
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
         internal static int LastIndexOfValueType<T, N>(ref T searchSpace, T value, int length)
             where T : struct, IEquatable<T>
diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs
index 709834937011c1..713f5011d9803c 100644
--- a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs
@@ -27,7 +27,8 @@ public bool Contains(string value, StringComparison comparisonType)
 #pragma warning restore CA2249
         }
 
-        public bool Contains(char value) => SpanHelpers.Contains(ref _firstChar, value, Length);
+        public bool Contains(char value)
+            => SpanHelpers.ContainsValueType(ref Unsafe.As<char, short>(ref _firstChar), (short)value, Length);
 
         public bool Contains(char value, StringComparison comparisonType)
         {

From 114ca88924a7a3f48a14a9a9f56c99da3ca749ee Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Fri, 12 Aug 2022 13:13:15 +0200
Subject: [PATCH 10/34] hide the implementation details

---
 .../src/System/Array.cs                       |  8 ++--
 .../src/System/MemoryExtensions.cs            | 40 +++++++++----------
 .../src/System/SpanHelpers.Char.cs            |  6 +--
 .../src/System/SpanHelpers.T.cs               | 38 ++++++++++++++----
 .../src/System/String.Searching.cs            |  7 +---
 5 files changed, 59 insertions(+), 40 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/Array.cs b/src/libraries/System.Private.CoreLib/src/System/Array.cs
index 1406d248f70549..ca44a415cee188 100644
--- a/src/libraries/System.Private.CoreLib/src/System/Array.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/Array.cs
@@ -1586,7 +1586,7 @@ public static int LastIndexOf<T>(T[] array, T value, int startIndex, int count)
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
                     int endIndex = startIndex - count + 1;
-                    int result = SpanHelpers.LastIndexOfValueType<byte, SpanHelpers.DontNegate<byte>>(
+                    int result = SpanHelpers.LastIndexOfValueType(
                         ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<byte[]>(array)), endIndex),
                         Unsafe.As<T, byte>(ref value),
                         count);
@@ -1596,7 +1596,7 @@ ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<byte[]>(array))
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
                     int endIndex = startIndex - count + 1;
-                    int result = SpanHelpers.LastIndexOfValueType<short, SpanHelpers.DontNegate<short>>(
+                    int result = SpanHelpers.LastIndexOfValueType(
                         ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<short[]>(array)), endIndex),
                         Unsafe.As<T, short>(ref value),
                         count);
@@ -1606,7 +1606,7 @@ ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<short[]>(array)
                 else if (Unsafe.SizeOf<T>() == sizeof(int))
                 {
                     int endIndex = startIndex - count + 1;
-                    int result = SpanHelpers.LastIndexOfValueType<int, SpanHelpers.DontNegate<int>>(
+                    int result = SpanHelpers.LastIndexOfValueType(
                         ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<int[]>(array)), endIndex),
                         Unsafe.As<T, int>(ref value),
                         count);
@@ -1616,7 +1616,7 @@ ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<int[]>(array)),
                 else if (Unsafe.SizeOf<T>() == sizeof(long))
                 {
                     int endIndex = startIndex - count + 1;
-                    int result = SpanHelpers.LastIndexOfValueType<long, SpanHelpers.DontNegate<long>>(
+                    int result = SpanHelpers.LastIndexOfValueType(
                         ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<long[]>(array)), endIndex),
                         Unsafe.As<T, long>(ref value),
                         count);
diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
index fc517ac6899b2d..ddc609a60e1b14 100644
--- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
@@ -652,28 +652,28 @@ public static int LastIndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value) wh
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.LastIndexOfValueType<byte, SpanHelpers.Negate<byte>>(
+                    return SpanHelpers.LastIndexOfAnyExceptValueType(
                         ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, byte>(ref value),
                         span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    return SpanHelpers.LastIndexOfValueType<short, SpanHelpers.Negate<short>>(
+                    return SpanHelpers.LastIndexOfAnyExceptValueType(
                         ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, short>(ref value),
                         span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(int))
                 {
-                    return SpanHelpers.LastIndexOfValueType<int, SpanHelpers.Negate<int>>(
+                    return SpanHelpers.LastIndexOfAnyExceptValueType(
                         ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, int>(ref value),
                         span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(long))
                 {
-                    return SpanHelpers.LastIndexOfValueType<long, SpanHelpers.Negate<long>>(
+                    return SpanHelpers.LastIndexOfAnyExceptValueType(
                         ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, long>(ref value),
                         span.Length);
@@ -706,7 +706,7 @@ public static int LastIndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value0, T
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType<byte, SpanHelpers.Negate<byte>>(
+                    return SpanHelpers.LastIndexOfAnyExceptValueType(
                         ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, byte>(ref value0),
                         Unsafe.As<T, byte>(ref value1),
@@ -714,7 +714,7 @@ ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType<short, SpanHelpers.Negate<short>>(
+                    return SpanHelpers.LastIndexOfAnyExceptValueType(
                         ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, short>(ref value0),
                         Unsafe.As<T, short>(ref value1),
@@ -722,7 +722,7 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(int))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType<int, SpanHelpers.Negate<int>>(
+                    return SpanHelpers.LastIndexOfAnyExceptValueType(
                         ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, int>(ref value0),
                         Unsafe.As<T, int>(ref value1),
@@ -730,7 +730,7 @@ ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(long))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType<long, SpanHelpers.Negate<long>>(
+                    return SpanHelpers.LastIndexOfAnyExceptValueType(
                         ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, long>(ref value0),
                         Unsafe.As<T, long>(ref value1),
@@ -766,7 +766,7 @@ public static int LastIndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value0, T
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType<byte, SpanHelpers.Negate<byte>>(
+                    return SpanHelpers.LastIndexOfAnyExceptValueType(
                         ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, byte>(ref value0),
                         Unsafe.As<T, byte>(ref value1),
@@ -775,7 +775,7 @@ ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType<short, SpanHelpers.Negate<short>>(
+                    return SpanHelpers.LastIndexOfAnyExceptValueType(
                         ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, short>(ref value0),
                         Unsafe.As<T, short>(ref value1),
@@ -953,28 +953,28 @@ public static int LastIndexOf<T>(this ReadOnlySpan<T> span, T value) where T : I
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.LastIndexOfValueType<byte, SpanHelpers.DontNegate<byte>>(
+                    return SpanHelpers.LastIndexOfValueType(
                             ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                             Unsafe.As<T, byte>(ref value),
                             span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    return SpanHelpers.LastIndexOfValueType<short, SpanHelpers.DontNegate<short>>(
+                    return SpanHelpers.LastIndexOfValueType(
                         ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, short>(ref value),
                         span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(int))
                 {
-                    return SpanHelpers.LastIndexOfValueType<int, SpanHelpers.DontNegate<int>>(
+                    return SpanHelpers.LastIndexOfValueType(
                         ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, int>(ref value),
                         span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(long))
                 {
-                    return SpanHelpers.LastIndexOfValueType<long, SpanHelpers.DontNegate<long>>(
+                    return SpanHelpers.LastIndexOfValueType(
                         ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, long>(ref value),
                         span.Length);
@@ -1318,7 +1318,7 @@ public static int LastIndexOfAny<T>(this ReadOnlySpan<T> span, T value0, T value
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType<byte, SpanHelpers.DontNegate<byte>>(
+                    return SpanHelpers.LastIndexOfAnyValueType(
                         ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, byte>(ref value0),
                         Unsafe.As<T, byte>(ref value1),
@@ -1326,7 +1326,7 @@ ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType<short, SpanHelpers.DontNegate<short>>(
+                    return SpanHelpers.LastIndexOfAnyValueType(
                         ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, short>(ref value0),
                         Unsafe.As<T, short>(ref value1),
@@ -1334,7 +1334,7 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(int))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType<int, SpanHelpers.DontNegate<int>>(
+                    return SpanHelpers.LastIndexOfAnyValueType(
                         ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, int>(ref value0),
                         Unsafe.As<T, int>(ref value1),
@@ -1342,7 +1342,7 @@ ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(long))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType<long, SpanHelpers.DontNegate<long>>(
+                    return SpanHelpers.LastIndexOfAnyValueType(
                         ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, long>(ref value0),
                         Unsafe.As<T, long>(ref value1),
@@ -1367,7 +1367,7 @@ public static int LastIndexOfAny<T>(this ReadOnlySpan<T> span, T value0, T value
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType<byte, SpanHelpers.DontNegate<byte>>(
+                    return SpanHelpers.LastIndexOfAnyValueType(
                         ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, byte>(ref value0),
                         Unsafe.As<T, byte>(ref value1),
@@ -1376,7 +1376,7 @@ ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType<short, SpanHelpers.DontNegate<short>>(
+                    return SpanHelpers.LastIndexOfAnyValueType(
                         ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, short>(ref value0),
                         Unsafe.As<T, short>(ref value1),
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
index 7ae06287837b31..114d00baf831b3 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
@@ -214,8 +214,7 @@ public static int LastIndexOf(ref char searchSpace, int searchSpaceLength, ref c
 
             int valueTailLength = valueLength - 1;
             if (valueTailLength == 0)
-                return LastIndexOfValueType<short, DontNegate<short>>(
-                    ref Unsafe.As<char, short>(ref searchSpace), (short)value, searchSpaceLength); // for single-char values use plain LastIndexOf
+                return LastIndexOfValueType(ref Unsafe.As<char, short>(ref searchSpace), (short)value, searchSpaceLength); // for single-char values use plain LastIndexOf
 
             int offset = 0;
             char valueHead = value;
@@ -235,8 +234,7 @@ public static int LastIndexOf(ref char searchSpace, int searchSpaceLength, ref c
                     break;  // The unsearched portion is now shorter than the sequence we're looking for. So it can't be there.
 
                 // Do a quick search for the first element of "value".
-                int relativeIndex = LastIndexOfValueType<short, DontNegate<short>>(
-                    ref Unsafe.As<char, short>(ref searchSpace), (short)valueHead, remainingSearchSpaceLength);
+                int relativeIndex = LastIndexOfValueType(ref Unsafe.As<char, short>(ref searchSpace), (short)valueHead, remainingSearchSpaceLength);
                 if (relativeIndex == -1)
                     break;
 
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
index 85bfffa1c51ebb..4a3d15bd439f24 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
@@ -1422,8 +1422,17 @@ internal static bool ContainsValueType<T>(ref T searchSpace, T value, int length
 
             return false;
         }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static int LastIndexOfValueType<T>(ref T searchSpace, T value, int length) where T : struct, IEquatable<T>
+            => LastIndexOfValueType<T, DontNegate<T>>(ref searchSpace, value, length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static int LastIndexOfAnyExceptValueType<T>(ref T searchSpace, T value, int length) where T : struct, IEquatable<T>
+            => LastIndexOfValueType<T, Negate<T>>(ref searchSpace, value, length);
+
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        internal static int LastIndexOfValueType<T, N>(ref T searchSpace, T value, int length)
+        private static int LastIndexOfValueType<T, N>(ref T searchSpace, T value, int length)
             where T : struct, IEquatable<T>
             where N : struct, INegator<T>
         {
@@ -1504,8 +1513,16 @@ internal static int LastIndexOfValueType<T, N>(ref T searchSpace, T value, int l
             return -1;
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static int LastIndexOfAnyValueType<T>(ref T searchSpace, T value0, T value1, int length) where T : struct, IEquatable<T>
+            => LastIndexOfAnyValueType<T, DontNegate<T>>(ref searchSpace, value0, value1, length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static int LastIndexOfAnyExceptValueType<T>(ref T searchSpace, T value0, T value1, int length) where T : struct, IEquatable<T>
+            => LastIndexOfAnyValueType<T, Negate<T>>(ref searchSpace, value0, value1, length);
+
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        internal static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T value1, int length)
+        private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T value1, int length)
             where T : struct, IEquatable<T>
             where N : struct, INegator<T>
         {
@@ -1591,8 +1608,16 @@ internal static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
             return -1;
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static int LastIndexOfAnyValueType<T>(ref T searchSpace, T value0, T value1, T value2, int length) where T : struct, IEquatable<T>
+            => LastIndexOfAnyValueType<T, DontNegate<T>>(ref searchSpace, value0, value1, value2, length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static int LastIndexOfAnyExceptValueType<T>(ref T searchSpace, T value0, T value1, T value2, int length) where T : struct, IEquatable<T>
+            => LastIndexOfAnyValueType<T, Negate<T>>(ref searchSpace, value0, value1, value2, length);
+
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        internal static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T value1, T value2, int length)
+        private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T value1, T value2, int length)
             where T : struct, IEquatable<T>
             where N : struct, INegator<T>
         {
@@ -1694,26 +1719,25 @@ private static int ComputeLastIndex<T>(ref T searchSpace, ref T current, Vector2
             return (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / Unsafe.SizeOf<T>()) + index;
         }
 
-        internal interface INegator<T> where T : struct, IEquatable<T>
+        private interface INegator<T> where T : struct, IEquatable<T>
         {
             bool NegateIfNeeded(bool equals);
             Vector128<T> NegateIfNeeded(Vector128<T> equals);
             Vector256<T> NegateIfNeeded(Vector256<T> equals);
         }
 
-        internal readonly struct DontNegate<T> : INegator<T> where T : struct, IEquatable<T>
+        private readonly struct DontNegate<T> : INegator<T> where T : struct, IEquatable<T>
         {
             public bool NegateIfNeeded(bool equals) => equals;
             public Vector128<T> NegateIfNeeded(Vector128<T> equals) => equals;
             public Vector256<T> NegateIfNeeded(Vector256<T> equals) => equals;
         }
 
-        internal readonly struct Negate<T> : INegator<T> where T : struct, IEquatable<T>
+        private readonly struct Negate<T> : INegator<T> where T : struct, IEquatable<T>
         {
             public bool NegateIfNeeded(bool equals) => !equals;
             public Vector128<T> NegateIfNeeded(Vector128<T> equals) => ~equals;
             public Vector256<T> NegateIfNeeded(Vector256<T> equals) => ~equals;
-
         }
     }
 }
diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs
index 713f5011d9803c..fb763edacc8e00 100644
--- a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs
@@ -281,10 +281,8 @@ public int IndexOf(string value, int startIndex, int count, StringComparison com
         // The search starts at startIndex and runs backwards to startIndex - count + 1.
         // The character at position startIndex is included in the search.  startIndex is the larger
         // index within the string.
-        //
         public int LastIndexOf(char value)
-            => SpanHelpers.LastIndexOfValueType<short, SpanHelpers.DontNegate<short>>(
-                    ref Unsafe.As<char, short>(ref _firstChar), (short)value, Length);
+            => SpanHelpers.LastIndexOfValueType(ref Unsafe.As<char, short>(ref _firstChar), (short)value, Length);
 
         public int LastIndexOf(char value, int startIndex)
         {
@@ -309,8 +307,7 @@ public unsafe int LastIndexOf(char value, int startIndex, int count)
             }
 
             int startSearchAt = startIndex + 1 - count;
-            int result = SpanHelpers.LastIndexOfValueType<short, SpanHelpers.DontNegate<short>>
-                (ref Unsafe.As<char, short>(ref Unsafe.Add(ref _firstChar, startSearchAt)), (short)value, count);
+            int result = SpanHelpers.LastIndexOfValueType(ref Unsafe.As<char, short>(ref Unsafe.Add(ref _firstChar, startSearchAt)), (short)value, count);
 
             return result < 0 ? result : result + startSearchAt;
         }

From 95a1b62de329541fa060b0573b143e3bc6fb088c Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Fri, 12 Aug 2022 13:40:55 +0200
Subject: [PATCH 11/34] address review from Jan, don't cast Span to ROS.
 Introduce new helper and avoid duplication by calling it from both Span and
 ROS

---
 .../src/System/MemoryExtensions.cs            | 330 ++++++------------
 1 file changed, 115 insertions(+), 215 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
index ddc609a60e1b14..83c767b3396231 100644
--- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
@@ -265,7 +265,7 @@ public static ReadOnlyMemory<char> AsMemory(this string? text, Range range)
         /// <param name="value">The value to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static bool Contains<T>(this Span<T> span, T value) where T : IEquatable<T>?
-            => Contains((ReadOnlySpan<T>)span, value);
+            => Contains(ref MemoryMarshal.GetReference(span), value, span.Length);
 
         /// <summary>
         /// Searches for the specified value and returns true if found. If not found, returns false. Values are compared using IEquatable{T}.Equals(T).
@@ -275,40 +275,32 @@ public static bool Contains<T>(this Span<T> span, T value) where T : IEquatable<
         /// <param name="value">The value to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static bool Contains<T>(this ReadOnlySpan<T> span, T value) where T : IEquatable<T>?
+            => Contains(ref MemoryMarshal.GetReference(span), value, span.Length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static bool Contains<T>(ref T searchSpace, T value, int length) where T : IEquatable<T>?
         {
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.ContainsValueType(
-                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, byte>(ref value),
-                        span.Length);
+                    return SpanHelpers.ContainsValueType(ref Unsafe.As<T, byte>(ref searchSpace), Unsafe.As<T, byte>(ref value), length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    return SpanHelpers.ContainsValueType(
-                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, short>(ref value),
-                        span.Length);
+                    return SpanHelpers.ContainsValueType(ref Unsafe.As<T, short>(ref searchSpace), Unsafe.As<T, short>(ref value), length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(int))
                 {
-                    return SpanHelpers.ContainsValueType(
-                        ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, int>(ref value),
-                        span.Length);
+                    return SpanHelpers.ContainsValueType(ref Unsafe.As<T, int>(ref searchSpace), Unsafe.As<T, int>(ref value), length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(long))
                 {
-                    return SpanHelpers.ContainsValueType(
-                        ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, long>(ref value),
-                        span.Length);
+                    return SpanHelpers.ContainsValueType(ref Unsafe.As<T, long>(ref searchSpace), Unsafe.As<T, long>(ref value), length);
                 }
             }
 
-            return SpanHelpers.Contains(ref MemoryMarshal.GetReference(span), value, span.Length);
+            return SpanHelpers.Contains(ref searchSpace, value, length);
         }
 
         /// <summary>
@@ -318,36 +310,7 @@ ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
         /// <param name="value">The value to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int IndexOf<T>(this Span<T> span, T value) where T : IEquatable<T>?
-        {
-            if (RuntimeHelpers.IsBitwiseEquatable<T>())
-            {
-                if (Unsafe.SizeOf<T>() == sizeof(byte))
-                    return SpanHelpers.IndexOf(
-                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, byte>(ref value),
-                        span.Length);
-
-                if (Unsafe.SizeOf<T>() == sizeof(char))
-                    return SpanHelpers.IndexOf(
-                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, char>(ref value),
-                        span.Length);
-
-                if (Unsafe.SizeOf<T>() == sizeof(int))
-                    return SpanHelpers.IndexOfValueType(
-                        ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, int>(ref value),
-                        span.Length);
-
-                if (Unsafe.SizeOf<T>() == sizeof(long))
-                    return SpanHelpers.IndexOfValueType(
-                        ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, long>(ref value),
-                        span.Length);
-            }
-
-            return SpanHelpers.IndexOf(ref MemoryMarshal.GetReference(span), value, span.Length);
-        }
+            => IndexOf(ref MemoryMarshal.GetReference(span), value, span.Length);
 
         /// <summary>
         /// Searches for the specified sequence and returns the index of its first occurrence. If not found, returns -1. Values are compared using IEquatable{T}.Equals(T).
@@ -356,26 +319,7 @@ ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
         /// <param name="value">The sequence to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int IndexOf<T>(this Span<T> span, ReadOnlySpan<T> value) where T : IEquatable<T>?
-        {
-            if (RuntimeHelpers.IsBitwiseEquatable<T>())
-            {
-                if (Unsafe.SizeOf<T>() == sizeof(byte))
-                    return SpanHelpers.IndexOf(
-                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
-                        span.Length,
-                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(value)),
-                        value.Length);
-
-                if (Unsafe.SizeOf<T>() == sizeof(char))
-                    return SpanHelpers.IndexOf(
-                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
-                        span.Length,
-                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(value)),
-                        value.Length);
-            }
-
-            return SpanHelpers.IndexOf(ref MemoryMarshal.GetReference(span), span.Length, ref MemoryMarshal.GetReference(value), value.Length);
-        }
+            => IndexOf(ref MemoryMarshal.GetReference(span), span.Length, ref MemoryMarshal.GetReference(value), value.Length);
 
         /// <summary>
         /// Searches for the specified value and returns the index of its last occurrence. If not found, returns -1. Values are compared using IEquatable{T}.Equals(T).
@@ -384,7 +328,7 @@ ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(value)),
         /// <param name="value">The value to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOf<T>(this Span<T> span, T value) where T : IEquatable<T>?
-            => LastIndexOf((ReadOnlySpan<T>)span, value);
+            => LastIndexOf(ref MemoryMarshal.GetReference(span), value, span.Length);
 
         /// <summary>
         /// Searches for the specified sequence and returns the index of its last occurrence. If not found, returns -1. Values are compared using IEquatable{T}.Equals(T).
@@ -393,16 +337,7 @@ public static int LastIndexOf<T>(this Span<T> span, T value) where T : IEquatabl
         /// <param name="value">The sequence to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOf<T>(this Span<T> span, ReadOnlySpan<T> value) where T : IEquatable<T>?
-        {
-            if (Unsafe.SizeOf<T>() == sizeof(byte) && RuntimeHelpers.IsBitwiseEquatable<T>())
-                return SpanHelpers.LastIndexOf(
-                    ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
-                    span.Length,
-                    ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(value)),
-                    value.Length);
-
-            return SpanHelpers.LastIndexOf<T>(ref MemoryMarshal.GetReference(span), span.Length, ref MemoryMarshal.GetReference(value), value.Length);
-        }
+            => LastIndexOf(ref MemoryMarshal.GetReference(span), span.Length, ref MemoryMarshal.GetReference(value), value.Length);
 
         /// <summary>Searches for the first index of any value other than the specified <paramref name="value"/>.</summary>
         /// <typeparam name="T">The type of the span and values.</typeparam>
@@ -412,8 +347,8 @@ ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(value)),
         /// The index in the span of the first occurrence of any value other than <paramref name="value"/>.
         /// If all of the values are <paramref name="value"/>, returns -1.
         /// </returns>
-        public static int IndexOfAnyExcept<T>(this Span<T> span, T value) where T : IEquatable<T>? =>
-            IndexOfAnyExcept((ReadOnlySpan<T>)span, value);
+        public static int IndexOfAnyExcept<T>(this Span<T> span, T value) where T : IEquatable<T>?
+            => IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), value, span.Length);
 
         /// <summary>Searches for the first index of any value other than the specified <paramref name="value0"/> or <paramref name="value1"/>.</summary>
         /// <typeparam name="T">The type of the span and values.</typeparam>
@@ -424,8 +359,8 @@ public static int IndexOfAnyExcept<T>(this Span<T> span, T value) where T : IEqu
         /// The index in the span of the first occurrence of any value other than <paramref name="value0"/> and <paramref name="value1"/>.
         /// If all of the values are <paramref name="value0"/> or <paramref name="value1"/>, returns -1.
         /// </returns>
-        public static int IndexOfAnyExcept<T>(this Span<T> span, T value0, T value1) where T : IEquatable<T>? =>
-            IndexOfAnyExcept((ReadOnlySpan<T>)span, value0, value1);
+        public static int IndexOfAnyExcept<T>(this Span<T> span, T value0, T value1) where T : IEquatable<T>?
+            => IndexOfAnyExcept((ReadOnlySpan<T>)span, value0, value1);
 
         /// <summary>Searches for the first index of any value other than the specified <paramref name="value0"/>, <paramref name="value1"/>, or <paramref name="value2"/>.</summary>
         /// <typeparam name="T">The type of the span and values.</typeparam>
@@ -437,8 +372,8 @@ public static int IndexOfAnyExcept<T>(this Span<T> span, T value0, T value1) whe
         /// The index in the span of the first occurrence of any value other than <paramref name="value0"/>, <paramref name="value1"/>, and <paramref name="value2"/>.
         /// If all of the values are <paramref name="value0"/>, <paramref name="value1"/>, and <paramref name="value2"/>, returns -1.
         /// </returns>
-        public static int IndexOfAnyExcept<T>(this Span<T> span, T value0, T value1, T value2) where T : IEquatable<T>? =>
-            IndexOfAnyExcept((ReadOnlySpan<T>)span, value0, value1, value2);
+        public static int IndexOfAnyExcept<T>(this Span<T> span, T value0, T value1, T value2) where T : IEquatable<T>?
+            => IndexOfAnyExcept((ReadOnlySpan<T>)span, value0, value1, value2);
 
         /// <summary>Searches for the first index of any value other than the specified <paramref name="values"/>.</summary>
         /// <typeparam name="T">The type of the span and values.</typeparam>
@@ -448,8 +383,8 @@ public static int IndexOfAnyExcept<T>(this Span<T> span, T value0, T value1, T v
         /// The index in the span of the first occurrence of any value other than those in <paramref name="values"/>.
         /// If all of the values are in <paramref name="values"/>, returns -1.
         /// </returns>
-        public static int IndexOfAnyExcept<T>(this Span<T> span, ReadOnlySpan<T> values) where T : IEquatable<T>? =>
-            IndexOfAnyExcept((ReadOnlySpan<T>)span, values);
+        public static int IndexOfAnyExcept<T>(this Span<T> span, ReadOnlySpan<T> values) where T : IEquatable<T>?
+            => IndexOfAnyExcept((ReadOnlySpan<T>)span, values);
 
         /// <summary>Searches for the first index of any value other than the specified <paramref name="value"/>.</summary>
         /// <typeparam name="T">The type of the span and values.</typeparam>
@@ -461,46 +396,32 @@ public static int IndexOfAnyExcept<T>(this Span<T> span, ReadOnlySpan<T> values)
         /// </returns>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int IndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value) where T : IEquatable<T>?
+            => IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), value, span.Length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static int IndexOfAnyExcept<T>(ref T searchSpace, T value, int length) where T : IEquatable<T>?
         {
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.IndexOfAnyExceptValueType(
-                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, byte>(ref value),
-                        span.Length);
+                    return SpanHelpers.IndexOfAnyExceptValueType(ref Unsafe.As<T, byte>(ref searchSpace), Unsafe.As<T, byte>(ref value), length);
                 }
-
-                if (Unsafe.SizeOf<T>() == sizeof(short))
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    return SpanHelpers.IndexOfAnyExceptValueType(
-                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, short>(ref value),
-                        span.Length);
+                    return SpanHelpers.IndexOfAnyExceptValueType(ref Unsafe.As<T, short>(ref searchSpace), Unsafe.As<T, short>(ref value), length);
                 }
-
-                if (Unsafe.SizeOf<T>() == sizeof(int))
+                else if (Unsafe.SizeOf<T>() == sizeof(int))
                 {
-                    return SpanHelpers.IndexOfAnyExceptValueType(
-                        ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, int>(ref value),
-                        span.Length);
+                    return SpanHelpers.IndexOfAnyExceptValueType(ref Unsafe.As<T, int>(ref searchSpace), Unsafe.As<T, int>(ref value), length);
                 }
-
-                if (Unsafe.SizeOf<T>() == sizeof(long))
+                else if (Unsafe.SizeOf<T>() == sizeof(long))
                 {
-                    return SpanHelpers.IndexOfAnyExceptValueType(
-                        ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, long>(ref value),
-                        span.Length);
+                    return SpanHelpers.IndexOfAnyExceptValueType(ref Unsafe.As<T, long>(ref searchSpace), Unsafe.As<T, long>(ref value), length);
                 }
             }
 
-            return SpanHelpers.IndexOfAnyExcept(
-                ref MemoryMarshal.GetReference(span),
-                value,
-                span.Length);
+            return SpanHelpers.IndexOfAnyExcept(ref searchSpace, value, length);
         }
 
         /// <summary>Searches for the first index of any value other than the specified <paramref name="value0"/> or <paramref name="value1"/>.</summary>
@@ -599,8 +520,8 @@ public static int IndexOfAnyExcept<T>(this ReadOnlySpan<T> span, ReadOnlySpan<T>
         /// The index in the span of the last occurrence of any value other than <paramref name="value"/>.
         /// If all of the values are <paramref name="value"/>, returns -1.
         /// </returns>
-        public static int LastIndexOfAnyExcept<T>(this Span<T> span, T value) where T : IEquatable<T>? =>
-            LastIndexOfAnyExcept((ReadOnlySpan<T>)span, value);
+        public static int LastIndexOfAnyExcept<T>(this Span<T> span, T value) where T : IEquatable<T>?
+            => LastIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), value, span.Length);
 
         /// <summary>Searches for the last index of any value other than the specified <paramref name="value0"/> or <paramref name="value1"/>.</summary>
         /// <typeparam name="T">The type of the span and values.</typeparam>
@@ -611,8 +532,8 @@ public static int LastIndexOfAnyExcept<T>(this Span<T> span, T value) where T :
         /// The index in the span of the last occurrence of any value other than <paramref name="value0"/> and <paramref name="value1"/>.
         /// If all of the values are <paramref name="value0"/> or <paramref name="value1"/>, returns -1.
         /// </returns>
-        public static int LastIndexOfAnyExcept<T>(this Span<T> span, T value0, T value1) where T : IEquatable<T>? =>
-            LastIndexOfAnyExcept((ReadOnlySpan<T>)span, value0, value1);
+        public static int LastIndexOfAnyExcept<T>(this Span<T> span, T value0, T value1) where T : IEquatable<T>?
+            => LastIndexOfAnyExcept((ReadOnlySpan<T>)span, value0, value1);
 
         /// <summary>Searches for the last index of any value other than the specified <paramref name="value0"/>, <paramref name="value1"/>, or <paramref name="value2"/>.</summary>
         /// <typeparam name="T">The type of the span and values.</typeparam>
@@ -647,42 +568,33 @@ public static int LastIndexOfAnyExcept<T>(this Span<T> span, ReadOnlySpan<T> val
         /// If all of the values are <paramref name="value"/>, returns -1.
         /// </returns>
         public static int LastIndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value) where T : IEquatable<T>?
+            => LastIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), value, span.Length);
+
+        private static int LastIndexOfAnyExcept<T>(ref T searchSpace, T value, int length) where T : IEquatable<T>?
         {
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.LastIndexOfAnyExceptValueType(
-                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, byte>(ref value),
-                        span.Length);
+                    return SpanHelpers.LastIndexOfAnyExceptValueType(ref Unsafe.As<T, byte>(ref searchSpace), Unsafe.As<T, byte>(ref value), length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    return SpanHelpers.LastIndexOfAnyExceptValueType(
-                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, short>(ref value),
-                        span.Length);
+                    return SpanHelpers.LastIndexOfAnyExceptValueType(ref Unsafe.As<T, short>(ref searchSpace), Unsafe.As<T, short>(ref value), length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(int))
                 {
-                    return SpanHelpers.LastIndexOfAnyExceptValueType(
-                        ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, int>(ref value),
-                        span.Length);
+                    return SpanHelpers.LastIndexOfAnyExceptValueType(ref Unsafe.As<T, int>(ref searchSpace), Unsafe.As<T, int>(ref value), length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(long))
                 {
-                    return SpanHelpers.LastIndexOfAnyExceptValueType(
-                        ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, long>(ref value),
-                        span.Length);
+                    return SpanHelpers.LastIndexOfAnyExceptValueType(ref Unsafe.As<T, long>(ref searchSpace), Unsafe.As<T, long>(ref value), length);
                 }
             }
 
-            for (int i = span.Length - 1; i >= 0; i--)
+            for (int i = length - 1; i >= 0; i--)
             {
-                if (!EqualityComparer<T>.Default.Equals(span[i], value))
+                if (!EqualityComparer<T>.Default.Equals(Unsafe.Add(ref searchSpace, i), value))
                 {
                     return i;
                 }
@@ -894,23 +806,32 @@ ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(other)),
         /// <param name="value">The value to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int IndexOf<T>(this ReadOnlySpan<T> span, T value) where T : IEquatable<T>?
+            => IndexOf(ref MemoryMarshal.GetReference(span), value, span.Length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static int IndexOf<T>(ref T searchSpace, T value, int length) where T : IEquatable<T>?
         {
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
-                    return SpanHelpers.IndexOf(
-                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, byte>(ref value),
-                        span.Length);
-
-                if (Unsafe.SizeOf<T>() == sizeof(char))
-                    return SpanHelpers.IndexOf(
-                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, char>(ref value),
-                        span.Length);
+                {
+                    return SpanHelpers.IndexOf(ref Unsafe.As<T, byte>(ref searchSpace), Unsafe.As<T, byte>(ref value), length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(char))
+                {
+                    return SpanHelpers.IndexOf(ref Unsafe.As<T, char>(ref searchSpace), Unsafe.As<T, char>(ref value), length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(int))
+                {
+                    return SpanHelpers.IndexOfValueType(ref Unsafe.As<T, int>(ref searchSpace), Unsafe.As<T, int>(ref value), length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(long))
+                {
+                    return SpanHelpers.IndexOfValueType(ref Unsafe.As<T, long>(ref searchSpace), Unsafe.As<T, long>(ref value), length);
+                }
             }
 
-            return SpanHelpers.IndexOf(ref MemoryMarshal.GetReference(span), value, span.Length);
+            return SpanHelpers.IndexOf(ref searchSpace, value, length);
         }
 
         /// <summary>
@@ -920,25 +841,24 @@ ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
         /// <param name="value">The sequence to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int IndexOf<T>(this ReadOnlySpan<T> span, ReadOnlySpan<T> value) where T : IEquatable<T>?
+            => IndexOf(ref MemoryMarshal.GetReference(span), span.Length, ref MemoryMarshal.GetReference(value), value.Length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static int IndexOf<T>(ref T searchSpace, int searchSpaceLength, ref T value, int valueLength) where T : IEquatable<T>?
         {
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
-                    return SpanHelpers.IndexOf(
-                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
-                        span.Length,
-                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(value)),
-                        value.Length);
-
-                if (Unsafe.SizeOf<T>() == sizeof(char))
-                    return SpanHelpers.IndexOf(
-                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
-                        span.Length,
-                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(value)),
-                        value.Length);
+                {
+                    return SpanHelpers.IndexOf(ref Unsafe.As<T, byte>(ref searchSpace), searchSpaceLength, ref Unsafe.As<T, byte>(ref value), valueLength);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(char))
+                {
+                    return SpanHelpers.IndexOf(ref Unsafe.As<T, char>(ref searchSpace), searchSpaceLength, ref Unsafe.As<T, char>(ref value), valueLength);
+                }
             }
 
-            return SpanHelpers.IndexOf(ref MemoryMarshal.GetReference(span), span.Length, ref MemoryMarshal.GetReference(value), value.Length);
+            return SpanHelpers.IndexOf(ref searchSpace, searchSpaceLength, ref value, valueLength);
         }
 
         /// <summary>
@@ -948,40 +868,32 @@ ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(value)),
         /// <param name="value">The value to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOf<T>(this ReadOnlySpan<T> span, T value) where T : IEquatable<T>?
+            => LastIndexOf(ref MemoryMarshal.GetReference(span), value, span.Length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static int LastIndexOf<T>(ref T searchSpace, T value, int length) where T : IEquatable<T>?
         {
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.LastIndexOfValueType(
-                            ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
-                            Unsafe.As<T, byte>(ref value),
-                            span.Length);
+                    return SpanHelpers.LastIndexOfValueType(ref Unsafe.As<T, byte>(ref searchSpace), Unsafe.As<T, byte>(ref value), length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    return SpanHelpers.LastIndexOfValueType(
-                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, short>(ref value),
-                        span.Length);
+                    return SpanHelpers.LastIndexOfValueType(ref Unsafe.As<T, short>(ref searchSpace), Unsafe.As<T, short>(ref value), length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(int))
                 {
-                    return SpanHelpers.LastIndexOfValueType(
-                        ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, int>(ref value),
-                        span.Length);
+                    return SpanHelpers.LastIndexOfValueType(ref Unsafe.As<T, int>(ref searchSpace), Unsafe.As<T, int>(ref value), length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(long))
                 {
-                    return SpanHelpers.LastIndexOfValueType(
-                        ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, long>(ref value),
-                        span.Length);
+                    return SpanHelpers.LastIndexOfValueType(ref Unsafe.As<T, long>(ref searchSpace), Unsafe.As<T, long>(ref value), length);
                 }
             }
 
-            return SpanHelpers.LastIndexOf<T>(ref MemoryMarshal.GetReference(span), value, span.Length);
+            return SpanHelpers.LastIndexOf<T>(ref searchSpace, value, length);
         }
 
         /// <summary>
@@ -991,28 +903,24 @@ ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
         /// <param name="value">The sequence to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOf<T>(this ReadOnlySpan<T> span, ReadOnlySpan<T> value) where T : IEquatable<T>?
+            => LastIndexOf(ref MemoryMarshal.GetReference(span), span.Length, ref MemoryMarshal.GetReference(value), value.Length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static int LastIndexOf<T>(ref T searchSpace, int searchSpaceLength, ref T value, int valueLength) where T : IEquatable<T>?
         {
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.LastIndexOf(
-                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
-                        span.Length,
-                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(value)),
-                        value.Length);
+                    return SpanHelpers.LastIndexOf(ref Unsafe.As<T, byte>(ref searchSpace), searchSpaceLength, ref Unsafe.As<T, byte>(ref value), valueLength);
                 }
                 if (Unsafe.SizeOf<T>() == sizeof(char))
                 {
-                    return SpanHelpers.LastIndexOf(
-                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
-                        span.Length,
-                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(value)),
-                        value.Length);
+                    return SpanHelpers.LastIndexOf(ref Unsafe.As<T, char>(ref searchSpace), searchSpaceLength, ref Unsafe.As<T, char>(ref value), valueLength);
                 }
             }
 
-            return SpanHelpers.LastIndexOf<T>(ref MemoryMarshal.GetReference(span), span.Length, ref MemoryMarshal.GetReference(value), value.Length);
+            return SpanHelpers.LastIndexOf<T>(ref searchSpace, searchSpaceLength, ref value, valueLength);
         }
 
         /// <summary>
@@ -1274,7 +1182,7 @@ private static unsafe int IndexOfAnyProbabilistic(ref char searchSpace, int sear
         /// <param name="value1">One of the values to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOfAny<T>(this Span<T> span, T value0, T value1) where T : IEquatable<T>?
-            => LastIndexOfAny((ReadOnlySpan<T>)span, value0, value1);
+            => LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, span.Length);
 
         /// <summary>
         /// Searches for the last index of any of the specified values similar to calling LastIndexOf several times with the logical OR operator. If not found, returns -1.
@@ -1285,7 +1193,7 @@ public static int LastIndexOfAny<T>(this Span<T> span, T value0, T value1) where
         /// <param name="value2">One of the values to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOfAny<T>(this Span<T> span, T value0, T value1, T value2) where T : IEquatable<T>?
-            => LastIndexOfAny((ReadOnlySpan<T>)span, value0, value1, value2);
+            => LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, value2, span.Length);
 
         /// <summary>
         /// Searches for the last index of any of the specified values similar to calling LastIndexOf several times with the logical OR operator. If not found, returns -1.
@@ -1313,44 +1221,32 @@ ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(values)),
         /// <param name="value1">One of the values to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOfAny<T>(this ReadOnlySpan<T> span, T value0, T value1) where T : IEquatable<T>?
+            => LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, span.Length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static int LastIndexOfAny<T>(ref T searchSpace, T value0, T value1, int length) where T : IEquatable<T>?
         {
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType(
-                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, byte>(ref value0),
-                        Unsafe.As<T, byte>(ref value1),
-                        span.Length);
+                    return SpanHelpers.LastIndexOfAnyValueType(ref Unsafe.As<T, byte>(ref searchSpace), Unsafe.As<T, byte>(ref value0), Unsafe.As<T, byte>(ref value1), length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType(
-                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, short>(ref value0),
-                        Unsafe.As<T, short>(ref value1),
-                        span.Length);
+                    return SpanHelpers.LastIndexOfAnyValueType(ref Unsafe.As<T, short>(ref searchSpace), Unsafe.As<T, short>(ref value0), Unsafe.As<T, short>(ref value1), length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(int))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType(
-                        ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, int>(ref value0),
-                        Unsafe.As<T, int>(ref value1),
-                        span.Length);
+                    return SpanHelpers.LastIndexOfAnyValueType(ref Unsafe.As<T, int>(ref searchSpace), Unsafe.As<T, int>(ref value0), Unsafe.As<T, int>(ref value1), length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(long))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType(
-                        ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, long>(ref value0),
-                        Unsafe.As<T, long>(ref value1),
-                        span.Length);
+                    return SpanHelpers.LastIndexOfAnyValueType(ref Unsafe.As<T, long>(ref searchSpace), Unsafe.As<T, long>(ref value0), Unsafe.As<T, long>(ref value1), length);
                 }
             }
 
-            return SpanHelpers.LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, span.Length);
+            return SpanHelpers.LastIndexOfAny(ref searchSpace, value0, value1, length);
         }
 
         /// <summary>
@@ -1362,31 +1258,35 @@ ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
         /// <param name="value2">One of the values to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOfAny<T>(this ReadOnlySpan<T> span, T value0, T value1, T value2) where T : IEquatable<T>?
+            => LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, value2, span.Length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static int LastIndexOfAny<T>(ref T searchSpace, T value0, T value1, T value2, int length) where T : IEquatable<T>?
         {
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
                     return SpanHelpers.LastIndexOfAnyValueType(
-                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        ref Unsafe.As<T, byte>(ref searchSpace),
                         Unsafe.As<T, byte>(ref value0),
                         Unsafe.As<T, byte>(ref value1),
                         Unsafe.As<T, byte>(ref value2),
-                        span.Length);
+                        length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
                     return SpanHelpers.LastIndexOfAnyValueType(
-                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        ref Unsafe.As<T, short>(ref searchSpace),
                         Unsafe.As<T, short>(ref value0),
                         Unsafe.As<T, short>(ref value1),
                         Unsafe.As<T, short>(ref value2),
-                        span.Length);
+                        length);
                 }
                 // we can easily add int and long here, but there must be an evidence that shows that it's actually needed
             }
 
-            return SpanHelpers.LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, value2, span.Length);
+            return SpanHelpers.LastIndexOfAny(ref searchSpace, value0, value1, value2, length);
         }
 
         /// <summary>

From ab8df3d6daa63f19a10b10d53983ad4d7acab8e9 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Fri, 12 Aug 2022 15:32:16 +0200
Subject: [PATCH 12/34] vectorize IndexOf(value)

---
 .../src/System/StubHelpers.cs                 |   4 +-
 .../src/System/Array.cs                       |  10 +-
 .../System.Private.CoreLib/src/System/Enum.cs |   2 +-
 .../src/System/Globalization/Ordinal.cs       |   2 +-
 .../src/System/MemoryExtensions.cs            |  11 +-
 .../src/System/SpanHelpers.Byte.cs            |   4 +-
 .../src/System/SpanHelpers.Char.cs            |   4 +-
 .../src/System/SpanHelpers.T.cs               | 277 +++++++-----------
 .../src/System/String.Manipulation.cs         |   2 +-
 .../src/System/String.Searching.cs            |   7 +-
 10 files changed, 133 insertions(+), 190 deletions(-)

diff --git a/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs b/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs
index 595f5482fca5b9..b0277baa2d56df 100644
--- a/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs
+++ b/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs
@@ -176,7 +176,7 @@ internal static unsafe void ConvertFixedToNative(int flags, string strManaged, I
 
         internal static unsafe string ConvertFixedToManaged(IntPtr cstr, int length)
         {
-            int end = SpanHelpers.IndexOf(ref *(byte*)cstr, 0, length);
+            int end = SpanHelpers.IndexOfValueType(ref *(byte*)cstr, (byte)0, length);
             if (end >= 0)
             {
                 length = end;
@@ -450,7 +450,7 @@ internal static unsafe void ConvertToNative(string? strManaged, IntPtr nativeHom
 
         internal static unsafe string ConvertToManaged(IntPtr nativeHome, int length)
         {
-            int end = SpanHelpers.IndexOf(ref *(char*)nativeHome, '\0', length);
+            int end = SpanHelpers.IndexOfValueType(ref *(short*)nativeHome, (short)'\0', length);
             if (end >= 0)
             {
                 length = end;
diff --git a/src/libraries/System.Private.CoreLib/src/System/Array.cs b/src/libraries/System.Private.CoreLib/src/System/Array.cs
index ca44a415cee188..b665d3094e1dc9 100644
--- a/src/libraries/System.Private.CoreLib/src/System/Array.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/Array.cs
@@ -1328,17 +1328,17 @@ public static int IndexOf<T>(T[] array, T value, int startIndex, int count)
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    int result = SpanHelpers.IndexOf(
+                    int result = SpanHelpers.IndexOfValueType(
                         ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<byte[]>(array)), startIndex),
                         Unsafe.As<T, byte>(ref value),
                         count);
                     return (result >= 0 ? startIndex : 0) + result;
                 }
-                else if (Unsafe.SizeOf<T>() == sizeof(char))
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    int result = SpanHelpers.IndexOf(
-                        ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<char[]>(array)), startIndex),
-                        Unsafe.As<T, char>(ref value),
+                    int result = SpanHelpers.IndexOfValueType(
+                        ref Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(Unsafe.As<short[]>(array)), startIndex),
+                        Unsafe.As<T, short>(ref value),
                         count);
                     return (result >= 0 ? startIndex : 0) + result;
                 }
diff --git a/src/libraries/System.Private.CoreLib/src/System/Enum.cs b/src/libraries/System.Private.CoreLib/src/System/Enum.cs
index 89404415b5e17d..cd4535867b0334 100644
--- a/src/libraries/System.Private.CoreLib/src/System/Enum.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/Enum.cs
@@ -436,7 +436,7 @@ private static int FindDefinedIndex(ulong[] ulValues, ulong ulValue)
             int ulValuesLength = ulValues.Length;
             ref ulong start = ref MemoryMarshal.GetArrayDataReference(ulValues);
             return ulValuesLength <= NumberOfValuesThreshold ?
-                SpanHelpers.IndexOf(ref start, ulValue, ulValuesLength) :
+                SpanHelpers.IndexOfValueType(ref Unsafe.As<ulong, long>(ref start), (long)ulValue, ulValuesLength) :
                 SpanHelpers.BinarySearch(ref start, ulValuesLength, ulValue);
         }
 
diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs
index ef51de122f7179..1a1b96642038c7 100644
--- a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs
@@ -265,7 +265,7 @@ internal static int IndexOfOrdinalIgnoreCase(ReadOnlySpan<char> source, ReadOnly
                 // Do a quick search for the first element of "value".
                 int relativeIndex = isLetter ?
                     SpanHelpers.IndexOfAny(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceLength) :
-                    SpanHelpers.IndexOf(ref Unsafe.Add(ref searchSpace, offset), valueChar, searchSpaceLength);
+                    SpanHelpers.IndexOfChar(ref Unsafe.Add(ref searchSpace, offset), valueChar, searchSpaceLength);
                 if (relativeIndex < 0)
                 {
                     break;
diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
index 83c767b3396231..23062b8a36a57f 100644
--- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
@@ -815,11 +815,11 @@ private static int IndexOf<T>(ref T searchSpace, T value, int length) where T :
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.IndexOf(ref Unsafe.As<T, byte>(ref searchSpace), Unsafe.As<T, byte>(ref value), length);
+                    return SpanHelpers.IndexOfValueType(ref Unsafe.As<T, byte>(ref searchSpace), Unsafe.As<T, byte>(ref value), length);
                 }
-                else if (Unsafe.SizeOf<T>() == sizeof(char))
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    return SpanHelpers.IndexOf(ref Unsafe.As<T, char>(ref searchSpace), Unsafe.As<T, char>(ref value), length);
+                    return SpanHelpers.IndexOfValueType(ref Unsafe.As<T, short>(ref searchSpace), Unsafe.As<T, short>(ref value), length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(int))
                 {
@@ -1096,10 +1096,7 @@ ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                             return -1;
 
                         case 1:
-                            return SpanHelpers.IndexOf(
-                                ref spanRef,
-                                valueRef,
-                                span.Length);
+                            return SpanHelpers.IndexOfChar(ref spanRef, valueRef, span.Length);
 
                         case 2:
                             return SpanHelpers.IndexOfAny(
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
index 0417b116ea8570..1318e38d7e64c5 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
@@ -22,7 +22,7 @@ public static int IndexOf(ref byte searchSpace, int searchSpaceLength, ref byte
 
             int valueTailLength = valueLength - 1;
             if (valueTailLength == 0)
-                return IndexOf(ref searchSpace, value, searchSpaceLength); // for single-byte values use plain IndexOf
+                return IndexOfValueType(ref searchSpace, value, searchSpaceLength); // for single-byte values use plain IndexOf
 
             nint offset = 0;
             byte valueHead = value;
@@ -38,7 +38,7 @@ public static int IndexOf(ref byte searchSpace, int searchSpaceLength, ref byte
             while (remainingSearchSpaceLength > 0)
             {
                 // Do a quick search for the first element of "value".
-                int relativeIndex = IndexOf(ref Unsafe.Add(ref searchSpace, offset), valueHead, remainingSearchSpaceLength);
+                int relativeIndex = IndexOfValueType(ref Unsafe.Add(ref searchSpace, offset), valueHead, remainingSearchSpaceLength);
                 if (relativeIndex < 0)
                     break;
 
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
index 114d00baf831b3..59d8ef43a7c00b 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
@@ -24,7 +24,7 @@ public static int IndexOf(ref char searchSpace, int searchSpaceLength, ref char
             if (valueTailLength == 0)
             {
                 // for single-char values use plain IndexOf
-                return IndexOf(ref searchSpace, value, searchSpaceLength);
+                return IndexOfChar(ref searchSpace, value, searchSpaceLength);
             }
 
             nint offset = 0;
@@ -41,7 +41,7 @@ public static int IndexOf(ref char searchSpace, int searchSpaceLength, ref char
             while (remainingSearchSpaceLength > 0)
             {
                 // Do a quick search for the first element of "value".
-                int relativeIndex = IndexOf(ref Unsafe.Add(ref searchSpace, offset), valueHead, remainingSearchSpaceLength);
+                int relativeIndex = IndexOfChar(ref Unsafe.Add(ref searchSpace, offset), valueHead, remainingSearchSpaceLength);
                 if (relativeIndex < 0)
                     break;
 
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
index 4a3d15bd439f24..3e05a41f1d3500 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
@@ -293,115 +293,6 @@ public static unsafe bool Contains<T>(ref T searchSpace, T value, int length) wh
             return true;
         }
 
-        internal static unsafe int IndexOfValueType<T>(ref T searchSpace, T value, int length) where T : struct, IEquatable<T>
-        {
-            Debug.Assert(length >= 0);
-
-            nint index = 0; // Use nint for arithmetic to avoid unnecessary 64->32->64 truncations
-            if (Vector.IsHardwareAccelerated && Vector<T>.IsSupported && (Vector<T>.Count * 2) <= length)
-            {
-                Vector<T> valueVector = new Vector<T>(value);
-                Vector<T> compareVector;
-                Vector<T> matchVector;
-                if ((uint)length % (uint)Vector<T>.Count != 0)
-                {
-                    // Number of elements is not a multiple of Vector<T>.Count, so do one
-                    // check and shift only enough for the remaining set to be a multiple
-                    // of Vector<T>.Count.
-                    compareVector = Unsafe.As<T, Vector<T>>(ref Unsafe.Add(ref searchSpace, index));
-                    matchVector = Vector.Equals(valueVector, compareVector);
-                    if (matchVector != Vector<T>.Zero)
-                    {
-                        goto VectorMatch;
-                    }
-                    index += length % Vector<T>.Count;
-                    length -= length % Vector<T>.Count;
-                }
-                while (length > 0)
-                {
-                    compareVector = Unsafe.As<T, Vector<T>>(ref Unsafe.Add(ref searchSpace, index));
-                    matchVector = Vector.Equals(valueVector, compareVector);
-                    if (matchVector != Vector<T>.Zero)
-                    {
-                        goto VectorMatch;
-                    }
-                    index += Vector<T>.Count;
-                    length -= Vector<T>.Count;
-                }
-                goto NotFound;
-            VectorMatch:
-                for (int i = 0; i < Vector<T>.Count; i++)
-                    if (compareVector[i].Equals(value))
-                        return (int)(index + i);
-            }
-
-            while (length >= 8)
-            {
-                if (value.Equals(Unsafe.Add(ref searchSpace, index)))
-                    goto Found;
-                if (value.Equals(Unsafe.Add(ref searchSpace, index + 1)))
-                    goto Found1;
-                if (value.Equals(Unsafe.Add(ref searchSpace, index + 2)))
-                    goto Found2;
-                if (value.Equals(Unsafe.Add(ref searchSpace, index + 3)))
-                    goto Found3;
-                if (value.Equals(Unsafe.Add(ref searchSpace, index + 4)))
-                    goto Found4;
-                if (value.Equals(Unsafe.Add(ref searchSpace, index + 5)))
-                    goto Found5;
-                if (value.Equals(Unsafe.Add(ref searchSpace, index + 6)))
-                    goto Found6;
-                if (value.Equals(Unsafe.Add(ref searchSpace, index + 7)))
-                    goto Found7;
-
-                length -= 8;
-                index += 8;
-            }
-
-            while (length >= 4)
-            {
-                if (value.Equals(Unsafe.Add(ref searchSpace, index)))
-                    goto Found;
-                if (value.Equals(Unsafe.Add(ref searchSpace, index + 1)))
-                    goto Found1;
-                if (value.Equals(Unsafe.Add(ref searchSpace, index + 2)))
-                    goto Found2;
-                if (value.Equals(Unsafe.Add(ref searchSpace, index + 3)))
-                    goto Found3;
-
-                length -= 4;
-                index += 4;
-            }
-
-            while (length > 0)
-            {
-                if (value.Equals(Unsafe.Add(ref searchSpace, index)))
-                    goto Found;
-
-                index += 1;
-                length--;
-            }
-        NotFound:
-            return -1;
-
-        Found: // Workaround for https://github.com/dotnet/runtime/issues/8795
-            return (int)index;
-        Found1:
-            return (int)(index + 1);
-        Found2:
-            return (int)(index + 2);
-        Found3:
-            return (int)(index + 3);
-        Found4:
-            return (int)(index + 4);
-        Found5:
-            return (int)(index + 5);
-        Found6:
-            return (int)(index + 6);
-        Found7:
-            return (int)(index + 7);
-        }
-
         public static unsafe int IndexOf<T>(ref T searchSpace, T value, int length) where T : IEquatable<T>?
         {
             Debug.Assert(length >= 0);
@@ -1177,62 +1068,6 @@ public static int IndexOfAnyExcept<T>(ref T searchSpace, T value0, int length)
             return -1;
         }
 
-        public static int IndexOfAnyExceptValueType<T>(ref T searchSpace, T value0, int length) where T : struct, IEquatable<T>
-        {
-            Debug.Assert(length >= 0, "Expected non-negative length");
-            Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types");
-
-            if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
-            {
-                for (int i = 0; i < length; i++)
-                {
-                    if (!Unsafe.Add(ref searchSpace, i).Equals(value0))
-                    {
-                        return i;
-                    }
-                }
-            }
-            else
-            {
-                Vector128<T> notEquals, value0Vector = Vector128.Create(value0);
-                ref T current = ref searchSpace;
-                ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128<T>.Count);
-
-                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
-                do
-                {
-                    notEquals = ~Vector128.Equals(value0Vector, Vector128.LoadUnsafe(ref current));
-                    if (notEquals != Vector128<T>.Zero)
-                    {
-                        return ComputeIndex(ref searchSpace, ref current, notEquals);
-                    }
-
-                    current = ref Unsafe.Add(ref current, Vector128<T>.Count);
-                }
-                while (!Unsafe.IsAddressGreaterThan(ref current, ref oneVectorAwayFromEnd));
-
-                // If any elements remain, process the last vector in the search space.
-                if ((uint)length % Vector128<T>.Count != 0)
-                {
-                    notEquals = ~Vector128.Equals(value0Vector, Vector128.LoadUnsafe(ref oneVectorAwayFromEnd));
-                    if (notEquals != Vector128<T>.Zero)
-                    {
-                        return ComputeIndex(ref searchSpace, ref oneVectorAwayFromEnd, notEquals);
-                    }
-                }
-
-                [MethodImpl(MethodImplOptions.AggressiveInlining)]
-                static int ComputeIndex(ref T searchSpace, ref T current, Vector128<T> notEquals)
-                {
-                    uint notEqualsElements = notEquals.ExtractMostSignificantBits();
-                    int index = BitOperations.TrailingZeroCount(notEqualsElements);
-                    return index + (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / Unsafe.SizeOf<T>());
-                }
-            }
-
-            return -1;
-        }
-
         public static bool SequenceEqual<T>(ref T first, ref T second, int length) where T : IEquatable<T>?
         {
             Debug.Assert(length >= 0);
@@ -1423,6 +1258,102 @@ internal static bool ContainsValueType<T>(ref T searchSpace, T value, int length
             return false;
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static int IndexOfChar(ref char searchSpace, char value, int length)
+            => IndexOfValueType(ref Unsafe.As<char, short>(ref searchSpace), (short)value, length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static int IndexOfValueType<T>(ref T searchSpace, T value, int length) where T : struct, IEquatable<T>
+            => IndexOfValueType<T, DontNegate<T>>(ref searchSpace, value, length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static int IndexOfAnyExceptValueType<T>(ref T searchSpace, T value, int length) where T : struct, IEquatable<T>
+            => IndexOfValueType<T, Negate<T>>(ref searchSpace, value, length);
+
+        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
+        private static int IndexOfValueType<T, N>(ref T searchSpace, T value, int length)
+            where T : struct, IEquatable<T>
+            where N : struct, INegator<T>
+        {
+            Debug.Assert(length >= 0, "Expected non-negative length");
+            Debug.Assert(value is byte or short or int or long, "Expected caller to normalize to one of these types");
+
+            N negator = default;
+
+            if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
+            {
+                for (int i = 0; i < length; i++)
+                {
+                    if (negator.NegateIfNeeded(Unsafe.Add(ref searchSpace, i).Equals(value)))
+                    {
+                        return i;
+                    }
+                }
+            }
+            else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
+            {
+                Vector256<T> equals, values = Vector256.Create(value);
+                ref T currentSearchSpace = ref searchSpace;
+                ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256<T>.Count);
+
+                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
+                do
+                {
+                    equals = negator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref currentSearchSpace)));
+                    if (equals == Vector256<T>.Zero)
+                    {
+                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector256<T>.Count);
+                        continue;
+                    }
+
+                    return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals);
+                }
+                while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd));
+
+                // If any elements remain, process the last vector in the search space.
+                if ((uint)length % Vector256<T>.Count != 0)
+                {
+                    equals = negator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref oneVectorAwayFromEnd)));
+                    if (equals != Vector256<T>.Zero)
+                    {
+                        return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
+                    }
+                }
+            }
+            else
+            {
+                Vector128<T> equals, values = Vector128.Create(value);
+                ref T currentSearchSpace = ref searchSpace;
+                ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128<T>.Count);
+
+                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
+                do
+                {
+                    equals = negator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref currentSearchSpace)));
+                    if (equals == Vector128<T>.Zero)
+                    {
+                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128<T>.Count);
+                        continue;
+                    }
+
+                    return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals);
+                }
+                while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd));
+
+                // If any elements remain, process the first vector in the search space.
+                if ((uint)length % Vector128<T>.Count != 0)
+                {
+                    equals = negator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref oneVectorAwayFromEnd)));
+                    if (equals != Vector128<T>.Zero)
+                    {
+                        return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
+                    }
+                }
+            }
+
+            return -1;
+        }
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal static int LastIndexOfValueType<T>(ref T searchSpace, T value, int length) where T : struct, IEquatable<T>
             => LastIndexOfValueType<T, DontNegate<T>>(ref searchSpace, value, length);
@@ -1703,6 +1634,22 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
             return -1;
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static int ComputeFirstIndex<T>(ref T searchSpace, ref T current, Vector128<T> equals) where T : struct, IEquatable<T>
+        {
+            uint notEqualsElements = equals.ExtractMostSignificantBits();
+            int index = BitOperations.TrailingZeroCount(notEqualsElements);
+            return index + (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / Unsafe.SizeOf<T>());
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static int ComputeFirstIndex<T>(ref T searchSpace, ref T current, Vector256<T> equals) where T : struct, IEquatable<T>
+        {
+            uint notEqualsElements = equals.ExtractMostSignificantBits();
+            int index = BitOperations.TrailingZeroCount(notEqualsElements);
+            return index + (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / Unsafe.SizeOf<T>());
+        }
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static int ComputeLastIndex<T>(ref T searchSpace, ref T current, Vector128<T> equals) where T : struct, IEquatable<T>
         {
diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs
index b386f7e5c36b37..846a56b004bf33 100644
--- a/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/String.Manipulation.cs
@@ -1067,7 +1067,7 @@ public string Replace(string oldValue, string? newValue)
                 int i = 0;
                 while (true)
                 {
-                    int pos = SpanHelpers.IndexOf(ref Unsafe.Add(ref _firstChar, i), c, Length - i);
+                    int pos = SpanHelpers.IndexOfChar(ref Unsafe.Add(ref _firstChar, i), c, Length - i);
                     if (pos < 0)
                     {
                         break;
diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs
index fb763edacc8e00..0f0dbe2b50b29d 100644
--- a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs
@@ -37,8 +37,7 @@ public bool Contains(char value, StringComparison comparisonType)
 
         // Returns the index of the first occurrence of a specified character in the current instance.
         // The search starts at startIndex and runs thorough the next count characters.
-        //
-        public int IndexOf(char value) => SpanHelpers.IndexOf(ref _firstChar, value, Length);
+        public int IndexOf(char value) => SpanHelpers.IndexOfChar(ref _firstChar, value, Length);
 
         public int IndexOf(char value, int startIndex)
         {
@@ -82,7 +81,7 @@ private int IndexOfCharOrdinalIgnoreCase(char value)
                 return SpanHelpers.IndexOfAny(ref _firstChar, valueLc, valueUc, Length);
             }
 
-            return SpanHelpers.IndexOf(ref _firstChar, value, Length);
+            return SpanHelpers.IndexOfChar(ref _firstChar, value, Length);
         }
 
         public unsafe int IndexOf(char value, int startIndex, int count)
@@ -97,7 +96,7 @@ public unsafe int IndexOf(char value, int startIndex, int count)
                 ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.count, ExceptionResource.ArgumentOutOfRange_Count);
             }
 
-            int result = SpanHelpers.IndexOf(ref Unsafe.Add(ref _firstChar, startIndex), value, count);
+            int result = SpanHelpers.IndexOfChar(ref Unsafe.Add(ref _firstChar, startIndex), value, count);
 
             return result < 0 ? result : result + startIndex;
         }

From bb1395759004457c2739c8d1e52069e8dcfa0a33 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Fri, 12 Aug 2022 16:05:33 +0200
Subject: [PATCH 13/34] rename IndexOf used only by strlen to IndexOfNullByte
 and optimize it to searching for zeros

---
 .../src/System/SpanHelpers.Byte.cs            | 56 +++--------------
 .../src/System/SpanHelpers.Char.cs            | 62 +++----------------
 .../src/System/String.cs                      |  4 +-
 3 files changed, 20 insertions(+), 102 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
index 1318e38d7e64c5..759e8a805c00f9 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
@@ -334,28 +334,22 @@ ref Unsafe.Add(ref searchSpace, offset + bitPos),
         }
 
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        public static unsafe int IndexOf(ref byte searchSpace, byte value, int length)
+        public static unsafe int IndexOfNullByte(ref byte searchSpace)
         {
-            Debug.Assert(length >= 0);
+            const int length = int.MaxValue;
 
-            uint uValue = value; // Use uint for comparisons to avoid unnecessary 8->32 extensions
+            const uint uValue = 0; // Use uint for comparisons to avoid unnecessary 8->32 extensions
             nuint offset = 0; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations
             nuint lengthToExamine = (nuint)(uint)length;
 
             if (Vector128.IsHardwareAccelerated)
             {
                 // Avx2 branch also operates on Sse2 sizes, so check is combined.
-                if (length >= Vector128<byte>.Count * 2)
-                {
-                    lengthToExamine = UnalignedCountVector128(ref searchSpace);
-                }
+                lengthToExamine = UnalignedCountVector128(ref searchSpace);
             }
             else if (Vector.IsHardwareAccelerated)
             {
-                if (length >= Vector<byte>.Count * 2)
-                {
-                    lengthToExamine = UnalignedCountVector(ref searchSpace);
-                }
+                lengthToExamine = UnalignedCountVector(ref searchSpace);
             }
         SequentialScan:
             while (lengthToExamine >= 8)
@@ -421,11 +415,10 @@ public static unsafe int IndexOf(ref byte searchSpace, byte value, int length)
                         // with no upper bound e.g. String.strlen.
                         // Start with a check on Vector128 to align to Vector256, before moving to processing Vector256.
                         // This ensures we do not fault across memory pages while searching for an end of string.
-                        Vector128<byte> values = Vector128.Create(value);
                         Vector128<byte> search = Vector128.LoadUnsafe(ref searchSpace, offset);
 
                         // Same method as below
-                        uint matches = Vector128.Equals(values, search).ExtractMostSignificantBits();
+                        uint matches = Vector128.Equals(Vector128<byte>.Zero, search).ExtractMostSignificantBits();
                         if (matches == 0)
                         {
                             // Zero flags set so no matches
@@ -441,11 +434,10 @@ public static unsafe int IndexOf(ref byte searchSpace, byte value, int length)
                     lengthToExamine = GetByteVector256SpanLength(offset, length);
                     if (lengthToExamine > offset)
                     {
-                        Vector256<byte> values = Vector256.Create(value);
                         do
                         {
                             Vector256<byte> search = Vector256.LoadUnsafe(ref searchSpace, offset);
-                            uint matches = Vector256.Equals(values, search).ExtractMostSignificantBits();
+                            uint matches = Vector256.Equals(Vector256<byte>.Zero, search).ExtractMostSignificantBits();
                             // Note that MoveMask has converted the equal vector elements into a set of bit flags,
                             // So the bit position in 'matches' corresponds to the element offset.
                             if (matches == 0)
@@ -463,11 +455,10 @@ public static unsafe int IndexOf(ref byte searchSpace, byte value, int length)
                     lengthToExamine = GetByteVector128SpanLength(offset, length);
                     if (lengthToExamine > offset)
                     {
-                        Vector128<byte> values = Vector128.Create(value);
                         Vector128<byte> search = Vector128.LoadUnsafe(ref searchSpace, offset);
 
                         // Same method as above
-                        uint matches = Vector128.Equals(values, search).ExtractMostSignificantBits();
+                        uint matches = Vector128.Equals(Vector128<byte>.Zero, search).ExtractMostSignificantBits();
                         if (matches == 0)
                         {
                             // Zero flags set so no matches
@@ -493,13 +484,12 @@ public static unsafe int IndexOf(ref byte searchSpace, byte value, int length)
                 {
                     lengthToExamine = GetByteVector128SpanLength(offset, length);
 
-                    Vector128<byte> values = Vector128.Create(value);
                     while (lengthToExamine > offset)
                     {
                         Vector128<byte> search = Vector128.LoadUnsafe(ref searchSpace, offset);
 
                         // Same method as above
-                        Vector128<byte> compareResult = Vector128.Equals(values, search);
+                        Vector128<byte> compareResult = Vector128.Equals(Vector128<byte>.Zero, search);
                         if (compareResult == Vector128<byte>.Zero)
                         {
                             // Zero flags set so no matches
@@ -519,34 +509,6 @@ public static unsafe int IndexOf(ref byte searchSpace, byte value, int length)
                     }
                 }
             }
-            else if (Vector.IsHardwareAccelerated)
-            {
-                if (offset < (nuint)(uint)length)
-                {
-                    lengthToExamine = GetByteVectorSpanLength(offset, length);
-
-                    Vector<byte> values = new Vector<byte>(value);
-
-                    while (lengthToExamine > offset)
-                    {
-                        var matches = Vector.Equals(values, LoadVector(ref searchSpace, offset));
-                        if (Vector<byte>.Zero.Equals(matches))
-                        {
-                            offset += (nuint)Vector<byte>.Count;
-                            continue;
-                        }
-
-                        // Find offset of first match and add to current offset
-                        return (int)offset + LocateFirstFoundByte(matches);
-                    }
-
-                    if (offset < (nuint)(uint)length)
-                    {
-                        lengthToExamine = ((nuint)(uint)length - offset);
-                        goto SequentialScan;
-                    }
-                }
-            }
             return -1;
         Found: // Workaround for https://github.com/dotnet/runtime/issues/8795
             return (int)offset;
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
index 59d8ef43a7c00b..5cd0d2efc0a290 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
@@ -422,9 +422,10 @@ public static unsafe int SequenceCompareTo(ref char first, int firstLength, ref
 
 
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        public static unsafe int IndexOf(ref char searchSpace, char value, int length)
+        public static unsafe int IndexOfNullCharacter(ref char searchSpace)
         {
-            Debug.Assert(length >= 0);
+            const char value = '\0';
+            const int length = int.MaxValue;
 
             nint offset = 0;
             nint lengthToExamine = length;
@@ -437,18 +438,12 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length)
             {
                 // Avx2 branch also operates on Sse2 sizes, so check is combined.
                 // Needs to be double length to allow us to align the data first.
-                if (length >= Vector128<ushort>.Count * 2)
-                {
-                    lengthToExamine = UnalignedCountVector128(ref searchSpace);
-                }
+                lengthToExamine = UnalignedCountVector128(ref searchSpace);
             }
             else if (Vector.IsHardwareAccelerated)
             {
                 // Needs to be double length to allow us to align the data first.
-                if (length >= Vector<ushort>.Count * 2)
-                {
-                    lengthToExamine = UnalignedCountVector(ref searchSpace);
-                }
+                lengthToExamine = UnalignedCountVector(ref searchSpace);
             }
 
         SequentialScan:
@@ -507,11 +502,10 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length)
                         // method, so the alignment only acts as best endeavour. The GC cost is likely to dominate over
                         // the misalignment that may occur after; to we default to giving the GC a free hand to relocate and
                         // its up to the caller whether they are operating over fixed data.
-                        Vector128<ushort> values = Vector128.Create((ushort)value);
                         Vector128<ushort> search = Vector128.LoadUnsafe(ref ushortSearchSpace, (nuint)offset);
 
                         // Same method as below
-                        uint matches = Vector128.Equals(values, search).AsByte().ExtractMostSignificantBits();
+                        uint matches = Vector128.Equals(Vector128<ushort>.Zero, search).AsByte().ExtractMostSignificantBits();
                         if (matches == 0)
                         {
                             // Zero flags set so no matches
@@ -527,13 +521,12 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length)
                     lengthToExamine = GetCharVector256SpanLength(offset, length);
                     if (lengthToExamine > 0)
                     {
-                        Vector256<ushort> values = Vector256.Create((ushort)value);
                         do
                         {
                             Debug.Assert(lengthToExamine >= Vector256<ushort>.Count);
 
                             Vector256<ushort> search = Vector256.LoadUnsafe(ref ushortSearchSpace, (nuint)offset);
-                            uint matches = Vector256.Equals(values, search).AsByte().ExtractMostSignificantBits();
+                            uint matches = Vector256.Equals(Vector256<ushort>.Zero, search).AsByte().ExtractMostSignificantBits();
                             // Note that MoveMask has converted the equal vector elements into a set of bit flags,
                             // So the bit position in 'matches' corresponds to the element offset.
                             if (matches == 0)
@@ -555,11 +548,10 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length)
                     {
                         Debug.Assert(lengthToExamine >= Vector128<ushort>.Count);
 
-                        Vector128<ushort> values = Vector128.Create((ushort)value);
                         Vector128<ushort> search = Vector128.LoadUnsafe(ref ushortSearchSpace, (nuint)offset);
 
                         // Same method as above
-                        uint matches = Vector128.Equals(values, search).AsByte().ExtractMostSignificantBits();
+                        uint matches = Vector128.Equals(Vector128<ushort>.Zero, search).AsByte().ExtractMostSignificantBits();
                         if (matches == 0)
                         {
                             // Zero flags set so no matches
@@ -591,7 +583,6 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length)
                     lengthToExamine = GetCharVector128SpanLength(offset, length);
                     if (lengthToExamine > 0)
                     {
-                        Vector128<ushort> values = Vector128.Create((ushort)value);
                         do
                         {
                             Debug.Assert(lengthToExamine >= Vector128<ushort>.Count);
@@ -599,7 +590,7 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length)
                             Vector128<ushort> search = Vector128.LoadUnsafe(ref ushortSearchSpace, (uint)offset);
 
                             // Same method as above
-                            Vector128<ushort> compareResult = Vector128.Equals(values, search);
+                            Vector128<ushort> compareResult = Vector128.Equals(Vector128<ushort>.Zero, search);
                             if (compareResult == Vector128<ushort>.Zero)
                             {
                                 // Zero flags set so no matches
@@ -622,41 +613,6 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length)
                     }
                 }
             }
-            else if (Vector.IsHardwareAccelerated)
-            {
-                if (offset < length)
-                {
-                    Debug.Assert(length - offset >= Vector<ushort>.Count);
-
-                    lengthToExamine = GetCharVectorSpanLength(offset, length);
-
-                    if (lengthToExamine > 0)
-                    {
-                        Vector<ushort> values = new Vector<ushort>((ushort)value);
-                        do
-                        {
-                            Debug.Assert(lengthToExamine >= Vector<ushort>.Count);
-
-                            var matches = Vector.Equals(values, LoadVector(ref searchSpace, offset));
-                            if (Vector<ushort>.Zero.Equals(matches))
-                            {
-                                offset += Vector<ushort>.Count;
-                                lengthToExamine -= Vector<ushort>.Count;
-                                continue;
-                            }
-
-                            // Find offset of first match
-                            return (int)(offset + LocateFirstFoundChar(matches));
-                        } while (lengthToExamine > 0);
-                    }
-
-                    if (offset < length)
-                    {
-                        lengthToExamine = length - offset;
-                        goto SequentialScan;
-                    }
-                }
-            }
             return -1;
         Found3:
             return (int)(offset + 3);
diff --git a/src/libraries/System.Private.CoreLib/src/System/String.cs b/src/libraries/System.Private.CoreLib/src/System/String.cs
index 77d2168b0d38b8..765571c03592af 100644
--- a/src/libraries/System.Private.CoreLib/src/System/String.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/String.cs
@@ -598,7 +598,7 @@ internal static unsafe int wcslen(char* ptr)
         {
             // IndexOf processes memory in aligned chunks, and thus it won't crash even if it accesses memory beyond the null terminator.
             // This IndexOf behavior is an implementation detail of the runtime and callers outside System.Private.CoreLib must not depend on it.
-            int length = SpanHelpers.IndexOf(ref *ptr, '\0', int.MaxValue);
+            int length = SpanHelpers.IndexOfNullCharacter(ref *ptr);
             if (length < 0)
             {
                 ThrowMustBeNullTerminatedString();
@@ -612,7 +612,7 @@ internal static unsafe int strlen(byte* ptr)
         {
             // IndexOf processes memory in aligned chunks, and thus it won't crash even if it accesses memory beyond the null terminator.
             // This IndexOf behavior is an implementation detail of the runtime and callers outside System.Private.CoreLib must not depend on it.
-            int length = SpanHelpers.IndexOf(ref *ptr, (byte)'\0', int.MaxValue);
+            int length = SpanHelpers.IndexOfNullByte(ref *ptr);
             if (length < 0)
             {
                 ThrowMustBeNullTerminatedString();

From bbb7e4cad0e0bab822f1fbd0e512f61d0bf5cbe7 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Fri, 12 Aug 2022 16:27:23 +0200
Subject: [PATCH 14/34] vectorize IndexOfAny<T>(value0, value1) and
 IndexOfAnyExcept<T>(value0, value1)

---
 .../src/System/Globalization/Ordinal.cs       |   2 +-
 .../src/System/MemoryExtensions.cs            |  62 ++--
 .../src/System/SpanHelpers.Byte.cs            | 318 ------------------
 .../src/System/SpanHelpers.Char.cs            | 237 -------------
 .../src/System/SpanHelpers.T.cs               | 101 ++++++
 .../src/System/String.Searching.cs            |   2 +-
 6 files changed, 145 insertions(+), 577 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs
index 1a1b96642038c7..76d2b06b20b903 100644
--- a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs
@@ -264,7 +264,7 @@ internal static int IndexOfOrdinalIgnoreCase(ReadOnlySpan<char> source, ReadOnly
             {
                 // Do a quick search for the first element of "value".
                 int relativeIndex = isLetter ?
-                    SpanHelpers.IndexOfAny(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceLength) :
+                    SpanHelpers.IndexOfAnyChar(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceLength) :
                     SpanHelpers.IndexOfChar(ref Unsafe.Add(ref searchSpace, offset), valueChar, searchSpaceLength);
                 if (relativeIndex < 0)
                 {
diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
index 23062b8a36a57f..a25803a5430302 100644
--- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
@@ -435,6 +435,26 @@ private static int IndexOfAnyExcept<T>(ref T searchSpace, T value, int length) w
         /// </returns>
         public static int IndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value0, T value1) where T : IEquatable<T>?
         {
+            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            {
+                if (Unsafe.SizeOf<T>() == sizeof(byte))
+                {
+                    return SpanHelpers.IndexOfAnyExceptValueType(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, byte>(ref value0),
+                        Unsafe.As<T, byte>(ref value1),
+                        span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
+                {
+                    return SpanHelpers.IndexOfAnyExceptValueType(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value0),
+                        Unsafe.As<T, short>(ref value1),
+                        span.Length);
+                }
+            }
+
             for (int i = 0; i < span.Length; i++)
             {
                 if (!EqualityComparer<T>.Default.Equals(span[i], value0) &&
@@ -935,18 +955,21 @@ public static int IndexOfAny<T>(this Span<T> span, T value0, T value1) where T :
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
-                    return SpanHelpers.IndexOfAny(
+                {
+                    return SpanHelpers.IndexOfAnyValueType(
                         ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, byte>(ref value0),
                         Unsafe.As<T, byte>(ref value1),
                         span.Length);
-
-                if (Unsafe.SizeOf<T>() == sizeof(char))
-                    return SpanHelpers.IndexOfAny(
-                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, char>(ref value0),
-                        Unsafe.As<T, char>(ref value1),
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
+                {
+                    return SpanHelpers.IndexOfAnyValueType(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value0),
+                        Unsafe.As<T, short>(ref value1),
                         span.Length);
+                }
             }
 
             return SpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, span.Length);
@@ -1005,18 +1028,21 @@ public static int IndexOfAny<T>(this ReadOnlySpan<T> span, T value0, T value1) w
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
-                    return SpanHelpers.IndexOfAny(
+                {
+                    return SpanHelpers.IndexOfAnyValueType(
                         ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, byte>(ref value0),
                         Unsafe.As<T, byte>(ref value1),
                         span.Length);
-
-                if (Unsafe.SizeOf<T>() == sizeof(char))
-                    return SpanHelpers.IndexOfAny(
-                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, char>(ref value0),
-                        Unsafe.As<T, char>(ref value1),
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
+                {
+                    return SpanHelpers.IndexOfAnyValueType(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value0),
+                        Unsafe.As<T, short>(ref value1),
                         span.Length);
+                }
             }
 
             return SpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, span.Length);
@@ -1069,7 +1095,7 @@ public static int IndexOfAny<T>(this ReadOnlySpan<T> span, ReadOnlySpan<T> value
                     ref byte valueRef = ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(values));
                     if (values.Length == 2)
                     {
-                        return SpanHelpers.IndexOfAny(
+                        return SpanHelpers.IndexOfAnyValueType(
                             ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                             valueRef,
                             Unsafe.Add(ref valueRef, 1),
@@ -1099,11 +1125,7 @@ ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                             return SpanHelpers.IndexOfChar(ref spanRef, valueRef, span.Length);
 
                         case 2:
-                            return SpanHelpers.IndexOfAny(
-                                ref spanRef,
-                                valueRef,
-                                Unsafe.Add(ref valueRef, 1),
-                                span.Length);
+                            return SpanHelpers.IndexOfAnyChar(ref spanRef, valueRef, Unsafe.Add(ref valueRef, 1), span.Length);
 
                         case 3:
                             return SpanHelpers.IndexOfAny(
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
index 759e8a805c00f9..8b02943e8f831c 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
@@ -528,324 +528,6 @@ public static unsafe int IndexOfNullByte(ref byte searchSpace)
             return (int)(offset + 7);
         }
 
-        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        public static int IndexOfAny(ref byte searchSpace, byte value0, byte value1, int length)
-        {
-            Debug.Assert(length >= 0);
-
-            uint uValue0 = value0; // Use uint for comparisons to avoid unnecessary 8->32 extensions
-            uint uValue1 = value1; // Use uint for comparisons to avoid unnecessary 8->32 extensions
-            nuint offset = 0; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations
-            nuint lengthToExamine = (nuint)(uint)length;
-
-            if (Sse2.IsSupported || AdvSimd.Arm64.IsSupported)
-            {
-                // Avx2 branch also operates on Sse2 sizes, so check is combined.
-                nint vectorDiff = (nint)length - Vector128<byte>.Count;
-                if (vectorDiff >= 0)
-                {
-                    // >= Sse2 intrinsics are supported, and length is enough to use them so use that path.
-                    // We jump forward to the intrinsics at the end of the method so a naive branch predict
-                    // will choose the non-intrinsic path so short lengths which don't gain anything aren't
-                    // overly disadvantaged by having to jump over a lot of code. Whereas the longer lengths
-                    // more than make this back from the intrinsics.
-                    lengthToExamine = (nuint)vectorDiff;
-                    goto IntrinsicsCompare;
-                }
-            }
-            else if (Vector.IsHardwareAccelerated)
-            {
-                // Calculate lengthToExamine here for test, as it is used later
-                nint vectorDiff = (nint)length - Vector<byte>.Count;
-                if (vectorDiff >= 0)
-                {
-                    // Similar as above for Vector version
-                    lengthToExamine = (nuint)vectorDiff;
-                    goto IntrinsicsCompare;
-                }
-            }
-
-            uint lookUp;
-            while (lengthToExamine >= 8)
-            {
-                lengthToExamine -= 8;
-
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 1);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found1;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 2);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found2;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 3);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found3;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 4);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found4;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 5);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found5;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 6);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found6;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 7);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found7;
-
-                offset += 8;
-            }
-
-            if (lengthToExamine >= 4)
-            {
-                lengthToExamine -= 4;
-
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 1);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found1;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 2);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found2;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 3);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found3;
-
-                offset += 4;
-            }
-
-            while (lengthToExamine > 0)
-            {
-
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset);
-                if (uValue0 == lookUp || uValue1 == lookUp)
-                    goto Found;
-
-                offset += 1;
-                lengthToExamine -= 1;
-            }
-
-        NotFound:
-            return -1;
-        Found: // Workaround for https://github.com/dotnet/runtime/issues/8795
-            return (int)offset;
-        Found1:
-            return (int)(offset + 1);
-        Found2:
-            return (int)(offset + 2);
-        Found3:
-            return (int)(offset + 3);
-        Found4:
-            return (int)(offset + 4);
-        Found5:
-            return (int)(offset + 5);
-        Found6:
-            return (int)(offset + 6);
-        Found7:
-            return (int)(offset + 7);
-
-        IntrinsicsCompare:
-            // When we move into a Vectorized block, we process everything of Vector size;
-            // and then for any remainder we do a final compare of Vector size but starting at
-            // the end and forwards, which may overlap on an earlier compare.
-
-            // We include the Supported check again here even though path will not be taken, so the asm isn't generated if not supported.
-            if (Sse2.IsSupported)
-            {
-                int matches;
-                if (Avx2.IsSupported)
-                {
-                    Vector256<byte> search;
-                    // Guard as we may only have a valid size for Vector128; when we will move to the Sse2
-                    // We have already subtracted Vector128<byte>.Count from lengthToExamine so compare against that
-                    // to see if we have double the size for Vector256<byte>.Count
-                    if (lengthToExamine >= (nuint)Vector128<byte>.Count)
-                    {
-                        Vector256<byte> values0 = Vector256.Create(value0);
-                        Vector256<byte> values1 = Vector256.Create(value1);
-
-                        // Subtract Vector128<byte>.Count so we have now subtracted Vector256<byte>.Count
-                        lengthToExamine -= (nuint)Vector128<byte>.Count;
-                        // First time this checks again against 0, however we will move into final compare if it fails.
-                        while (lengthToExamine > offset)
-                        {
-                            search = LoadVector256(ref searchSpace, offset);
-                            // Bitwise Or to combine the flagged matches for the second value to our match flags
-                            matches = Avx2.MoveMask(
-                                            Avx2.Or(
-                                                Avx2.CompareEqual(values0, search),
-                                                Avx2.CompareEqual(values1, search)));
-                            // Note that MoveMask has converted the equal vector elements into a set of bit flags,
-                            // So the bit position in 'matches' corresponds to the element offset.
-                            if (matches == 0)
-                            {
-                                // None matched
-                                offset += (nuint)Vector256<byte>.Count;
-                                continue;
-                            }
-
-                            goto IntrinsicsMatch;
-                        }
-
-                        // Move to Vector length from end for final compare
-                        search = LoadVector256(ref searchSpace, lengthToExamine);
-                        offset = lengthToExamine;
-                        // Same as method as above
-                        matches = Avx2.MoveMask(
-                                    Avx2.Or(
-                                        Avx2.CompareEqual(values0, search),
-                                        Avx2.CompareEqual(values1, search)));
-                        if (matches == 0)
-                        {
-                            // None matched
-                            goto NotFound;
-                        }
-
-                        goto IntrinsicsMatch;
-                    }
-                }
-
-                // Initial size check was done on method entry.
-                Debug.Assert(length >= Vector128<byte>.Count);
-                {
-                    Vector128<byte> search;
-                    Vector128<byte> values0 = Vector128.Create(value0);
-                    Vector128<byte> values1 = Vector128.Create(value1);
-                    // First time this checks against 0 and we will move into final compare if it fails.
-                    while (lengthToExamine > offset)
-                    {
-                        search = LoadVector128(ref searchSpace, offset);
-
-                        matches = Sse2.MoveMask(
-                            Sse2.Or(
-                                Sse2.CompareEqual(values0, search),
-                                Sse2.CompareEqual(values1, search))
-                            .AsByte());
-                        // Note that MoveMask has converted the equal vector elements into a set of bit flags,
-                        // So the bit position in 'matches' corresponds to the element offset.
-                        if (matches == 0)
-                        {
-                            // None matched
-                            offset += (nuint)Vector128<byte>.Count;
-                            continue;
-                        }
-
-                        goto IntrinsicsMatch;
-                    }
-                    // Move to Vector length from end for final compare
-                    search = LoadVector128(ref searchSpace, lengthToExamine);
-                    offset = lengthToExamine;
-                    // Same as method as above
-                    matches = Sse2.MoveMask(
-                        Sse2.Or(
-                            Sse2.CompareEqual(values0, search),
-                            Sse2.CompareEqual(values1, search)));
-                    if (matches == 0)
-                    {
-                        // None matched
-                        goto NotFound;
-                    }
-                }
-
-            IntrinsicsMatch:
-                // Find bitflag offset of first difference and add to current offset
-                offset += (nuint)BitOperations.TrailingZeroCount(matches);
-                goto Found;
-            }
-            else if (AdvSimd.Arm64.IsSupported)
-            {
-                Vector128<byte> search;
-                Vector128<byte> matches;
-                Vector128<byte> values0 = Vector128.Create(value0);
-                Vector128<byte> values1 = Vector128.Create(value1);
-                // First time this checks against 0 and we will move into final compare if it fails.
-                while (lengthToExamine > offset)
-                {
-                    search = LoadVector128(ref searchSpace, offset);
-
-                    matches = AdvSimd.Or(
-                            AdvSimd.CompareEqual(values0, search),
-                            AdvSimd.CompareEqual(values1, search));
-
-                    if (matches == Vector128<byte>.Zero)
-                    {
-                        offset += (nuint)Vector128<byte>.Count;
-                        continue;
-                    }
-
-                    // Find bitflag offset of first match and add to current offset
-                    offset += FindFirstMatchedLane(matches);
-
-                    goto Found;
-                }
-
-                // Move to Vector length from end for final compare
-                search = LoadVector128(ref searchSpace, lengthToExamine);
-                offset = lengthToExamine;
-                // Same as method as above
-                matches = AdvSimd.Or(
-                        AdvSimd.CompareEqual(values0, search),
-                        AdvSimd.CompareEqual(values1, search));
-
-                if (matches == Vector128<byte>.Zero)
-                {
-                    // None matched
-                    goto NotFound;
-                }
-
-                // Find bitflag offset of first match and add to current offset
-                offset += FindFirstMatchedLane(matches);
-
-                goto Found;
-            }
-            else if (Vector.IsHardwareAccelerated)
-            {
-                Vector<byte> values0 = new Vector<byte>(value0);
-                Vector<byte> values1 = new Vector<byte>(value1);
-
-                Vector<byte> search;
-                // First time this checks against 0 and we will move into final compare if it fails.
-                while (lengthToExamine > offset)
-                {
-                    search = LoadVector(ref searchSpace, offset);
-                    search = Vector.BitwiseOr(
-                                    Vector.Equals(search, values0),
-                                    Vector.Equals(search, values1));
-                    if (Vector<byte>.Zero.Equals(search))
-                    {
-                        // None matched
-                        offset += (nuint)Vector<byte>.Count;
-                        continue;
-                    }
-
-                    goto VectorMatch;
-                }
-
-                // Move to Vector length from end for final compare
-                search = LoadVector(ref searchSpace, lengthToExamine);
-                offset = lengthToExamine;
-                search = Vector.BitwiseOr(
-                                Vector.Equals(search, values0),
-                                Vector.Equals(search, values1));
-                if (Vector<byte>.Zero.Equals(search))
-                {
-                    // None matched
-                    goto NotFound;
-                }
-
-            VectorMatch:
-                offset += (nuint)LocateFirstFoundByte(search);
-                goto Found;
-            }
-
-            Debug.Fail("Unreachable");
-            goto NotFound;
-        }
-
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
         public static int IndexOfAny(ref byte searchSpace, byte value0, byte value1, byte value2, int length)
         {
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
index 5cd0d2efc0a290..3f90f7031b1214 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
@@ -624,243 +624,6 @@ public static unsafe int IndexOfNullCharacter(ref char searchSpace)
             return (int)(offset);
         }
 
-        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        public static unsafe int IndexOfAny(ref char searchStart, char value0, char value1, int length)
-        {
-            Debug.Assert(length >= 0);
-
-            nuint offset = 0; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations
-            nuint lengthToExamine = (nuint)(uint)length;
-
-            if (Sse2.IsSupported)
-            {
-                // Calculate lengthToExamine here for test, rather than just testing as it used later, rather than doing it twice.
-                nint vectorDiff = (nint)length - Vector128<ushort>.Count;
-                if (vectorDiff >= 0)
-                {
-                    // >= Sse2 intrinsics are supported and length is enough to use them, so use that path.
-                    // We jump forward to the intrinsics at the end of them method so a naive branch predict
-                    // will choose the non-intrinsic path so short lengths which don't gain anything aren't
-                    // overly disadvantaged by having to jump over a lot of code. Whereas the longer lengths
-                    // more than make this back from the intrinsics.
-                    lengthToExamine = (nuint)vectorDiff;
-                    goto IntrinsicsCompare;
-                }
-            }
-            else if (Vector.IsHardwareAccelerated)
-            {
-                // Calculate lengthToExamine here for test, rather than just testing as it used later, rather than doing it twice.
-                nint vectorDiff = (nint)length - Vector<ushort>.Count;
-                if (vectorDiff >= 0)
-                {
-                    // Similar as above for Vector version
-                    lengthToExamine = (nuint)vectorDiff;
-                    goto VectorCompare;
-                }
-            }
-
-            int lookUp;
-            while (lengthToExamine >= 4)
-            {
-                ref char current = ref Add(ref searchStart, offset);
-
-                lookUp = current;
-                if (value0 == lookUp || value1 == lookUp)
-                    goto Found;
-                lookUp = Unsafe.Add(ref current, 1);
-                if (value0 == lookUp || value1 == lookUp)
-                    goto Found1;
-                lookUp = Unsafe.Add(ref current, 2);
-                if (value0 == lookUp || value1 == lookUp)
-                    goto Found2;
-                lookUp = Unsafe.Add(ref current, 3);
-                if (value0 == lookUp || value1 == lookUp)
-                    goto Found3;
-
-                offset += 4;
-                lengthToExamine -= 4;
-            }
-
-            while (lengthToExamine > 0)
-            {
-                lookUp = Add(ref searchStart, offset);
-                if (value0 == lookUp || value1 == lookUp)
-                    goto Found;
-
-                offset += 1;
-                lengthToExamine -= 1;
-            }
-
-        NotFound:
-            return -1;
-        Found3:
-            return (int)(offset + 3);
-        Found2:
-            return (int)(offset + 2);
-        Found1:
-            return (int)(offset + 1);
-        Found:
-            return (int)offset;
-
-        IntrinsicsCompare:
-            // When we move into a Vectorized block, we process everything of Vector size;
-            // and then for any remainder we do a final compare of Vector size but starting at
-            // the end and forwards, which may overlap on an earlier compare.
-
-            // We include the Supported check again here even though path will not be taken, so the asm isn't generated if not supported.
-            if (Sse2.IsSupported)
-            {
-                int matches;
-                if (Avx2.IsSupported)
-                {
-                    Vector256<ushort> search;
-                    // Guard as we may only have a valid size for Vector128; when we will move to the Sse2
-                    // We have already subtracted Vector128<ushort>.Count from lengthToExamine so compare against that
-                    // to see if we have double the size for Vector256<ushort>.Count
-                    if (lengthToExamine >= (nuint)Vector128<ushort>.Count)
-                    {
-                        Vector256<ushort> values0 = Vector256.Create((ushort)value0);
-                        Vector256<ushort> values1 = Vector256.Create((ushort)value1);
-
-                        // Subtract Vector128<ushort>.Count so we have now subtracted Vector256<ushort>.Count
-                        lengthToExamine -= (nuint)Vector128<ushort>.Count;
-                        // First time this checks again against 0, however we will move into final compare if it fails.
-                        while (lengthToExamine > offset)
-                        {
-                            search = LoadVector256(ref searchStart, offset);
-                            // Bitwise Or to combine the flagged matches for the second value to our match flags
-                            matches = Avx2.MoveMask(
-                                            Avx2.Or(
-                                                Avx2.CompareEqual(values0, search),
-                                                Avx2.CompareEqual(values1, search))
-                                            .AsByte());
-                            // Note that MoveMask has converted the equal vector elements into a set of bit flags,
-                            // So the bit position in 'matches' corresponds to the element offset.
-                            if (matches == 0)
-                            {
-                                // None matched
-                                offset += (nuint)Vector256<ushort>.Count;
-                                continue;
-                            }
-
-                            goto IntrinsicsMatch;
-                        }
-
-                        // Move to Vector length from end for final compare
-                        search = LoadVector256(ref searchStart, lengthToExamine);
-                        offset = lengthToExamine;
-                        // Same as method as above
-                        matches = Avx2.MoveMask(
-                                    Avx2.Or(
-                                        Avx2.CompareEqual(values0, search),
-                                        Avx2.CompareEqual(values1, search))
-                                    .AsByte());
-                        if (matches == 0)
-                        {
-                            // None matched
-                            goto NotFound;
-                        }
-
-                        goto IntrinsicsMatch;
-                    }
-                }
-
-                // Initial size check was done on method entry.
-                Debug.Assert(length >= Vector128<ushort>.Count);
-                {
-                    Vector128<ushort> search;
-                    Vector128<ushort> values0 = Vector128.Create((ushort)value0);
-                    Vector128<ushort> values1 = Vector128.Create((ushort)value1);
-                    // First time this checks against 0 and we will move into final compare if it fails.
-                    while (lengthToExamine > offset)
-                    {
-                        search = LoadVector128(ref searchStart, offset);
-
-                        matches = Sse2.MoveMask(
-                            Sse2.Or(
-                                Sse2.CompareEqual(values0, search),
-                                Sse2.CompareEqual(values1, search))
-                            .AsByte());
-                        // Note that MoveMask has converted the equal vector elements into a set of bit flags,
-                        // So the bit position in 'matches' corresponds to the element offset.
-                        if (matches == 0)
-                        {
-                            // None matched
-                            offset += (nuint)Vector128<ushort>.Count;
-                            continue;
-                        }
-
-                        goto IntrinsicsMatch;
-                    }
-                    // Move to Vector length from end for final compare
-                    search = LoadVector128(ref searchStart, lengthToExamine);
-                    offset = lengthToExamine;
-                    // Same as method as above
-                    matches = Sse2.MoveMask(
-                        Sse2.Or(
-                            Sse2.CompareEqual(values0, search),
-                            Sse2.CompareEqual(values1, search))
-                        .AsByte());
-                    if (matches == 0)
-                    {
-                        // None matched
-                        goto NotFound;
-                    }
-                }
-
-            IntrinsicsMatch:
-                // Find bitflag offset of first difference and add to current offset,
-                // flags are in bytes so divide by 2 for chars (shift right by 1)
-                offset += (nuint)(uint)BitOperations.TrailingZeroCount(matches) >> 1;
-                goto Found;
-            }
-
-        VectorCompare:
-            // We include the Supported check again here even though path will not be taken, so the asm isn't generated if not supported.
-            if (!Sse2.IsSupported && Vector.IsHardwareAccelerated)
-            {
-                Vector<ushort> values0 = new Vector<ushort>(value0);
-                Vector<ushort> values1 = new Vector<ushort>(value1);
-
-                Vector<ushort> search;
-                // First time this checks against 0 and we will move into final compare if it fails.
-                while (lengthToExamine > offset)
-                {
-                    search = LoadVector(ref searchStart, offset);
-                    search = Vector.BitwiseOr(
-                                    Vector.Equals(search, values0),
-                                    Vector.Equals(search, values1));
-                    if (Vector<ushort>.Zero.Equals(search))
-                    {
-                        // None matched
-                        offset += (nuint)Vector<ushort>.Count;
-                        continue;
-                    }
-
-                    goto VectorMatch;
-                }
-
-                // Move to Vector length from end for final compare
-                search = LoadVector(ref searchStart, lengthToExamine);
-                offset = lengthToExamine;
-                search = Vector.BitwiseOr(
-                                Vector.Equals(search, values0),
-                                Vector.Equals(search, values1));
-                if (Vector<ushort>.Zero.Equals(search))
-                {
-                    // None matched
-                    goto NotFound;
-                }
-
-            VectorMatch:
-                offset += (nuint)(uint)LocateFirstFoundChar(search);
-                goto Found;
-            }
-
-            Debug.Fail("Unreachable");
-            goto NotFound;
-        }
-
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
         public static unsafe int IndexOfAny(ref char searchStart, char value0, char value1, char value2, int length)
         {
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
index 3e05a41f1d3500..272f6989c17b30 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
@@ -1354,6 +1354,107 @@ private static int IndexOfValueType<T, N>(ref T searchSpace, T value, int length
             return -1;
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static int IndexOfAnyChar(ref char searchSpace, char value0, char value1, int length)
+            => IndexOfAnyValueType(ref Unsafe.As<char, short>(ref searchSpace), (short)value0, (short)value1, length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static int IndexOfAnyValueType<T>(ref T searchSpace, T value0, T value1, int length) where T : struct, IEquatable<T>
+            => IndexOfAnyValueType<T, DontNegate<T>>(ref searchSpace, value0, value1, length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static int IndexOfAnyExceptValueType<T>(ref T searchSpace, T value0, T value1, int length) where T : struct, IEquatable<T>
+            => IndexOfAnyValueType<T, Negate<T>>(ref searchSpace, value0, value1, length);
+
+        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
+        private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T value1, int length)
+            where T : struct, IEquatable<T>
+            where N : struct, INegator<T>
+        {
+            Debug.Assert(length >= 0, "Expected non-negative length");
+            Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types");
+
+            N negator = default;
+
+            if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
+            {
+                for (int i = 0; i < length; i++)
+                {
+                    T current = Unsafe.Add(ref searchSpace, i);
+                    if (negator.NegateIfNeeded(current.Equals(value0) | current.Equals(value1)))
+                    {
+                        return i;
+                    }
+                }
+            }
+            else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
+            {
+                Vector256<T> equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1);
+                ref T currentSearchSpace = ref searchSpace;
+                ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256<T>.Count);
+
+                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
+                do
+                {
+                    current = Vector256.LoadUnsafe(ref currentSearchSpace);
+                    equals = negator.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current));
+                    if (equals == Vector256<T>.Zero)
+                    {
+                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector256<T>.Count);
+                        continue;
+                    }
+
+                    return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals);
+                }
+                while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd));
+
+                // If any elements remain, process the last vector in the search space.
+                if ((uint)length % Vector256<T>.Count != 0)
+                {
+                    current = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd);
+                    equals = negator.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current));
+                    if (equals != Vector256<T>.Zero)
+                    {
+                        return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
+                    }
+                }
+            }
+            else
+            {
+                Vector128<T> equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1);
+                ref T currentSearchSpace = ref searchSpace;
+                ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128<T>.Count);
+
+                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
+                do
+                {
+                    current = Vector128.LoadUnsafe(ref currentSearchSpace);
+                    equals = negator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current));
+                    if (equals == Vector128<T>.Zero)
+                    {
+                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128<T>.Count);
+                        continue;
+                    }
+
+                    return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals);
+                }
+                while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd));
+
+                // If any elements remain, process the first vector in the search space.
+                if ((uint)length % Vector128<T>.Count != 0)
+                {
+                    current = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd);
+                    equals = negator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current));
+                    if (equals != Vector128<T>.Zero)
+                    {
+                        return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
+                    }
+                }
+            }
+
+            return -1;
+        }
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal static int LastIndexOfValueType<T>(ref T searchSpace, T value, int length) where T : struct, IEquatable<T>
             => LastIndexOfValueType<T, DontNegate<T>>(ref searchSpace, value, length);
diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs
index 0f0dbe2b50b29d..df95295a116fa5 100644
--- a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs
@@ -78,7 +78,7 @@ private int IndexOfCharOrdinalIgnoreCase(char value)
             {
                 char valueUc = (char)(value | 0x20);
                 char valueLc = (char)(value & ~0x20);
-                return SpanHelpers.IndexOfAny(ref _firstChar, valueLc, valueUc, Length);
+                return SpanHelpers.IndexOfAnyChar(ref _firstChar, valueLc, valueUc, Length);
             }
 
             return SpanHelpers.IndexOfChar(ref _firstChar, value, Length);

From 9186b8f9eb405dc50529859793b5ac76fbc59ed8 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Fri, 12 Aug 2022 17:04:45 +0200
Subject: [PATCH 15/34] vectorize IndexOfAny<T>(value0, value1, value2) and
 IndexOfAnyExcept<T>(value0, value1, value2)

---
 .../src/System/MemoryExtensions.cs            |  72 +++--
 .../src/System/SpanHelpers.Byte.cs            | 268 ------------------
 .../src/System/SpanHelpers.Char.cs            | 252 ----------------
 .../src/System/SpanHelpers.T.cs               |  97 +++++++
 4 files changed, 147 insertions(+), 542 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
index a25803a5430302..d0dfae8c684d39 100644
--- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
@@ -479,6 +479,28 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
         /// </returns>
         public static int IndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value0, T value1, T value2) where T : IEquatable<T>?
         {
+            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            {
+                if (Unsafe.SizeOf<T>() == sizeof(byte))
+                {
+                    return SpanHelpers.IndexOfAnyExceptValueType(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, byte>(ref value0),
+                        Unsafe.As<T, byte>(ref value1),
+                        Unsafe.As<T, byte>(ref value2),
+                        span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
+                {
+                    return SpanHelpers.IndexOfAnyExceptValueType(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value0),
+                        Unsafe.As<T, short>(ref value1),
+                        Unsafe.As<T, short>(ref value2),
+                        span.Length);
+                }
+            }
+
             for (int i = 0; i < span.Length; i++)
             {
                 if (!EqualityComparer<T>.Default.Equals(span[i], value0) &&
@@ -988,20 +1010,23 @@ public static int IndexOfAny<T>(this Span<T> span, T value0, T value1, T value2)
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
-                    return SpanHelpers.IndexOfAny(
+                {
+                    return SpanHelpers.IndexOfAnyValueType(
                         ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, byte>(ref value0),
                         Unsafe.As<T, byte>(ref value1),
                         Unsafe.As<T, byte>(ref value2),
                         span.Length);
-
-                if (Unsafe.SizeOf<T>() == sizeof(char))
-                    return SpanHelpers.IndexOfAny(
-                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, char>(ref value0),
-                        Unsafe.As<T, char>(ref value1),
-                        Unsafe.As<T, char>(ref value2),
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
+                {
+                    return SpanHelpers.IndexOfAnyValueType(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value0),
+                        Unsafe.As<T, short>(ref value1),
+                        Unsafe.As<T, short>(ref value2),
                         span.Length);
+                }
             }
 
             return SpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, value2, span.Length);
@@ -1061,20 +1086,23 @@ public static int IndexOfAny<T>(this ReadOnlySpan<T> span, T value0, T value1, T
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
-                    return SpanHelpers.IndexOfAny(
+                {
+                    return SpanHelpers.IndexOfAnyValueType(
                         ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, byte>(ref value0),
                         Unsafe.As<T, byte>(ref value1),
                         Unsafe.As<T, byte>(ref value2),
                         span.Length);
-
-                if (Unsafe.SizeOf<T>() == sizeof(char))
-                    return SpanHelpers.IndexOfAny(
-                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, char>(ref value0),
-                        Unsafe.As<T, char>(ref value1),
-                        Unsafe.As<T, char>(ref value2),
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
+                {
+                    return SpanHelpers.IndexOfAnyValueType(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value0),
+                        Unsafe.As<T, short>(ref value1),
+                        Unsafe.As<T, short>(ref value2),
                         span.Length);
+                }
             }
 
             return SpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, value2, span.Length);
@@ -1103,7 +1131,7 @@ ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                     }
                     else if (values.Length == 3)
                     {
-                        return SpanHelpers.IndexOfAny(
+                        return SpanHelpers.IndexOfAnyValueType(
                             ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                             valueRef,
                             Unsafe.Add(ref valueRef, 1),
@@ -1128,11 +1156,11 @@ ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                             return SpanHelpers.IndexOfAnyChar(ref spanRef, valueRef, Unsafe.Add(ref valueRef, 1), span.Length);
 
                         case 3:
-                            return SpanHelpers.IndexOfAny(
-                                ref spanRef,
-                                valueRef,
-                                Unsafe.Add(ref valueRef, 1),
-                                Unsafe.Add(ref valueRef, 2),
+                            return SpanHelpers.IndexOfAnyValueType(
+                                ref Unsafe.As<char, short>(ref spanRef),
+                                Unsafe.As<char, short>(ref valueRef),
+                                Unsafe.As<char, short>(ref Unsafe.Add(ref valueRef, 1)),
+                                Unsafe.As<char, short>(ref Unsafe.Add(ref valueRef, 2)),
                                 span.Length);
 
                         case 4:
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
index 8b02943e8f831c..dc13e0e7bf06b0 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
@@ -528,274 +528,6 @@ public static unsafe int IndexOfNullByte(ref byte searchSpace)
             return (int)(offset + 7);
         }
 
-        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        public static int IndexOfAny(ref byte searchSpace, byte value0, byte value1, byte value2, int length)
-        {
-            Debug.Assert(length >= 0);
-
-            uint uValue0 = value0; // Use uint for comparisons to avoid unnecessary 8->32 extensions
-            uint uValue1 = value1; // Use uint for comparisons to avoid unnecessary 8->32 extensions
-            uint uValue2 = value2; // Use uint for comparisons to avoid unnecessary 8->32 extensions
-            nuint offset = 0; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations
-            nuint lengthToExamine = (nuint)(uint)length;
-
-            if (Vector128.IsHardwareAccelerated)
-            {
-                // Avx2 branch also operates on Sse2 sizes, so check is combined.
-                nint vectorDiff = (nint)length - Vector128<byte>.Count;
-                if (vectorDiff >= 0)
-                {
-                    // >= Vector128 is accelerated, and length is enough to use them so use that path.
-                    // We jump forward to the intrinsics at the end of the method so a naive branch predict
-                    // will choose the non-intrinsic path so short lengths which don't gain anything aren't
-                    // overly disadvantaged by having to jump over a lot of code. Whereas the longer lengths
-                    // more than make this back from the intrinsics.
-                    lengthToExamine = (nuint)vectorDiff;
-                    goto IntrinsicsCompare;
-                }
-            }
-            else if (Vector.IsHardwareAccelerated)
-            {
-                // Calculate lengthToExamine here for test, as it is used later
-                nint vectorDiff = (nint)length - Vector<byte>.Count;
-                if (vectorDiff >= 0)
-                {
-                    // Similar as above for Vector version
-                    lengthToExamine = (nuint)vectorDiff;
-                    goto IntrinsicsCompare;
-                }
-            }
-
-            uint lookUp;
-            while (lengthToExamine >= 8)
-            {
-                lengthToExamine -= 8;
-
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 1);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found1;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 2);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found2;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 3);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found3;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 4);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found4;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 5);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found5;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 6);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found6;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 7);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found7;
-
-                offset += 8;
-            }
-
-            if (lengthToExamine >= 4)
-            {
-                lengthToExamine -= 4;
-
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 1);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found1;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 2);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found2;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 3);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found3;
-
-                offset += 4;
-            }
-
-            while (lengthToExamine > 0)
-            {
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found;
-
-                offset += 1;
-                lengthToExamine -= 1;
-            }
-
-        NotFound:
-            return -1;
-        Found: // Workaround for https://github.com/dotnet/runtime/issues/8795
-            return (int)offset;
-        Found1:
-            return (int)(offset + 1);
-        Found2:
-            return (int)(offset + 2);
-        Found3:
-            return (int)(offset + 3);
-        Found4:
-            return (int)(offset + 4);
-        Found5:
-            return (int)(offset + 5);
-        Found6:
-            return (int)(offset + 6);
-        Found7:
-            return (int)(offset + 7);
-
-        IntrinsicsCompare:
-            // When we move into a Vectorized block, we process everything of Vector size;
-            // and then for any remainder we do a final compare of Vector size but starting at
-            // the end and forwards, which may overlap on an earlier compare.
-
-            // We include the Supported check again here even though path will not be taken, so the asm isn't generated if not supported.
-            if (Vector128.IsHardwareAccelerated)
-            {
-                uint matches;
-                if (Vector256.IsHardwareAccelerated)
-                {
-                    Vector256<byte> search;
-                    // Guard as we may only have a valid size for Vector128; when we will move to the Sse2
-                    // We have already subtracted Vector128<byte>.Count from lengthToExamine so compare against that
-                    // to see if we have double the size for Vector256<byte>.Count
-                    if (lengthToExamine >= (nuint)Vector128<byte>.Count)
-                    {
-                        Vector256<byte> values0 = Vector256.Create(value0);
-                        Vector256<byte> values1 = Vector256.Create(value1);
-                        Vector256<byte> values2 = Vector256.Create(value2);
-
-                        // Subtract Vector128<byte>.Count so we have now subtracted Vector256<byte>.Count
-                        lengthToExamine -= (nuint)Vector128<byte>.Count;
-                        // First time this checks again against 0, however we will move into final compare if it fails.
-                        while (lengthToExamine > offset)
-                        {
-                            search = Vector256.LoadUnsafe(ref searchSpace, offset);
-                            // Bitwise Or to combine the flagged matches for the second value to our match flags
-                            matches = (Vector256.Equals(values0, search) | Vector256.Equals(values1, search) | Vector256.Equals(values2, search)).ExtractMostSignificantBits();
-                            // Note that ExtractMostSignificantBits has converted the equal vector elements into a set of bit flags,
-                            // So the bit position in 'matches' corresponds to the element offset.
-                            if (matches == 0)
-                            {
-                                // None matched
-                                offset += (nuint)Vector256<byte>.Count;
-                                continue;
-                            }
-
-                            goto IntrinsicsMatch;
-                        }
-
-                        // Move to Vector length from end for final compare
-                        search = Vector256.LoadUnsafe(ref searchSpace, lengthToExamine);
-                        offset = lengthToExamine;
-                        // Same as method as above
-                        matches = (Vector256.Equals(values0, search) | Vector256.Equals(values1, search) | Vector256.Equals(values2, search)).ExtractMostSignificantBits();
-                        if (matches == 0)
-                        {
-                            // None matched
-                            goto NotFound;
-                        }
-
-                        goto IntrinsicsMatch;
-                    }
-                }
-
-                // Initial size check was done on method entry.
-                Debug.Assert(length >= Vector128<byte>.Count);
-                {
-                    Vector128<byte> search, compareResult;
-                    Vector128<byte> values0 = Vector128.Create(value0);
-                    Vector128<byte> values1 = Vector128.Create(value1);
-                    Vector128<byte> values2 = Vector128.Create(value2);
-                    // First time this checks against 0 and we will move into final compare if it fails.
-                    while (lengthToExamine > offset)
-                    {
-                        search = Vector128.LoadUnsafe(ref searchSpace, offset);
-
-                        compareResult = Vector128.Equals(values0, search) | Vector128.Equals(values1, search) | Vector128.Equals(values2, search);
-                        if (compareResult == Vector128<byte>.Zero)
-                        {
-                            // None matched
-                            offset += (nuint)Vector128<byte>.Count;
-                            continue;
-                        }
-
-                        // Note that ExtractMostSignificantBits has converted the equal vector elements into a set of bit flags,
-                        // So the bit position in 'matches' corresponds to the element offset.
-                        matches = compareResult.ExtractMostSignificantBits();
-                        goto IntrinsicsMatch;
-                    }
-                    // Move to Vector length from end for final compare
-                    search = Vector128.LoadUnsafe(ref searchSpace, lengthToExamine);
-                    offset = lengthToExamine;
-                    // Same as method as above
-                    compareResult = Vector128.Equals(values0, search) | Vector128.Equals(values1, search) | Vector128.Equals(values2, search);
-                    if (compareResult == Vector128<byte>.Zero)
-                    {
-                        // None matched
-                        goto NotFound;
-                    }
-                    matches = compareResult.ExtractMostSignificantBits();
-                }
-
-            IntrinsicsMatch:
-                // Find bitflag offset of first difference and add to current offset
-                offset += (nuint)BitOperations.TrailingZeroCount(matches);
-                goto Found;
-            }
-            else if (Vector.IsHardwareAccelerated)
-            {
-                Vector<byte> values0 = new Vector<byte>(value0);
-                Vector<byte> values1 = new Vector<byte>(value1);
-                Vector<byte> values2 = new Vector<byte>(value2);
-
-                Vector<byte> search;
-                // First time this checks against 0 and we will move into final compare if it fails.
-                while (lengthToExamine > offset)
-                {
-                    search = LoadVector(ref searchSpace, offset);
-                    search = Vector.BitwiseOr(
-                                Vector.BitwiseOr(
-                                    Vector.Equals(search, values0),
-                                    Vector.Equals(search, values1)),
-                                Vector.Equals(search, values2));
-                    if (Vector<byte>.Zero.Equals(search))
-                    {
-                        // None matched
-                        offset += (nuint)Vector<byte>.Count;
-                        continue;
-                    }
-
-                    goto VectorMatch;
-                }
-
-                // Move to Vector length from end for final compare
-                search = LoadVector(ref searchSpace, lengthToExamine);
-                offset = lengthToExamine;
-                search = Vector.BitwiseOr(
-                            Vector.BitwiseOr(
-                                Vector.Equals(search, values0),
-                                Vector.Equals(search, values1)),
-                            Vector.Equals(search, values2));
-                if (Vector<byte>.Zero.Equals(search))
-                {
-                    // None matched
-                    goto NotFound;
-                }
-
-            VectorMatch:
-                offset += (nuint)LocateFirstFoundByte(search);
-                goto Found;
-            }
-
-            Debug.Fail("Unreachable");
-            goto NotFound;
-        }
-
         public static int LastIndexOfAny(ref byte searchSpace, byte value0, byte value1, byte value2, int length)
         {
             Debug.Assert(length >= 0);
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
index 3f90f7031b1214..cc97ba6c4b2c5f 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
@@ -624,258 +624,6 @@ public static unsafe int IndexOfNullCharacter(ref char searchSpace)
             return (int)(offset);
         }
 
-        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        public static unsafe int IndexOfAny(ref char searchStart, char value0, char value1, char value2, int length)
-        {
-            Debug.Assert(length >= 0);
-
-            nuint offset = 0; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations
-            nuint lengthToExamine = (nuint)(uint)length;
-
-            if (Sse2.IsSupported)
-            {
-                // Calculate lengthToExamine here for test, rather than just testing as it used later, rather than doing it twice.
-                nint vectorDiff = (nint)length - Vector128<ushort>.Count;
-                if (vectorDiff >= 0)
-                {
-                    // >= Sse2 intrinsics are supported and length is enough to use them, so use that path.
-                    // We jump forward to the intrinsics at the end of them method so a naive branch predict
-                    // will choose the non-intrinsic path so short lengths which don't gain anything aren't
-                    // overly disadvantaged by having to jump over a lot of code. Whereas the longer lengths
-                    // more than make this back from the intrinsics.
-                    lengthToExamine = (nuint)vectorDiff;
-                    goto IntrinsicsCompare;
-                }
-            }
-            else if (Vector.IsHardwareAccelerated)
-            {
-                // Calculate lengthToExamine here for test, rather than just testing as it used later, rather than doing it twice.
-                nint vectorDiff = (nint)length - Vector<ushort>.Count;
-                if (vectorDiff >= 0)
-                {
-                    // Similar as above for Vector version
-                    lengthToExamine = (nuint)vectorDiff;
-                    goto VectorCompare;
-                }
-            }
-
-            int lookUp;
-            while (lengthToExamine >= 4)
-            {
-                ref char current = ref Add(ref searchStart, offset);
-
-                lookUp = current;
-                if (value0 == lookUp || value1 == lookUp || value2 == lookUp)
-                    goto Found;
-                lookUp = Unsafe.Add(ref current, 1);
-                if (value0 == lookUp || value1 == lookUp || value2 == lookUp)
-                    goto Found1;
-                lookUp = Unsafe.Add(ref current, 2);
-                if (value0 == lookUp || value1 == lookUp || value2 == lookUp)
-                    goto Found2;
-                lookUp = Unsafe.Add(ref current, 3);
-                if (value0 == lookUp || value1 == lookUp || value2 == lookUp)
-                    goto Found3;
-
-                offset += 4;
-                lengthToExamine -= 4;
-            }
-
-            while (lengthToExamine > 0)
-            {
-                lookUp = Add(ref searchStart, offset);
-                if (value0 == lookUp || value1 == lookUp || value2 == lookUp)
-                    goto Found;
-
-                offset += 1;
-                lengthToExamine -= 1;
-            }
-
-        NotFound:
-            return -1;
-        Found3:
-            return (int)(offset + 3);
-        Found2:
-            return (int)(offset + 2);
-        Found1:
-            return (int)(offset + 1);
-        Found:
-            return (int)offset;
-
-        IntrinsicsCompare:
-            // When we move into a Vectorized block, we process everything of Vector size;
-            // and then for any remainder we do a final compare of Vector size but starting at
-            // the end and forwards, which may overlap on an earlier compare.
-
-            // We include the Supported check again here even though path will not be taken, so the asm isn't generated if not supported.
-            if (Sse2.IsSupported)
-            {
-                int matches;
-                if (Avx2.IsSupported)
-                {
-                    Vector256<ushort> search;
-                    // Guard as we may only have a valid size for Vector128; when we will move to the Sse2
-                    // We have already subtracted Vector128<ushort>.Count from lengthToExamine so compare against that
-                    // to see if we have double the size for Vector256<ushort>.Count
-                    if (lengthToExamine >= (nuint)Vector128<ushort>.Count)
-                    {
-                        Vector256<ushort> values0 = Vector256.Create((ushort)value0);
-                        Vector256<ushort> values1 = Vector256.Create((ushort)value1);
-                        Vector256<ushort> values2 = Vector256.Create((ushort)value2);
-
-                        // Subtract Vector128<ushort>.Count so we have now subtracted Vector256<ushort>.Count
-                        lengthToExamine -= (nuint)Vector128<ushort>.Count;
-                        // First time this checks again against 0, however we will move into final compare if it fails.
-                        while (lengthToExamine > offset)
-                        {
-                            search = LoadVector256(ref searchStart, offset);
-                            // Bitwise Or to combine the flagged matches for the second value to our match flags
-                            matches = Avx2.MoveMask(
-                                            Avx2.Or(
-                                                Avx2.Or(
-                                                    Avx2.CompareEqual(values0, search),
-                                                    Avx2.CompareEqual(values1, search)),
-                                                Avx2.CompareEqual(values2, search))
-                                            .AsByte());
-                            // Note that MoveMask has converted the equal vector elements into a set of bit flags,
-                            // So the bit position in 'matches' corresponds to the element offset.
-                            if (matches == 0)
-                            {
-                                // None matched
-                                offset += (nuint)Vector256<ushort>.Count;
-                                continue;
-                            }
-
-                            goto IntrinsicsMatch;
-                        }
-
-                        // Move to Vector length from end for final compare
-                        search = LoadVector256(ref searchStart, lengthToExamine);
-                        offset = lengthToExamine;
-                        // Same as method as above
-                        matches = Avx2.MoveMask(
-                                    Avx2.Or(
-                                        Avx2.Or(
-                                            Avx2.CompareEqual(values0, search),
-                                            Avx2.CompareEqual(values1, search)),
-                                        Avx2.CompareEqual(values2, search))
-                                    .AsByte());
-                        if (matches == 0)
-                        {
-                            // None matched
-                            goto NotFound;
-                        }
-
-                        goto IntrinsicsMatch;
-                    }
-                }
-
-                // Initial size check was done on method entry.
-                Debug.Assert(length >= Vector128<ushort>.Count);
-                {
-                    Vector128<ushort> search;
-                    Vector128<ushort> values0 = Vector128.Create((ushort)value0);
-                    Vector128<ushort> values1 = Vector128.Create((ushort)value1);
-                    Vector128<ushort> values2 = Vector128.Create((ushort)value2);
-                    // First time this checks against 0 and we will move into final compare if it fails.
-                    while (lengthToExamine > offset)
-                    {
-                        search = LoadVector128(ref searchStart, offset);
-
-                        matches = Sse2.MoveMask(
-                                    Sse2.Or(
-                                        Sse2.Or(
-                                            Sse2.CompareEqual(values0, search),
-                                            Sse2.CompareEqual(values1, search)),
-                                        Sse2.CompareEqual(values2, search))
-                                    .AsByte());
-                        // Note that MoveMask has converted the equal vector elements into a set of bit flags,
-                        // So the bit position in 'matches' corresponds to the element offset.
-                        if (matches == 0)
-                        {
-                            // None matched
-                            offset += (nuint)Vector128<ushort>.Count;
-                            continue;
-                        }
-
-                        goto IntrinsicsMatch;
-                    }
-                    // Move to Vector length from end for final compare
-                    search = LoadVector128(ref searchStart, lengthToExamine);
-                    offset = lengthToExamine;
-                    // Same as method as above
-                    matches = Sse2.MoveMask(
-                                    Sse2.Or(
-                                        Sse2.Or(
-                                            Sse2.CompareEqual(values0, search),
-                                            Sse2.CompareEqual(values1, search)),
-                                        Sse2.CompareEqual(values2, search))
-                                    .AsByte());
-                    if (matches == 0)
-                    {
-                        // None matched
-                        goto NotFound;
-                    }
-                }
-
-            IntrinsicsMatch:
-                // Find bitflag offset of first difference and add to current offset,
-                // flags are in bytes so divide by 2 for chars (shift right by 1)
-                offset += (nuint)(uint)BitOperations.TrailingZeroCount(matches) >> 1;
-                goto Found;
-            }
-
-        VectorCompare:
-            // We include the Supported check again here even though path will not be taken, so the asm isn't generated if not supported.
-            if (!Sse2.IsSupported && Vector.IsHardwareAccelerated)
-            {
-                Vector<ushort> values0 = new Vector<ushort>(value0);
-                Vector<ushort> values1 = new Vector<ushort>(value1);
-                Vector<ushort> values2 = new Vector<ushort>(value2);
-
-                Vector<ushort> search;
-                // First time this checks against 0 and we will move into final compare if it fails.
-                while (lengthToExamine > offset)
-                {
-                    search = LoadVector(ref searchStart, offset);
-                    search = Vector.BitwiseOr(
-                                    Vector.BitwiseOr(
-                                        Vector.Equals(search, values0),
-                                        Vector.Equals(search, values1)),
-                                    Vector.Equals(search, values2));
-                    if (Vector<ushort>.Zero.Equals(search))
-                    {
-                        // None matched
-                        offset += (nuint)Vector<ushort>.Count;
-                        continue;
-                    }
-
-                    goto VectorMatch;
-                }
-
-                // Move to Vector length from end for final compare
-                search = LoadVector(ref searchStart, lengthToExamine);
-                offset = lengthToExamine;
-                search = Vector.BitwiseOr(
-                                    Vector.BitwiseOr(
-                                        Vector.Equals(search, values0),
-                                        Vector.Equals(search, values1)),
-                                    Vector.Equals(search, values2));
-                if (Vector<ushort>.Zero.Equals(search))
-                {
-                    // None matched
-                    goto NotFound;
-                }
-
-            VectorMatch:
-                offset += (nuint)(uint)LocateFirstFoundChar(search);
-                goto Found;
-            }
-
-            Debug.Fail("Unreachable");
-            goto NotFound;
-        }
-
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
         public static unsafe int IndexOfAny(ref char searchStart, char value0, char value1, char value2, char value3, int length)
         {
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
index 272f6989c17b30..ca17be4b4fcee9 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
@@ -1455,6 +1455,103 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
             return -1;
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static int IndexOfAnyValueType<T>(ref T searchSpace, T value0, T value1, T value2, int length) where T : struct, IEquatable<T>
+            => IndexOfAnyValueType<T, DontNegate<T>>(ref searchSpace, value0, value1, value2, length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static int IndexOfAnyExceptValueType<T>(ref T searchSpace, T value0, T value1, T value2, int length) where T : struct, IEquatable<T>
+            => IndexOfAnyValueType<T, Negate<T>>(ref searchSpace, value0, value1, value2, length);
+
+        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
+        private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T value1, T value2, int length)
+            where T : struct, IEquatable<T>
+            where N : struct, INegator<T>
+        {
+            Debug.Assert(length >= 0, "Expected non-negative length");
+            Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types");
+
+            N negator = default;
+
+            if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
+            {
+                for (int i = 0; i < length; i++)
+                {
+                    T current = Unsafe.Add(ref searchSpace, i);
+                    if (negator.NegateIfNeeded(current.Equals(value0) | current.Equals(value1) | current.Equals(value2)))
+                    {
+                        return i;
+                    }
+                }
+            }
+            else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
+            {
+                Vector256<T> equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1), values2 = Vector256.Create(value2);
+                ref T currentSearchSpace = ref searchSpace;
+                ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256<T>.Count);
+
+                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
+                do
+                {
+                    current = Vector256.LoadUnsafe(ref currentSearchSpace);
+                    equals = negator.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current) | Vector256.Equals(values2, current));
+                    if (equals == Vector256<T>.Zero)
+                    {
+                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector256<T>.Count);
+                        continue;
+                    }
+
+                    return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals);
+                }
+                while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd));
+
+                // If any elements remain, process the last vector in the search space.
+                if ((uint)length % Vector256<T>.Count != 0)
+                {
+                    current = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd);
+                    equals = negator.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current) | Vector256.Equals(values2, current));
+                    if (equals != Vector256<T>.Zero)
+                    {
+                        return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
+                    }
+                }
+            }
+            else
+            {
+                Vector128<T> equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1), values2 = Vector128.Create(value2);
+                ref T currentSearchSpace = ref searchSpace;
+                ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128<T>.Count);
+
+                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
+                do
+                {
+                    current = Vector128.LoadUnsafe(ref currentSearchSpace);
+                    equals = negator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current) | Vector128.Equals(values2, current));
+                    if (equals == Vector128<T>.Zero)
+                    {
+                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128<T>.Count);
+                        continue;
+                    }
+
+                    return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals);
+                }
+                while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd));
+
+                // If any elements remain, process the first vector in the search space.
+                if ((uint)length % Vector128<T>.Count != 0)
+                {
+                    current = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd);
+                    equals = negator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current) | Vector128.Equals(values2, current));
+                    if (equals != Vector128<T>.Zero)
+                    {
+                        return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
+                    }
+                }
+            }
+
+            return -1;
+        }
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal static int LastIndexOfValueType<T>(ref T searchSpace, T value, int length) where T : struct, IEquatable<T>
             => LastIndexOfValueType<T, DontNegate<T>>(ref searchSpace, value, length);

From 5a39d213d677a241814acc172f482112e40bd119 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Fri, 12 Aug 2022 17:36:48 +0200
Subject: [PATCH 16/34] vectorize IndexOfAny<T>(value0, value1, value2, value3)
 and IndexOfAnyExcept<T>(value0, value1, value2, value3)

---
 .../src/System/MemoryExtensions.cs            |  55 ++++++++--
 .../src/System/SpanHelpers.T.cs               | 101 ++++++++++++++++++
 2 files changed, 150 insertions(+), 6 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
index d0dfae8c684d39..f8a0fd1989ef28 100644
--- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
@@ -514,6 +514,46 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
             return -1;
         }
 
+        private static int IndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value0, T value1, T value2, T value3) where T : IEquatable<T>?
+        {
+            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            {
+                if (Unsafe.SizeOf<T>() == sizeof(byte))
+                {
+                    return SpanHelpers.IndexOfAnyExceptValueType(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, byte>(ref value0),
+                        Unsafe.As<T, byte>(ref value1),
+                        Unsafe.As<T, byte>(ref value2),
+                        Unsafe.As<T, byte>(ref value3),
+                        span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
+                {
+                    return SpanHelpers.IndexOfAnyExceptValueType(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value0),
+                        Unsafe.As<T, short>(ref value1),
+                        Unsafe.As<T, short>(ref value2),
+                        Unsafe.As<T, short>(ref value3),
+                        span.Length);
+                }
+            }
+
+            for (int i = 0; i < span.Length; i++)
+            {
+                if (!EqualityComparer<T>.Default.Equals(span[i], value0) &&
+                    !EqualityComparer<T>.Default.Equals(span[i], value1) &&
+                    !EqualityComparer<T>.Default.Equals(span[i], value2) &&
+                    !EqualityComparer<T>.Default.Equals(span[i], value3))
+                {
+                    return i;
+                }
+            }
+
+            return -1;
+        }
+
         /// <summary>Searches for the first index of any value other than the specified <paramref name="values"/>.</summary>
         /// <typeparam name="T">The type of the span and values.</typeparam>
         /// <param name="span">The span to search.</param>
@@ -541,6 +581,9 @@ public static int IndexOfAnyExcept<T>(this ReadOnlySpan<T> span, ReadOnlySpan<T>
                 case 3:
                     return IndexOfAnyExcept(span, values[0], values[1], values[2]);
 
+                case 4: // used to search for non whitespaces " \t\r\n"
+                    return IndexOfAnyExcept(span, values[0], values[1], values[2], values[3]);
+
                 default:
                     for (int i = 0; i < span.Length; i++)
                     {
@@ -1164,12 +1207,12 @@ ref Unsafe.As<char, short>(ref spanRef),
                                 span.Length);
 
                         case 4:
-                            return SpanHelpers.IndexOfAny(
-                                ref spanRef,
-                                valueRef,
-                                Unsafe.Add(ref valueRef, 1),
-                                Unsafe.Add(ref valueRef, 2),
-                                Unsafe.Add(ref valueRef, 3),
+                            return SpanHelpers.IndexOfAnyValueType(
+                                ref Unsafe.As<char, short>(ref spanRef),
+                                Unsafe.As<char, short>(ref valueRef),
+                                Unsafe.As<char, short>(ref Unsafe.Add(ref valueRef, 1)),
+                                Unsafe.As<char, short>(ref Unsafe.Add(ref valueRef, 2)),
+                                Unsafe.As<char, short>(ref Unsafe.Add(ref valueRef, 3)),
                                 span.Length);
 
                         case 5:
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
index ca17be4b4fcee9..a6d3be35c0bfd8 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
@@ -1552,6 +1552,107 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
             return -1;
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static int IndexOfAnyValueType<T>(ref T searchSpace, T value0, T value1, T value2, T value3, int length) where T : struct, IEquatable<T>
+            => IndexOfAnyValueType<T, DontNegate<T>>(ref searchSpace, value0, value1, value2, value3, length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static int IndexOfAnyExceptValueType<T>(ref T searchSpace, T value0, T value1, T value2, T value3, int length) where T : struct, IEquatable<T>
+            => IndexOfAnyValueType<T, Negate<T>>(ref searchSpace, value0, value1, value2, value3, length);
+
+        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
+        private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T value1, T value2, T value3, int length)
+            where T : struct, IEquatable<T>
+            where N : struct, INegator<T>
+        {
+            Debug.Assert(length >= 0, "Expected non-negative length");
+            Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types");
+
+            N negator = default;
+
+            if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
+            {
+                for (int i = 0; i < length; i++)
+                {
+                    T current = Unsafe.Add(ref searchSpace, i);
+                    if (negator.NegateIfNeeded(current.Equals(value0) | current.Equals(value1) | current.Equals(value2) | current.Equals(value3)))
+                    {
+                        return i;
+                    }
+                }
+            }
+            else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
+            {
+                Vector256<T> equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1), values2 = Vector256.Create(value2), values3 = Vector256.Create(value3);
+                ref T currentSearchSpace = ref searchSpace;
+                ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256<T>.Count);
+
+                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
+                do
+                {
+                    current = Vector256.LoadUnsafe(ref currentSearchSpace);
+                    equals = negator.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current)
+                        | Vector256.Equals(values2, current) | Vector256.Equals(values3, current));
+                    if (equals == Vector256<T>.Zero)
+                    {
+                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector256<T>.Count);
+                        continue;
+                    }
+
+                    return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals);
+                }
+                while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd));
+
+                // If any elements remain, process the last vector in the search space.
+                if ((uint)length % Vector256<T>.Count != 0)
+                {
+                    current = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd);
+                    equals = negator.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current)
+                        | Vector256.Equals(values2, current) | Vector256.Equals(values3, current));
+                    if (equals != Vector256<T>.Zero)
+                    {
+                        return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
+                    }
+                }
+            }
+            else
+            {
+                Vector128<T> equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1), values2 = Vector128.Create(value2), values3 = Vector128.Create(value3);
+                ref T currentSearchSpace = ref searchSpace;
+                ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128<T>.Count);
+
+                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
+                do
+                {
+                    current = Vector128.LoadUnsafe(ref currentSearchSpace);
+                    equals = negator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current)
+                        | Vector128.Equals(values2, current) | Vector128.Equals(values3, current));
+                    if (equals == Vector128<T>.Zero)
+                    {
+                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128<T>.Count);
+                        continue;
+                    }
+
+                    return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals);
+                }
+                while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd));
+
+                // If any elements remain, process the first vector in the search space.
+                if ((uint)length % Vector128<T>.Count != 0)
+                {
+                    current = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd);
+                    equals = negator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current)
+                        | Vector128.Equals(values2, current) | Vector128.Equals(values3, current));
+                    if (equals != Vector128<T>.Zero)
+                    {
+                        return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
+                    }
+                }
+            }
+
+            return -1;
+        }
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal static int LastIndexOfValueType<T>(ref T searchSpace, T value, int length) where T : struct, IEquatable<T>
             => LastIndexOfValueType<T, DontNegate<T>>(ref searchSpace, value, length);

From 9099ef91defbce718a513263afca9b8bd569d7bf Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Fri, 12 Aug 2022 17:44:30 +0200
Subject: [PATCH 17/34] vectorize IndexOfAny<T>(value0, value1, value2, value3,
 value4)

---
 .../src/System/MemoryExtensions.cs            |  36 +-
 .../src/System/SpanHelpers.Char.cs            | 511 ------------------
 .../src/System/SpanHelpers.T.cs               |  92 ++++
 3 files changed, 112 insertions(+), 527 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
index f8a0fd1989ef28..4fbd7cf82eba53 100644
--- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
@@ -1183,40 +1183,44 @@ ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                     }
                 }
 
-                if (Unsafe.SizeOf<T>() == sizeof(char))
+                if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    ref char spanRef = ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span));
-                    ref char valueRef = ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(values));
+                    ref short spanRef = ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span));
+                    ref short valueRef = ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(values));
                     switch (values.Length)
                     {
                         case 0:
                             return -1;
 
                         case 1:
-                            return SpanHelpers.IndexOfChar(ref spanRef, valueRef, span.Length);
+                            return SpanHelpers.IndexOfValueType(ref spanRef, valueRef, span.Length);
 
                         case 2:
-                            return SpanHelpers.IndexOfAnyChar(ref spanRef, valueRef, Unsafe.Add(ref valueRef, 1), span.Length);
+                            return SpanHelpers.IndexOfAnyValueType(
+                                ref spanRef,
+                                valueRef,
+                                Unsafe.Add(ref valueRef, 1),
+                                span.Length);
 
                         case 3:
                             return SpanHelpers.IndexOfAnyValueType(
-                                ref Unsafe.As<char, short>(ref spanRef),
-                                Unsafe.As<char, short>(ref valueRef),
-                                Unsafe.As<char, short>(ref Unsafe.Add(ref valueRef, 1)),
-                                Unsafe.As<char, short>(ref Unsafe.Add(ref valueRef, 2)),
+                                ref spanRef,
+                                valueRef,
+                                Unsafe.Add(ref valueRef, 1),
+                                Unsafe.Add(ref valueRef, 2),
                                 span.Length);
 
                         case 4:
                             return SpanHelpers.IndexOfAnyValueType(
-                                ref Unsafe.As<char, short>(ref spanRef),
-                                Unsafe.As<char, short>(ref valueRef),
-                                Unsafe.As<char, short>(ref Unsafe.Add(ref valueRef, 1)),
-                                Unsafe.As<char, short>(ref Unsafe.Add(ref valueRef, 2)),
-                                Unsafe.As<char, short>(ref Unsafe.Add(ref valueRef, 3)),
+                                ref spanRef,
+                                valueRef,
+                                Unsafe.Add(ref valueRef, 1),
+                                Unsafe.Add(ref valueRef, 2),
+                                Unsafe.Add(ref valueRef, 3),
                                 span.Length);
 
                         case 5:
-                            return SpanHelpers.IndexOfAny(
+                            return SpanHelpers.IndexOfAnyValueType(
                                 ref spanRef,
                                 valueRef,
                                 Unsafe.Add(ref valueRef, 1),
@@ -1226,7 +1230,7 @@ ref Unsafe.As<char, short>(ref spanRef),
                                 span.Length);
 
                         default:
-                            return IndexOfAnyProbabilistic(ref spanRef, span.Length, ref valueRef, values.Length);
+                            return IndexOfAnyProbabilistic(ref Unsafe.As<short, char>(ref spanRef), span.Length, ref Unsafe.As<short, char>(ref valueRef), values.Length);
                     }
                 }
             }
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
index cc97ba6c4b2c5f..8434420e4a73d6 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
@@ -624,517 +624,6 @@ public static unsafe int IndexOfNullCharacter(ref char searchSpace)
             return (int)(offset);
         }
 
-        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        public static unsafe int IndexOfAny(ref char searchStart, char value0, char value1, char value2, char value3, int length)
-        {
-            Debug.Assert(length >= 0);
-
-            nuint offset = 0; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations
-            nuint lengthToExamine = (nuint)(uint)length;
-
-            if (Sse2.IsSupported)
-            {
-                // Calculate lengthToExamine here for test, rather than just testing as it used later, rather than doing it twice.
-                nint vectorDiff = (nint)length - Vector128<ushort>.Count;
-                if (vectorDiff >= 0)
-                {
-                    // >= Sse2 intrinsics are supported and length is enough to use them, so use that path.
-                    // We jump forward to the intrinsics at the end of them method so a naive branch predict
-                    // will choose the non-intrinsic path so short lengths which don't gain anything aren't
-                    // overly disadvantaged by having to jump over a lot of code. Whereas the longer lengths
-                    // more than make this back from the intrinsics.
-                    lengthToExamine = (nuint)vectorDiff;
-                    goto IntrinsicsCompare;
-                }
-            }
-            else if (Vector.IsHardwareAccelerated)
-            {
-                // Calculate lengthToExamine here for test, rather than just testing as it used later, rather than doing it twice.
-                nint vectorDiff = (nint)length - Vector<ushort>.Count;
-                if (vectorDiff >= 0)
-                {
-                    // Similar as above for Vector version
-                    lengthToExamine = (nuint)vectorDiff;
-                    goto VectorCompare;
-                }
-            }
-
-            int lookUp;
-            while (lengthToExamine >= 4)
-            {
-                ref char current = ref Add(ref searchStart, offset);
-
-                lookUp = current;
-                if (value0 == lookUp || value1 == lookUp || value2 == lookUp || value3 == lookUp)
-                    goto Found;
-                lookUp = Unsafe.Add(ref current, 1);
-                if (value0 == lookUp || value1 == lookUp || value2 == lookUp || value3 == lookUp)
-                    goto Found1;
-                lookUp = Unsafe.Add(ref current, 2);
-                if (value0 == lookUp || value1 == lookUp || value2 == lookUp || value3 == lookUp)
-                    goto Found2;
-                lookUp = Unsafe.Add(ref current, 3);
-                if (value0 == lookUp || value1 == lookUp || value2 == lookUp || value3 == lookUp)
-                    goto Found3;
-
-                offset += 4;
-                lengthToExamine -= 4;
-            }
-
-            while (lengthToExamine > 0)
-            {
-                lookUp = Add(ref searchStart, offset);
-                if (value0 == lookUp || value1 == lookUp || value2 == lookUp || value3 == lookUp)
-                    goto Found;
-
-                offset += 1;
-                lengthToExamine -= 1;
-            }
-
-        NotFound:
-            return -1;
-        Found3:
-            return (int)(offset + 3);
-        Found2:
-            return (int)(offset + 2);
-        Found1:
-            return (int)(offset + 1);
-        Found:
-            return (int)offset;
-
-        IntrinsicsCompare:
-            // When we move into a Vectorized block, we process everything of Vector size;
-            // and then for any remainder we do a final compare of Vector size but starting at
-            // the end and forwards, which may overlap on an earlier compare.
-
-            // We include the Supported check again here even though path will not be taken, so the asm isn't generated if not supported.
-            if (Sse2.IsSupported)
-            {
-                int matches;
-                if (Avx2.IsSupported)
-                {
-                    Vector256<ushort> search;
-                    // Guard as we may only have a valid size for Vector128; when we will move to the Sse2
-                    // We have already subtracted Vector128<ushort>.Count from lengthToExamine so compare against that
-                    // to see if we have double the size for Vector256<ushort>.Count
-                    if (lengthToExamine >= (nuint)Vector128<ushort>.Count)
-                    {
-                        Vector256<ushort> values0 = Vector256.Create((ushort)value0);
-                        Vector256<ushort> values1 = Vector256.Create((ushort)value1);
-                        Vector256<ushort> values2 = Vector256.Create((ushort)value2);
-                        Vector256<ushort> values3 = Vector256.Create((ushort)value3);
-
-                        // Subtract Vector128<ushort>.Count so we have now subtracted Vector256<ushort>.Count
-                        lengthToExamine -= (nuint)Vector128<ushort>.Count;
-                        // First time this checks again against 0, however we will move into final compare if it fails.
-                        while (lengthToExamine > offset)
-                        {
-                            search = LoadVector256(ref searchStart, offset);
-                            // We preform the Or at non-Vector level as we are using the maximum number of non-preserved registers,
-                            // and more causes them first to be pushed to stack and then popped on exit to preseve their values.
-                            matches = Avx2.MoveMask(Avx2.CompareEqual(values0, search).AsByte());
-                            // Bitwise Or to combine the flagged matches for the second, third and fourth values to our match flags
-                            matches |= Avx2.MoveMask(Avx2.CompareEqual(values1, search).AsByte());
-                            matches |= Avx2.MoveMask(Avx2.CompareEqual(values2, search).AsByte());
-                            matches |= Avx2.MoveMask(Avx2.CompareEqual(values3, search).AsByte());
-                            // Note that MoveMask has converted the equal vector elements into a set of bit flags,
-                            // So the bit position in 'matches' corresponds to the element offset.
-                            if (matches == 0)
-                            {
-                                // None matched
-                                offset += (nuint)Vector256<ushort>.Count;
-                                continue;
-                            }
-
-                            goto IntrinsicsMatch;
-                        }
-
-                        // Move to Vector length from end for final compare
-                        search = LoadVector256(ref searchStart, lengthToExamine);
-                        offset = lengthToExamine;
-                        // Same as method as above
-                        matches = Avx2.MoveMask(Avx2.CompareEqual(values0, search).AsByte());
-                        // Bitwise Or to combine the flagged matches for the second, third and fourth values to our match flags
-                        matches |= Avx2.MoveMask(Avx2.CompareEqual(values1, search).AsByte());
-                        matches |= Avx2.MoveMask(Avx2.CompareEqual(values2, search).AsByte());
-                        matches |= Avx2.MoveMask(Avx2.CompareEqual(values3, search).AsByte());
-                        if (matches == 0)
-                        {
-                            // None matched
-                            goto NotFound;
-                        }
-
-                        goto IntrinsicsMatch;
-                    }
-                }
-
-                // Initial size check was done on method entry.
-                Debug.Assert(length >= Vector128<ushort>.Count);
-                {
-                    Vector128<ushort> search;
-                    Vector128<ushort> values0 = Vector128.Create((ushort)value0);
-                    Vector128<ushort> values1 = Vector128.Create((ushort)value1);
-                    Vector128<ushort> values2 = Vector128.Create((ushort)value2);
-                    Vector128<ushort> values3 = Vector128.Create((ushort)value3);
-                    // First time this checks against 0 and we will move into final compare if it fails.
-                    while (lengthToExamine > offset)
-                    {
-                        search = LoadVector128(ref searchStart, offset);
-
-                        matches = Sse2.MoveMask(Sse2.CompareEqual(values0, search).AsByte());
-                        matches |= Sse2.MoveMask(Sse2.CompareEqual(values1, search).AsByte());
-                        matches |= Sse2.MoveMask(Sse2.CompareEqual(values2, search).AsByte());
-                        matches |= Sse2.MoveMask(Sse2.CompareEqual(values3, search).AsByte());
-                        // Note that MoveMask has converted the equal vector elements into a set of bit flags,
-                        // So the bit position in 'matches' corresponds to the element offset.
-                        if (matches == 0)
-                        {
-                            // None matched
-                            offset += (nuint)Vector128<ushort>.Count;
-                            continue;
-                        }
-
-                        goto IntrinsicsMatch;
-                    }
-                    // Move to Vector length from end for final compare
-                    search = LoadVector128(ref searchStart, lengthToExamine);
-                    offset = lengthToExamine;
-                    // Same as method as above
-                    matches = Sse2.MoveMask(Sse2.CompareEqual(values0, search).AsByte());
-                    matches |= Sse2.MoveMask(Sse2.CompareEqual(values1, search).AsByte());
-                    matches |= Sse2.MoveMask(Sse2.CompareEqual(values2, search).AsByte());
-                    matches |= Sse2.MoveMask(Sse2.CompareEqual(values3, search).AsByte());
-                    if (matches == 0)
-                    {
-                        // None matched
-                        goto NotFound;
-                    }
-                }
-
-            IntrinsicsMatch:
-                // Find bitflag offset of first difference and add to current offset,
-                // flags are in bytes so divide by 2 for chars (shift right by 1)
-                offset += (nuint)(uint)BitOperations.TrailingZeroCount(matches) >> 1;
-                goto Found;
-            }
-
-        VectorCompare:
-            // We include the Supported check again here even though path will not be taken, so the asm isn't generated if not supported.
-            if (!Sse2.IsSupported && Vector.IsHardwareAccelerated)
-            {
-                Vector<ushort> values0 = new Vector<ushort>(value0);
-                Vector<ushort> values1 = new Vector<ushort>(value1);
-                Vector<ushort> values2 = new Vector<ushort>(value2);
-                Vector<ushort> values3 = new Vector<ushort>(value3);
-
-                Vector<ushort> search;
-                // First time this checks against 0 and we will move into final compare if it fails.
-                while (lengthToExamine > offset)
-                {
-                    search = LoadVector(ref searchStart, offset);
-                    search = Vector.BitwiseOr(
-                                    Vector.BitwiseOr(
-                                        Vector.BitwiseOr(
-                                            Vector.Equals(search, values0),
-                                            Vector.Equals(search, values1)),
-                                        Vector.Equals(search, values2)),
-                                    Vector.Equals(search, values3));
-                    if (Vector<ushort>.Zero.Equals(search))
-                    {
-                        // None matched
-                        offset += (nuint)Vector<ushort>.Count;
-                        continue;
-                    }
-
-                    goto VectorMatch;
-                }
-
-                // Move to Vector length from end for final compare
-                search = LoadVector(ref searchStart, lengthToExamine);
-                offset = lengthToExamine;
-                search = Vector.BitwiseOr(
-                                Vector.BitwiseOr(
-                                    Vector.BitwiseOr(
-                                        Vector.Equals(search, values0),
-                                        Vector.Equals(search, values1)),
-                                    Vector.Equals(search, values2)),
-                                Vector.Equals(search, values3));
-                if (Vector<ushort>.Zero.Equals(search))
-                {
-                    // None matched
-                    goto NotFound;
-                }
-
-            VectorMatch:
-                offset += (nuint)(uint)LocateFirstFoundChar(search);
-                goto Found;
-            }
-
-            Debug.Fail("Unreachable");
-            goto NotFound;
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        public static unsafe int IndexOfAny(ref char searchStart, char value0, char value1, char value2, char value3, char value4, int length)
-        {
-            Debug.Assert(length >= 0);
-
-            nuint offset = 0; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations
-            nuint lengthToExamine = (nuint)(uint)length;
-
-            if (Sse2.IsSupported)
-            {
-                // Calculate lengthToExamine here for test, rather than just testing as it used later, rather than doing it twice.
-                nint vectorDiff = (nint)length - Vector128<ushort>.Count;
-                if (vectorDiff >= 0)
-                {
-                    // >= Sse2 intrinsics are supported and length is enough to use them, so use that path.
-                    // We jump forward to the intrinsics at the end of them method so a naive branch predict
-                    // will choose the non-intrinsic path so short lengths which don't gain anything aren't
-                    // overly disadvantaged by having to jump over a lot of code. Whereas the longer lengths
-                    // more than make this back from the intrinsics.
-                    lengthToExamine = (nuint)vectorDiff;
-                    goto IntrinsicsCompare;
-                }
-            }
-            else if (Vector.IsHardwareAccelerated)
-            {
-                // Calculate lengthToExamine here for test, rather than just testing as it used later, rather than doing it twice.
-                nint vectorDiff = (nint)length - Vector<ushort>.Count;
-                if (vectorDiff >= 0)
-                {
-                    // Similar as above for Vector version
-                    lengthToExamine = (nuint)vectorDiff;
-                    goto VectorCompare;
-                }
-            }
-
-            int lookUp;
-            while (lengthToExamine >= 4)
-            {
-                ref char current = ref Add(ref searchStart, offset);
-
-                lookUp = current;
-                if (value0 == lookUp || value1 == lookUp || value2 == lookUp || value3 == lookUp || value4 == lookUp)
-                    goto Found;
-                lookUp = Unsafe.Add(ref current, 1);
-                if (value0 == lookUp || value1 == lookUp || value2 == lookUp || value3 == lookUp || value4 == lookUp)
-                    goto Found1;
-                lookUp = Unsafe.Add(ref current, 2);
-                if (value0 == lookUp || value1 == lookUp || value2 == lookUp || value3 == lookUp || value4 == lookUp)
-                    goto Found2;
-                lookUp = Unsafe.Add(ref current, 3);
-                if (value0 == lookUp || value1 == lookUp || value2 == lookUp || value3 == lookUp || value4 == lookUp)
-                    goto Found3;
-
-                offset += 4;
-                lengthToExamine -= 4;
-            }
-
-            while (lengthToExamine > 0)
-            {
-                lookUp = Add(ref searchStart, offset);
-                if (value0 == lookUp || value1 == lookUp || value2 == lookUp || value3 == lookUp || value4 == lookUp)
-                    goto Found;
-
-                offset += 1;
-                lengthToExamine -= 1;
-            }
-
-        NotFound:
-            return -1;
-        Found3:
-            return (int)(offset + 3);
-        Found2:
-            return (int)(offset + 2);
-        Found1:
-            return (int)(offset + 1);
-        Found:
-            return (int)offset;
-
-        IntrinsicsCompare:
-            // When we move into a Vectorized block, we process everything of Vector size;
-            // and then for any remainder we do a final compare of Vector size but starting at
-            // the end and forwards, which may overlap on an earlier compare.
-
-            // We include the Supported check again here even though path will not be taken, so the asm isn't generated if not supported.
-            if (Sse2.IsSupported)
-            {
-                int matches;
-                if (Avx2.IsSupported)
-                {
-                    Vector256<ushort> search;
-                    // Guard as we may only have a valid size for Vector128; when we will move to the Sse2
-                    // We have already subtracted Vector128<ushort>.Count from lengthToExamine so compare against that
-                    // to see if we have double the size for Vector256<ushort>.Count
-                    if (lengthToExamine >= (nuint)Vector128<ushort>.Count)
-                    {
-                        Vector256<ushort> values0 = Vector256.Create((ushort)value0);
-                        Vector256<ushort> values1 = Vector256.Create((ushort)value1);
-                        Vector256<ushort> values2 = Vector256.Create((ushort)value2);
-                        Vector256<ushort> values3 = Vector256.Create((ushort)value3);
-                        Vector256<ushort> values4 = Vector256.Create((ushort)value4);
-
-                        // Subtract Vector128<ushort>.Count so we have now subtracted Vector256<ushort>.Count
-                        lengthToExamine -= (nuint)Vector128<ushort>.Count;
-                        // First time this checks again against 0, however we will move into final compare if it fails.
-                        while (lengthToExamine > offset)
-                        {
-                            search = LoadVector256(ref searchStart, offset);
-                            // We preform the Or at non-Vector level as we are using the maximum number of non-preserved registers (+ 1),
-                            // and more causes them first to be pushed to stack and then popped on exit to preseve their values.
-                            matches = Avx2.MoveMask(Avx2.CompareEqual(values0, search).AsByte());
-                            // Bitwise Or to combine the flagged matches for the second, third and fourth values to our match flags
-                            matches |= Avx2.MoveMask(Avx2.CompareEqual(values1, search).AsByte());
-                            matches |= Avx2.MoveMask(Avx2.CompareEqual(values2, search).AsByte());
-                            matches |= Avx2.MoveMask(Avx2.CompareEqual(values3, search).AsByte());
-                            matches |= Avx2.MoveMask(Avx2.CompareEqual(values4, search).AsByte());
-                            // Note that MoveMask has converted the equal vector elements into a set of bit flags,
-                            // So the bit position in 'matches' corresponds to the element offset.
-                            if (matches == 0)
-                            {
-                                // None matched
-                                offset += (nuint)Vector256<ushort>.Count;
-                                continue;
-                            }
-
-                            goto IntrinsicsMatch;
-                        }
-
-                        // Move to Vector length from end for final compare
-                        search = LoadVector256(ref searchStart, lengthToExamine);
-                        offset = lengthToExamine;
-                        // Same as method as above
-                        matches = Avx2.MoveMask(Avx2.CompareEqual(values0, search).AsByte());
-                        // Bitwise Or to combine the flagged matches for the second, third and fourth values to our match flags
-                        matches |= Avx2.MoveMask(Avx2.CompareEqual(values1, search).AsByte());
-                        matches |= Avx2.MoveMask(Avx2.CompareEqual(values2, search).AsByte());
-                        matches |= Avx2.MoveMask(Avx2.CompareEqual(values3, search).AsByte());
-                        matches |= Avx2.MoveMask(Avx2.CompareEqual(values4, search).AsByte());
-                        if (matches == 0)
-                        {
-                            // None matched
-                            goto NotFound;
-                        }
-
-                        goto IntrinsicsMatch;
-                    }
-                }
-
-                // Initial size check was done on method entry.
-                Debug.Assert(length >= Vector128<ushort>.Count);
-                {
-                    Vector128<ushort> search;
-                    Vector128<ushort> values0 = Vector128.Create((ushort)value0);
-                    Vector128<ushort> values1 = Vector128.Create((ushort)value1);
-                    Vector128<ushort> values2 = Vector128.Create((ushort)value2);
-                    Vector128<ushort> values3 = Vector128.Create((ushort)value3);
-                    Vector128<ushort> values4 = Vector128.Create((ushort)value4);
-                    // First time this checks against 0 and we will move into final compare if it fails.
-                    while (lengthToExamine > offset)
-                    {
-                        search = LoadVector128(ref searchStart, offset);
-
-                        matches = Sse2.MoveMask(Sse2.CompareEqual(values0, search).AsByte());
-                        matches |= Sse2.MoveMask(Sse2.CompareEqual(values1, search).AsByte());
-                        matches |= Sse2.MoveMask(Sse2.CompareEqual(values2, search).AsByte());
-                        matches |= Sse2.MoveMask(Sse2.CompareEqual(values3, search).AsByte());
-                        matches |= Sse2.MoveMask(Sse2.CompareEqual(values4, search).AsByte());
-                        // Note that MoveMask has converted the equal vector elements into a set of bit flags,
-                        // So the bit position in 'matches' corresponds to the element offset.
-                        if (matches == 0)
-                        {
-                            // None matched
-                            offset += (nuint)Vector128<ushort>.Count;
-                            continue;
-                        }
-
-                        goto IntrinsicsMatch;
-                    }
-                    // Move to Vector length from end for final compare
-                    search = LoadVector128(ref searchStart, lengthToExamine);
-                    offset = lengthToExamine;
-                    // Same as method as above
-                    matches = Sse2.MoveMask(Sse2.CompareEqual(values0, search).AsByte());
-                    matches |= Sse2.MoveMask(Sse2.CompareEqual(values1, search).AsByte());
-                    matches |= Sse2.MoveMask(Sse2.CompareEqual(values2, search).AsByte());
-                    matches |= Sse2.MoveMask(Sse2.CompareEqual(values3, search).AsByte());
-                    matches |= Sse2.MoveMask(Sse2.CompareEqual(values4, search).AsByte());
-                    if (matches == 0)
-                    {
-                        // None matched
-                        goto NotFound;
-                    }
-                }
-
-            IntrinsicsMatch:
-                // Find bitflag offset of first difference and add to current offset,
-                // flags are in bytes so divide by 2 for chars (shift right by 1)
-                offset += (nuint)(uint)BitOperations.TrailingZeroCount(matches) >> 1;
-                goto Found;
-            }
-
-        VectorCompare:
-            // We include the Supported check again here even though path will not be taken, so the asm isn't generated if not supported.
-            if (!Sse2.IsSupported && Vector.IsHardwareAccelerated)
-            {
-                Vector<ushort> values0 = new Vector<ushort>(value0);
-                Vector<ushort> values1 = new Vector<ushort>(value1);
-                Vector<ushort> values2 = new Vector<ushort>(value2);
-                Vector<ushort> values3 = new Vector<ushort>(value3);
-                Vector<ushort> values4 = new Vector<ushort>(value4);
-
-                Vector<ushort> search;
-                // First time this checks against 0 and we will move into final compare if it fails.
-                while (lengthToExamine > offset)
-                {
-                    search = LoadVector(ref searchStart, offset);
-                    search = Vector.BitwiseOr(
-                                Vector.BitwiseOr(
-                                    Vector.BitwiseOr(
-                                        Vector.BitwiseOr(
-                                            Vector.Equals(search, values0),
-                                            Vector.Equals(search, values1)),
-                                        Vector.Equals(search, values2)),
-                                    Vector.Equals(search, values3)),
-                                Vector.Equals(search, values4));
-                    if (Vector<ushort>.Zero.Equals(search))
-                    {
-                        // None matched
-                        offset += (nuint)Vector<ushort>.Count;
-                        continue;
-                    }
-
-                    goto VectorMatch;
-                }
-
-                // Move to Vector length from end for final compare
-                search = LoadVector(ref searchStart, lengthToExamine);
-                offset = lengthToExamine;
-                search = Vector.BitwiseOr(
-                            Vector.BitwiseOr(
-                                Vector.BitwiseOr(
-                                    Vector.BitwiseOr(
-                                        Vector.Equals(search, values0),
-                                        Vector.Equals(search, values1)),
-                                    Vector.Equals(search, values2)),
-                                Vector.Equals(search, values3)),
-                            Vector.Equals(search, values4));
-                if (Vector<ushort>.Zero.Equals(search))
-                {
-                    // None matched
-                    goto NotFound;
-                }
-
-            VectorMatch:
-                offset += (nuint)(uint)LocateFirstFoundChar(search);
-                goto Found;
-            }
-
-            Debug.Fail("Unreachable");
-            goto NotFound;
-        }
-
         // Vector sub-search adapted from https://github.com/aspnet/KestrelHttpServer/pull/1138
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static int LocateFirstFoundChar(Vector<ushort> match)
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
index a6d3be35c0bfd8..b9c22bbbec2cc7 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
@@ -1653,6 +1653,98 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
             return -1;
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
+        internal static int IndexOfAnyValueType<T>(ref T searchSpace, T value0, T value1, T value2, T value3, T value4, int length)
+            where T : struct, IEquatable<T>
+        {
+            Debug.Assert(length >= 0, "Expected non-negative length");
+            Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types");
+
+            if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
+            {
+                for (int i = 0; i < length; i++)
+                {
+                    T current = Unsafe.Add(ref searchSpace, i);
+                    if (current.Equals(value0) | current.Equals(value1) | current.Equals(value2) | current.Equals(value3) | current.Equals(value4))
+                    {
+                        return i;
+                    }
+                }
+            }
+            else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
+            {
+                Vector256<T> equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1),
+                    values2 = Vector256.Create(value2), values3 = Vector256.Create(value3), values4 = Vector256.Create(value4);
+                ref T currentSearchSpace = ref searchSpace;
+                ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256<T>.Count);
+
+                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
+                do
+                {
+                    current = Vector256.LoadUnsafe(ref currentSearchSpace);
+                    equals = Vector256.Equals(values0, current) | Vector256.Equals(values1, current) | Vector256.Equals(values2, current)
+                           | Vector256.Equals(values3, current) | Vector256.Equals(values4, current);
+                    if (equals == Vector256<T>.Zero)
+                    {
+                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector256<T>.Count);
+                        continue;
+                    }
+
+                    return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals);
+                }
+                while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd));
+
+                // If any elements remain, process the last vector in the search space.
+                if ((uint)length % Vector256<T>.Count != 0)
+                {
+                    current = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd);
+                    equals = Vector256.Equals(values0, current) | Vector256.Equals(values1, current) | Vector256.Equals(values2, current)
+                           | Vector256.Equals(values3, current) | Vector256.Equals(values4, current);
+                    if (equals != Vector256<T>.Zero)
+                    {
+                        return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
+                    }
+                }
+            }
+            else
+            {
+                Vector128<T> equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1),
+                    values2 = Vector128.Create(value2), values3 = Vector128.Create(value3), values4 = Vector128.Create(value4);
+                ref T currentSearchSpace = ref searchSpace;
+                ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128<T>.Count);
+
+                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
+                do
+                {
+                    current = Vector128.LoadUnsafe(ref currentSearchSpace);
+                    equals = Vector128.Equals(values0, current) | Vector128.Equals(values1, current) | Vector128.Equals(values2, current)
+                           | Vector128.Equals(values3, current) | Vector128.Equals(values4, current);
+                    if (equals == Vector128<T>.Zero)
+                    {
+                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128<T>.Count);
+                        continue;
+                    }
+
+                    return ComputeFirstIndex(ref searchSpace, ref currentSearchSpace, equals);
+                }
+                while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd));
+
+                // If any elements remain, process the first vector in the search space.
+                if ((uint)length % Vector128<T>.Count != 0)
+                {
+                    current = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd);
+                    equals = Vector128.Equals(values0, current) | Vector128.Equals(values1, current) | Vector128.Equals(values2, current)
+                           | Vector128.Equals(values3, current) | Vector128.Equals(values4, current);
+                    if (equals != Vector128<T>.Zero)
+                    {
+                        return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
+                    }
+                }
+            }
+
+            return -1;
+        }
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         internal static int LastIndexOfValueType<T>(ref T searchSpace, T value, int length) where T : struct, IEquatable<T>
             => LastIndexOfValueType<T, DontNegate<T>>(ref searchSpace, value, length);

From 56617937a166a34c84c066c83dafe11c045c6d7f Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Fri, 12 Aug 2022 17:57:07 +0200
Subject: [PATCH 18/34] remove dead code

---
 .../src/System/SpanHelpers.Byte.cs            | 217 ------------------
 .../src/System/SpanHelpers.Char.cs            |  86 -------
 .../src/System/SpanHelpers.cs                 |   2 -
 3 files changed, 305 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
index dc13e0e7bf06b0..ad722fa3408bbb 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
@@ -5,7 +5,6 @@
 using System.Numerics;
 using System.Runtime.CompilerServices;
 using System.Runtime.Intrinsics;
-using System.Runtime.Intrinsics.Arm;
 using System.Runtime.Intrinsics.X86;
 
 namespace System
@@ -528,136 +527,6 @@ public static unsafe int IndexOfNullByte(ref byte searchSpace)
             return (int)(offset + 7);
         }
 
-        public static int LastIndexOfAny(ref byte searchSpace, byte value0, byte value1, byte value2, int length)
-        {
-            Debug.Assert(length >= 0);
-
-            uint uValue0 = value0; // Use uint for comparisons to avoid unnecessary 8->32 extensions
-            uint uValue1 = value1;
-            uint uValue2 = value2;
-            nuint offset = (nuint)(uint)length; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations
-            nuint lengthToExamine = (nuint)(uint)length;
-
-            if (Vector.IsHardwareAccelerated && length >= Vector<byte>.Count * 2)
-            {
-                lengthToExamine = UnalignedCountVectorFromEnd(ref searchSpace, length);
-            }
-        SequentialScan:
-            uint lookUp;
-            while (lengthToExamine >= 8)
-            {
-                lengthToExamine -= 8;
-                offset -= 8;
-
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 7);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found7;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 6);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found6;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 5);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found5;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 4);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found4;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 3);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found3;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 2);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found2;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 1);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found1;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found;
-            }
-
-            if (lengthToExamine >= 4)
-            {
-                lengthToExamine -= 4;
-                offset -= 4;
-
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 3);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found3;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 2);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found2;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset + 1);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found1;
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found;
-            }
-
-            while (lengthToExamine > 0)
-            {
-                lengthToExamine -= 1;
-                offset -= 1;
-
-                lookUp = Unsafe.AddByteOffset(ref searchSpace, offset);
-                if (uValue0 == lookUp || uValue1 == lookUp || uValue2 == lookUp)
-                    goto Found;
-            }
-
-            if (Vector.IsHardwareAccelerated && (offset > 0))
-            {
-                lengthToExamine = (offset & (nuint)~(Vector<byte>.Count - 1));
-
-                Vector<byte> values0 = new Vector<byte>(value0);
-                Vector<byte> values1 = new Vector<byte>(value1);
-                Vector<byte> values2 = new Vector<byte>(value2);
-
-                while (lengthToExamine > (nuint)(Vector<byte>.Count - 1))
-                {
-                    Vector<byte> search = LoadVector(ref searchSpace, offset - (nuint)Vector<byte>.Count);
-
-                    var matches = Vector.BitwiseOr(
-                                    Vector.BitwiseOr(
-                                        Vector.Equals(search, values0),
-                                        Vector.Equals(search, values1)),
-                                    Vector.Equals(search, values2));
-
-                    if (Vector<byte>.Zero.Equals(matches))
-                    {
-                        offset -= (nuint)Vector<byte>.Count;
-                        lengthToExamine -= (nuint)Vector<byte>.Count;
-                        continue;
-                    }
-
-                    // Find offset of first match and add to current offset
-                    return (int)(offset) - Vector<byte>.Count + LocateLastFoundByte(matches);
-                }
-
-                if (offset > 0)
-                {
-                    lengthToExamine = offset;
-                    goto SequentialScan;
-                }
-            }
-            return -1;
-        Found: // Workaround for https://github.com/dotnet/runtime/issues/8795
-            return (int)offset;
-        Found1:
-            return (int)(offset + 1);
-        Found2:
-            return (int)(offset + 2);
-        Found3:
-            return (int)(offset + 3);
-        Found4:
-            return (int)(offset + 4);
-        Found5:
-            return (int)(offset + 5);
-        Found6:
-            return (int)(offset + 6);
-        Found7:
-            return (int)(offset + 7);
-        }
-
         // Optimized byte-based SequenceEquals. The "length" parameter for this one is declared a nuint rather than int as we also use it for types other than byte
         // where the length can exceed 2Gb once scaled by sizeof(T).
         public static unsafe bool SequenceEqual(ref byte first, ref byte second, nuint length)
@@ -861,27 +730,6 @@ public static unsafe bool SequenceEqual(ref byte first, ref byte second, nuint l
             return false;
         }
 
-        // Vector sub-search adapted from https://github.com/aspnet/KestrelHttpServer/pull/1138
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int LocateFirstFoundByte(Vector<byte> match)
-        {
-            var vector64 = Vector.AsVectorUInt64(match);
-            ulong candidate = 0;
-            int i = 0;
-            // Pattern unrolled by jit https://github.com/dotnet/coreclr/pull/8001
-            for (; i < Vector<ulong>.Count; i++)
-            {
-                candidate = vector64[i];
-                if (candidate != 0)
-                {
-                    break;
-                }
-            }
-
-            // Single LEA instruction with jitted const (using function result)
-            return i * 8 + LocateFirstFoundByte(candidate);
-        }
-
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
         public static unsafe int SequenceCompareTo(ref byte first, int firstLength, ref byte second, int secondLength)
         {
@@ -1157,39 +1005,6 @@ public static nuint CommonPrefixLength(ref byte first, ref byte second, nuint le
             return i + uint.TrailingZeroCount(mask);
         }
 
-        // Vector sub-search adapted from https://github.com/aspnet/KestrelHttpServer/pull/1138
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int LocateLastFoundByte(Vector<byte> match)
-        {
-            var vector64 = Vector.AsVectorUInt64(match);
-            ulong candidate = 0;
-            int i = Vector<ulong>.Count - 1;
-
-            // This pattern is only unrolled by the Jit if the limit is Vector<T>.Count
-            // As such, we need a dummy iteration variable for that condition to be satisfied
-            for (int j = 0; j < Vector<ulong>.Count; j++)
-            {
-                candidate = vector64[i];
-                if (candidate != 0)
-                {
-                    break;
-                }
-
-                i--;
-            }
-
-            // Single LEA instruction with jitted const (using function result)
-            return i * 8 + LocateLastFoundByte(candidate);
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int LocateFirstFoundByte(ulong match)
-            => BitOperations.TrailingZeroCount(match) >> 3;
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int LocateLastFoundByte(ulong match)
-            => BitOperations.Log2(match) >> 3;
-
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static ushort LoadUShort(ref byte start)
             => Unsafe.ReadUnaligned<ushort>(ref start);
@@ -1222,10 +1037,6 @@ private static Vector128<byte> LoadVector128(ref byte start, nuint offset)
         private static Vector256<byte> LoadVector256(ref byte start, nuint offset)
             => Unsafe.ReadUnaligned<Vector256<byte>>(ref Unsafe.AddByteOffset(ref start, offset));
 
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static nuint GetByteVectorSpanLength(nuint offset, int length)
-            => (nuint)(uint)((length - (int)offset) & ~(Vector<byte>.Count - 1));
-
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static nuint GetByteVector128SpanLength(nuint offset, int length)
             => (nuint)(uint)((length - (int)offset) & ~(Vector128<byte>.Count - 1));
@@ -1248,34 +1059,6 @@ private static unsafe nuint UnalignedCountVector128(ref byte searchSpace)
             return (nuint)(uint)((Vector128<byte>.Count - unaligned) & (Vector128<byte>.Count - 1));
         }
 
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static unsafe nuint UnalignedCountVectorFromEnd(ref byte searchSpace, int length)
-        {
-            nint unaligned = (nint)Unsafe.AsPointer(ref searchSpace) & (Vector<byte>.Count - 1);
-            return (nuint)(uint)(((length & (Vector<byte>.Count - 1)) + unaligned) & (Vector<byte>.Count - 1));
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static uint FindFirstMatchedLane(Vector128<byte> compareResult)
-        {
-            Debug.Assert(AdvSimd.Arm64.IsSupported);
-
-            // Mask to help find the first lane in compareResult that is set.
-            // MSB 0x10 corresponds to 1st lane, 0x01 corresponds to 0th lane and so forth.
-            Vector128<byte> mask = Vector128.Create((ushort)0x1001).AsByte();
-
-            // Find the first lane that is set inside compareResult.
-            Vector128<byte> maskedSelectedLanes = AdvSimd.And(compareResult, mask);
-            Vector128<byte> pairwiseSelectedLane = AdvSimd.Arm64.AddPairwise(maskedSelectedLanes, maskedSelectedLanes);
-            ulong selectedLanes = pairwiseSelectedLane.AsUInt64().ToScalar();
-
-            // It should be handled by compareResult != Vector.Zero
-            Debug.Assert(selectedLanes != 0);
-
-            // Find the first lane that is set inside compareResult.
-            return (uint)BitOperations.TrailingZeroCount(selectedLanes) >> 2;
-        }
-
         public static void Reverse(ref byte buf, nuint length)
         {
             if (Avx2.IsSupported && (nuint)Vector256<byte>.Count * 2 <= length)
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
index 8434420e4a73d6..343efb36445e63 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
@@ -5,7 +5,6 @@
 using System.Numerics;
 using System.Runtime.CompilerServices;
 using System.Runtime.Intrinsics;
-using System.Runtime.Intrinsics.Arm;
 using System.Runtime.Intrinsics.X86;
 
 namespace System
@@ -420,7 +419,6 @@ public static unsafe int SequenceCompareTo(ref char first, int firstLength, ref
             return lengthDelta;
         }
 
-
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
         public static unsafe int IndexOfNullCharacter(ref char searchSpace)
         {
@@ -624,68 +622,6 @@ public static unsafe int IndexOfNullCharacter(ref char searchSpace)
             return (int)(offset);
         }
 
-        // Vector sub-search adapted from https://github.com/aspnet/KestrelHttpServer/pull/1138
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int LocateFirstFoundChar(Vector<ushort> match)
-        {
-            var vector64 = Vector.AsVectorUInt64(match);
-            ulong candidate = 0;
-            int i = 0;
-            // Pattern unrolled by jit https://github.com/dotnet/coreclr/pull/8001
-            for (; i < Vector<ulong>.Count; i++)
-            {
-                candidate = vector64[i];
-                if (candidate != 0)
-                {
-                    break;
-                }
-            }
-
-            // Single LEA instruction with jitted const (using function result)
-            return i * 4 + LocateFirstFoundChar(candidate);
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int LocateFirstFoundChar(ulong match)
-            => BitOperations.TrailingZeroCount(match) >> 4;
-
-        // Vector sub-search adapted from https://github.com/aspnet/KestrelHttpServer/pull/1138
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int LocateLastFoundChar(Vector<ushort> match)
-        {
-            var vector64 = Vector.AsVectorUInt64(match);
-            ulong candidate = 0;
-            int i = Vector<ulong>.Count - 1;
-
-            // This pattern is only unrolled by the Jit if the limit is Vector<T>.Count
-            // As such, we need a dummy iteration variable for that condition to be satisfied
-            for (int j = 0; j < Vector<ulong>.Count; j++)
-            {
-                candidate = vector64[i];
-                if (candidate != 0)
-                {
-                    break;
-                }
-
-                i--;
-            }
-
-            // Single LEA instruction with jitted const (using function result)
-            return i * 4 + LocateLastFoundChar(candidate);
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int LocateLastFoundChar(ulong match)
-            => BitOperations.Log2(match) >> 4;
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static Vector<ushort> LoadVector(ref char start, nint offset)
-            => Unsafe.ReadUnaligned<Vector<ushort>>(ref Unsafe.As<char, byte>(ref Unsafe.Add(ref start, offset)));
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static Vector<ushort> LoadVector(ref char start, nuint offset)
-            => Unsafe.ReadUnaligned<Vector<ushort>>(ref Unsafe.As<char, byte>(ref Unsafe.Add(ref start, (nint)offset)));
-
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static Vector128<ushort> LoadVector128(ref char start, nint offset)
             => Unsafe.ReadUnaligned<Vector128<ushort>>(ref Unsafe.As<char, byte>(ref Unsafe.Add(ref start, offset)));
@@ -702,13 +638,6 @@ private static Vector256<ushort> LoadVector256(ref char start, nint offset)
         private static Vector256<ushort> LoadVector256(ref char start, nuint offset)
             => Unsafe.ReadUnaligned<Vector256<ushort>>(ref Unsafe.As<char, byte>(ref Unsafe.Add(ref start, (nint)offset)));
 
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static ref char Add(ref char start, nuint offset) => ref Unsafe.Add(ref start, (nint)offset);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static nint GetCharVectorSpanLength(nint offset, nint length)
-            => (length - offset) & ~(Vector<ushort>.Count - 1);
-
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static nint GetCharVector128SpanLength(nint offset, nint length)
             => (length - offset) & ~(Vector128<ushort>.Count - 1);
@@ -742,21 +671,6 @@ private static unsafe nint UnalignedCountVector128(ref char searchSpace)
             return (nint)(uint)(-(int)Unsafe.AsPointer(ref searchSpace) / ElementsPerByte) & (Vector128<ushort>.Count - 1);
         }
 
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int FindFirstMatchedLane(Vector128<ushort> compareResult)
-        {
-            Debug.Assert(AdvSimd.Arm64.IsSupported);
-
-            Vector128<byte> pairwiseSelectedLane = AdvSimd.Arm64.AddPairwise(compareResult.AsByte(), compareResult.AsByte());
-            ulong selectedLanes = pairwiseSelectedLane.AsUInt64().ToScalar();
-
-            // It should be handled by compareResult != Vector.Zero
-            Debug.Assert(selectedLanes != 0);
-
-            return BitOperations.TrailingZeroCount(selectedLanes) >> 3;
-        }
-
         public static void Reverse(ref char buf, nuint length)
         {
             if (Avx2.IsSupported && (nuint)Vector256<short>.Count * 2 <= length)
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs
index d35b5e2d2034ee..c46f5ec8ae4251 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs
@@ -3,9 +3,7 @@
 
 using System.Diagnostics;
 using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
 using System.Runtime.Intrinsics;
-using System.Runtime.Intrinsics.Arm;
 using System.Runtime.Intrinsics.X86;
 
 namespace System

From 8cb8208ab76bd5d41fd093843a16302e7dad4f31 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Fri, 12 Aug 2022 18:23:55 +0200
Subject: [PATCH 19/34] use built-in helpers

---
 .../src/System/SpanHelpers.Char.cs            | 34 ++++++-------------
 1 file changed, 10 insertions(+), 24 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
index 343efb36445e63..5f98699a6b955a 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
@@ -67,6 +67,7 @@ ref Unsafe.As<char, byte>(ref Unsafe.Add(ref searchSpace, offset + 1)),
             // Based on http://0x80.pl/articles/simd-strfind.html#algorithm-1-generic-simd "Algorithm 1: Generic SIMD" by Wojciech Muła
             // Some details about the implementation can also be found in https://github.com/dotnet/runtime/pull/63285
         SEARCH_TWO_CHARS:
+            ref ushort ushortSearchSpace = ref Unsafe.As<char, ushort>(ref searchSpace);
             if (Vector256.IsHardwareAccelerated && searchSpaceMinusValueTailLength - Vector256<ushort>.Count >= 0)
             {
                 // Find the last unique (which is not equal to ch1) character
@@ -87,8 +88,8 @@ ref Unsafe.As<char, byte>(ref Unsafe.Add(ref searchSpace, offset + 1)),
                     // Make sure we don't go out of bounds
                     Debug.Assert(offset + ch1ch2Distance + Vector256<ushort>.Count <= searchSpaceLength);
 
-                    Vector256<ushort> cmpCh2 = Vector256.Equals(ch2, LoadVector256(ref searchSpace, offset + ch1ch2Distance));
-                    Vector256<ushort> cmpCh1 = Vector256.Equals(ch1, LoadVector256(ref searchSpace, offset));
+                    Vector256<ushort> cmpCh2 = Vector256.Equals(ch2, Vector256.LoadUnsafe(ref ushortSearchSpace, (nuint)(offset + ch1ch2Distance)));
+                    Vector256<ushort> cmpCh1 = Vector256.Equals(ch1, Vector256.LoadUnsafe(ref ushortSearchSpace, (nuint)offset));
                     Vector256<byte> cmpAnd = (cmpCh1 & cmpCh2).AsByte();
 
                     // Early out: cmpAnd is all zeros
@@ -154,8 +155,8 @@ ref Unsafe.As<char, byte>(ref value), (nuint)(uint)valueLength * 2))
                     // Make sure we don't go out of bounds
                     Debug.Assert(offset + ch1ch2Distance + Vector128<ushort>.Count <= searchSpaceLength);
 
-                    Vector128<ushort> cmpCh2 = Vector128.Equals(ch2, LoadVector128(ref searchSpace, offset + ch1ch2Distance));
-                    Vector128<ushort> cmpCh1 = Vector128.Equals(ch1, LoadVector128(ref searchSpace, offset));
+                    Vector128<ushort> cmpCh2 = Vector128.Equals(ch2, Vector128.LoadUnsafe(ref ushortSearchSpace, (nuint)(offset + ch1ch2Distance)));
+                    Vector128<ushort> cmpCh1 = Vector128.Equals(ch1, Vector128.LoadUnsafe(ref ushortSearchSpace, (nuint)offset));
                     Vector128<byte> cmpAnd = (cmpCh1 & cmpCh2).AsByte();
 
                     // Early out: cmpAnd is all zeros
@@ -252,6 +253,7 @@ ref Unsafe.As<char, byte>(ref Unsafe.Add(ref searchSpace, relativeIndex + 1)),
             // Based on http://0x80.pl/articles/simd-strfind.html#algorithm-1-generic-simd "Algorithm 1: Generic SIMD" by Wojciech Muła
             // Some details about the implementation can also be found in https://github.com/dotnet/runtime/pull/63285
         SEARCH_TWO_CHARS:
+            ref ushort ushortSearchSpace = ref Unsafe.As<char, ushort>(ref searchSpace);
             if (Vector256.IsHardwareAccelerated && searchSpaceMinusValueTailLength >= Vector256<ushort>.Count)
             {
                 offset = searchSpaceMinusValueTailLength - Vector256<ushort>.Count;
@@ -269,8 +271,8 @@ ref Unsafe.As<char, byte>(ref Unsafe.Add(ref searchSpace, relativeIndex + 1)),
                 do
                 {
 
-                    Vector256<ushort> cmpCh1 = Vector256.Equals(ch1, LoadVector256(ref searchSpace, (nuint)offset));
-                    Vector256<ushort> cmpCh2 = Vector256.Equals(ch2, LoadVector256(ref searchSpace, (nuint)(offset + ch1ch2Distance)));
+                    Vector256<ushort> cmpCh1 = Vector256.Equals(ch1, Vector256.LoadUnsafe(ref ushortSearchSpace, (nuint)offset));
+                    Vector256<ushort> cmpCh2 = Vector256.Equals(ch2, Vector256.LoadUnsafe(ref ushortSearchSpace, (nuint)(offset + ch1ch2Distance)));
                     Vector256<byte> cmpAnd = (cmpCh1 & cmpCh2).AsByte();
 
                     // Early out: cmpAnd is all zeros
@@ -318,8 +320,8 @@ ref Unsafe.As<char, byte>(ref value), (nuint)(uint)valueLength * 2))
 
                 do
                 {
-                    Vector128<ushort> cmpCh1 = Vector128.Equals(ch1, LoadVector128(ref searchSpace, (nuint)offset));
-                    Vector128<ushort> cmpCh2 = Vector128.Equals(ch2, LoadVector128(ref searchSpace, (nuint)(offset + ch1ch2Distance)));
+                    Vector128<ushort> cmpCh1 = Vector128.Equals(ch1, Vector128.LoadUnsafe(ref ushortSearchSpace, (nuint)offset));
+                    Vector128<ushort> cmpCh2 = Vector128.Equals(ch2, Vector128.LoadUnsafe(ref ushortSearchSpace, (nuint)(offset + ch1ch2Distance)));
                     Vector128<byte> cmpAnd = (cmpCh1 & cmpCh2).AsByte();
 
                     // Early out: cmpAnd is all zeros
@@ -622,22 +624,6 @@ public static unsafe int IndexOfNullCharacter(ref char searchSpace)
             return (int)(offset);
         }
 
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static Vector128<ushort> LoadVector128(ref char start, nint offset)
-            => Unsafe.ReadUnaligned<Vector128<ushort>>(ref Unsafe.As<char, byte>(ref Unsafe.Add(ref start, offset)));
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static Vector128<ushort> LoadVector128(ref char start, nuint offset)
-            => Unsafe.ReadUnaligned<Vector128<ushort>>(ref Unsafe.As<char, byte>(ref Unsafe.Add(ref start, (nint)offset)));
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static Vector256<ushort> LoadVector256(ref char start, nint offset)
-            => Unsafe.ReadUnaligned<Vector256<ushort>>(ref Unsafe.As<char, byte>(ref Unsafe.Add(ref start, offset)));
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static Vector256<ushort> LoadVector256(ref char start, nuint offset)
-            => Unsafe.ReadUnaligned<Vector256<ushort>>(ref Unsafe.As<char, byte>(ref Unsafe.Add(ref start, (nint)offset)));
-
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static nint GetCharVector128SpanLength(nint offset, nint length)
             => (length - offset) & ~(Vector128<ushort>.Count - 1);

From ec447c7f768f711458df29c7a162a21217cb207c Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Fri, 12 Aug 2022 18:26:01 +0200
Subject: [PATCH 20/34] Revert "address review from Jan, don't cast Span to
 ROS. Introduce new helper and avoid duplication by calling it from both Span
 and ROS"

This reverts commit 95a1b62de329541fa060b0573b143e3bc6fb088c.

# Conflicts:
#	src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
---
 .../src/System/MemoryExtensions.cs            | 330 ++++++++++++------
 1 file changed, 215 insertions(+), 115 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
index 4fbd7cf82eba53..de1d1f3627debb 100644
--- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
@@ -265,7 +265,7 @@ public static ReadOnlyMemory<char> AsMemory(this string? text, Range range)
         /// <param name="value">The value to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static bool Contains<T>(this Span<T> span, T value) where T : IEquatable<T>?
-            => Contains(ref MemoryMarshal.GetReference(span), value, span.Length);
+            => Contains((ReadOnlySpan<T>)span, value);
 
         /// <summary>
         /// Searches for the specified value and returns true if found. If not found, returns false. Values are compared using IEquatable{T}.Equals(T).
@@ -275,32 +275,40 @@ public static bool Contains<T>(this Span<T> span, T value) where T : IEquatable<
         /// <param name="value">The value to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static bool Contains<T>(this ReadOnlySpan<T> span, T value) where T : IEquatable<T>?
-            => Contains(ref MemoryMarshal.GetReference(span), value, span.Length);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static bool Contains<T>(ref T searchSpace, T value, int length) where T : IEquatable<T>?
         {
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.ContainsValueType(ref Unsafe.As<T, byte>(ref searchSpace), Unsafe.As<T, byte>(ref value), length);
+                    return SpanHelpers.ContainsValueType(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, byte>(ref value),
+                        span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    return SpanHelpers.ContainsValueType(ref Unsafe.As<T, short>(ref searchSpace), Unsafe.As<T, short>(ref value), length);
+                    return SpanHelpers.ContainsValueType(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value),
+                        span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(int))
                 {
-                    return SpanHelpers.ContainsValueType(ref Unsafe.As<T, int>(ref searchSpace), Unsafe.As<T, int>(ref value), length);
+                    return SpanHelpers.ContainsValueType(
+                        ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, int>(ref value),
+                        span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(long))
                 {
-                    return SpanHelpers.ContainsValueType(ref Unsafe.As<T, long>(ref searchSpace), Unsafe.As<T, long>(ref value), length);
+                    return SpanHelpers.ContainsValueType(
+                        ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, long>(ref value),
+                        span.Length);
                 }
             }
 
-            return SpanHelpers.Contains(ref searchSpace, value, length);
+            return SpanHelpers.Contains(ref MemoryMarshal.GetReference(span), value, span.Length);
         }
 
         /// <summary>
@@ -310,7 +318,36 @@ private static bool Contains<T>(ref T searchSpace, T value, int length) where T
         /// <param name="value">The value to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int IndexOf<T>(this Span<T> span, T value) where T : IEquatable<T>?
-            => IndexOf(ref MemoryMarshal.GetReference(span), value, span.Length);
+        {
+            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            {
+                if (Unsafe.SizeOf<T>() == sizeof(byte))
+                    return SpanHelpers.IndexOf(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, byte>(ref value),
+                        span.Length);
+
+                if (Unsafe.SizeOf<T>() == sizeof(char))
+                    return SpanHelpers.IndexOf(
+                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, char>(ref value),
+                        span.Length);
+
+                if (Unsafe.SizeOf<T>() == sizeof(int))
+                    return SpanHelpers.IndexOfValueType(
+                        ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, int>(ref value),
+                        span.Length);
+
+                if (Unsafe.SizeOf<T>() == sizeof(long))
+                    return SpanHelpers.IndexOfValueType(
+                        ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, long>(ref value),
+                        span.Length);
+            }
+
+            return SpanHelpers.IndexOf(ref MemoryMarshal.GetReference(span), value, span.Length);
+        }
 
         /// <summary>
         /// Searches for the specified sequence and returns the index of its first occurrence. If not found, returns -1. Values are compared using IEquatable{T}.Equals(T).
@@ -319,7 +356,26 @@ public static int IndexOf<T>(this Span<T> span, T value) where T : IEquatable<T>
         /// <param name="value">The sequence to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int IndexOf<T>(this Span<T> span, ReadOnlySpan<T> value) where T : IEquatable<T>?
-            => IndexOf(ref MemoryMarshal.GetReference(span), span.Length, ref MemoryMarshal.GetReference(value), value.Length);
+        {
+            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            {
+                if (Unsafe.SizeOf<T>() == sizeof(byte))
+                    return SpanHelpers.IndexOf(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        span.Length,
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(value)),
+                        value.Length);
+
+                if (Unsafe.SizeOf<T>() == sizeof(char))
+                    return SpanHelpers.IndexOf(
+                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
+                        span.Length,
+                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(value)),
+                        value.Length);
+            }
+
+            return SpanHelpers.IndexOf(ref MemoryMarshal.GetReference(span), span.Length, ref MemoryMarshal.GetReference(value), value.Length);
+        }
 
         /// <summary>
         /// Searches for the specified value and returns the index of its last occurrence. If not found, returns -1. Values are compared using IEquatable{T}.Equals(T).
@@ -328,7 +384,7 @@ public static int IndexOf<T>(this Span<T> span, ReadOnlySpan<T> value) where T :
         /// <param name="value">The value to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOf<T>(this Span<T> span, T value) where T : IEquatable<T>?
-            => LastIndexOf(ref MemoryMarshal.GetReference(span), value, span.Length);
+            => LastIndexOf((ReadOnlySpan<T>)span, value);
 
         /// <summary>
         /// Searches for the specified sequence and returns the index of its last occurrence. If not found, returns -1. Values are compared using IEquatable{T}.Equals(T).
@@ -337,7 +393,16 @@ public static int LastIndexOf<T>(this Span<T> span, T value) where T : IEquatabl
         /// <param name="value">The sequence to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOf<T>(this Span<T> span, ReadOnlySpan<T> value) where T : IEquatable<T>?
-            => LastIndexOf(ref MemoryMarshal.GetReference(span), span.Length, ref MemoryMarshal.GetReference(value), value.Length);
+        {
+            if (Unsafe.SizeOf<T>() == sizeof(byte) && RuntimeHelpers.IsBitwiseEquatable<T>())
+                return SpanHelpers.LastIndexOf(
+                    ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                    span.Length,
+                    ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(value)),
+                    value.Length);
+
+            return SpanHelpers.LastIndexOf<T>(ref MemoryMarshal.GetReference(span), span.Length, ref MemoryMarshal.GetReference(value), value.Length);
+        }
 
         /// <summary>Searches for the first index of any value other than the specified <paramref name="value"/>.</summary>
         /// <typeparam name="T">The type of the span and values.</typeparam>
@@ -347,8 +412,8 @@ public static int LastIndexOf<T>(this Span<T> span, ReadOnlySpan<T> value) where
         /// The index in the span of the first occurrence of any value other than <paramref name="value"/>.
         /// If all of the values are <paramref name="value"/>, returns -1.
         /// </returns>
-        public static int IndexOfAnyExcept<T>(this Span<T> span, T value) where T : IEquatable<T>?
-            => IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), value, span.Length);
+        public static int IndexOfAnyExcept<T>(this Span<T> span, T value) where T : IEquatable<T>? =>
+            IndexOfAnyExcept((ReadOnlySpan<T>)span, value);
 
         /// <summary>Searches for the first index of any value other than the specified <paramref name="value0"/> or <paramref name="value1"/>.</summary>
         /// <typeparam name="T">The type of the span and values.</typeparam>
@@ -359,8 +424,8 @@ public static int IndexOfAnyExcept<T>(this Span<T> span, T value) where T : IEqu
         /// The index in the span of the first occurrence of any value other than <paramref name="value0"/> and <paramref name="value1"/>.
         /// If all of the values are <paramref name="value0"/> or <paramref name="value1"/>, returns -1.
         /// </returns>
-        public static int IndexOfAnyExcept<T>(this Span<T> span, T value0, T value1) where T : IEquatable<T>?
-            => IndexOfAnyExcept((ReadOnlySpan<T>)span, value0, value1);
+        public static int IndexOfAnyExcept<T>(this Span<T> span, T value0, T value1) where T : IEquatable<T>? =>
+            IndexOfAnyExcept((ReadOnlySpan<T>)span, value0, value1);
 
         /// <summary>Searches for the first index of any value other than the specified <paramref name="value0"/>, <paramref name="value1"/>, or <paramref name="value2"/>.</summary>
         /// <typeparam name="T">The type of the span and values.</typeparam>
@@ -372,8 +437,8 @@ public static int IndexOfAnyExcept<T>(this Span<T> span, T value0, T value1) whe
         /// The index in the span of the first occurrence of any value other than <paramref name="value0"/>, <paramref name="value1"/>, and <paramref name="value2"/>.
         /// If all of the values are <paramref name="value0"/>, <paramref name="value1"/>, and <paramref name="value2"/>, returns -1.
         /// </returns>
-        public static int IndexOfAnyExcept<T>(this Span<T> span, T value0, T value1, T value2) where T : IEquatable<T>?
-            => IndexOfAnyExcept((ReadOnlySpan<T>)span, value0, value1, value2);
+        public static int IndexOfAnyExcept<T>(this Span<T> span, T value0, T value1, T value2) where T : IEquatable<T>? =>
+            IndexOfAnyExcept((ReadOnlySpan<T>)span, value0, value1, value2);
 
         /// <summary>Searches for the first index of any value other than the specified <paramref name="values"/>.</summary>
         /// <typeparam name="T">The type of the span and values.</typeparam>
@@ -383,8 +448,8 @@ public static int IndexOfAnyExcept<T>(this Span<T> span, T value0, T value1, T v
         /// The index in the span of the first occurrence of any value other than those in <paramref name="values"/>.
         /// If all of the values are in <paramref name="values"/>, returns -1.
         /// </returns>
-        public static int IndexOfAnyExcept<T>(this Span<T> span, ReadOnlySpan<T> values) where T : IEquatable<T>?
-            => IndexOfAnyExcept((ReadOnlySpan<T>)span, values);
+        public static int IndexOfAnyExcept<T>(this Span<T> span, ReadOnlySpan<T> values) where T : IEquatable<T>? =>
+            IndexOfAnyExcept((ReadOnlySpan<T>)span, values);
 
         /// <summary>Searches for the first index of any value other than the specified <paramref name="value"/>.</summary>
         /// <typeparam name="T">The type of the span and values.</typeparam>
@@ -396,32 +461,46 @@ public static int IndexOfAnyExcept<T>(this Span<T> span, ReadOnlySpan<T> values)
         /// </returns>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int IndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value) where T : IEquatable<T>?
-            => IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), value, span.Length);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int IndexOfAnyExcept<T>(ref T searchSpace, T value, int length) where T : IEquatable<T>?
         {
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.IndexOfAnyExceptValueType(ref Unsafe.As<T, byte>(ref searchSpace), Unsafe.As<T, byte>(ref value), length);
+                    return SpanHelpers.IndexOfAnyExceptValueType(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, byte>(ref value),
+                        span.Length);
                 }
-                else if (Unsafe.SizeOf<T>() == sizeof(short))
+
+                if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    return SpanHelpers.IndexOfAnyExceptValueType(ref Unsafe.As<T, short>(ref searchSpace), Unsafe.As<T, short>(ref value), length);
+                    return SpanHelpers.IndexOfAnyExceptValueType(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value),
+                        span.Length);
                 }
-                else if (Unsafe.SizeOf<T>() == sizeof(int))
+
+                if (Unsafe.SizeOf<T>() == sizeof(int))
                 {
-                    return SpanHelpers.IndexOfAnyExceptValueType(ref Unsafe.As<T, int>(ref searchSpace), Unsafe.As<T, int>(ref value), length);
+                    return SpanHelpers.IndexOfAnyExceptValueType(
+                        ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, int>(ref value),
+                        span.Length);
                 }
-                else if (Unsafe.SizeOf<T>() == sizeof(long))
+
+                if (Unsafe.SizeOf<T>() == sizeof(long))
                 {
-                    return SpanHelpers.IndexOfAnyExceptValueType(ref Unsafe.As<T, long>(ref searchSpace), Unsafe.As<T, long>(ref value), length);
+                    return SpanHelpers.IndexOfAnyExceptValueType(
+                        ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, long>(ref value),
+                        span.Length);
                 }
             }
 
-            return SpanHelpers.IndexOfAnyExcept(ref searchSpace, value, length);
+            return SpanHelpers.IndexOfAnyExcept(
+                ref MemoryMarshal.GetReference(span),
+                value,
+                span.Length);
         }
 
         /// <summary>Searches for the first index of any value other than the specified <paramref name="value0"/> or <paramref name="value1"/>.</summary>
@@ -605,8 +684,8 @@ public static int IndexOfAnyExcept<T>(this ReadOnlySpan<T> span, ReadOnlySpan<T>
         /// The index in the span of the last occurrence of any value other than <paramref name="value"/>.
         /// If all of the values are <paramref name="value"/>, returns -1.
         /// </returns>
-        public static int LastIndexOfAnyExcept<T>(this Span<T> span, T value) where T : IEquatable<T>?
-            => LastIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), value, span.Length);
+        public static int LastIndexOfAnyExcept<T>(this Span<T> span, T value) where T : IEquatable<T>? =>
+            LastIndexOfAnyExcept((ReadOnlySpan<T>)span, value);
 
         /// <summary>Searches for the last index of any value other than the specified <paramref name="value0"/> or <paramref name="value1"/>.</summary>
         /// <typeparam name="T">The type of the span and values.</typeparam>
@@ -617,8 +696,8 @@ public static int LastIndexOfAnyExcept<T>(this Span<T> span, T value) where T :
         /// The index in the span of the last occurrence of any value other than <paramref name="value0"/> and <paramref name="value1"/>.
         /// If all of the values are <paramref name="value0"/> or <paramref name="value1"/>, returns -1.
         /// </returns>
-        public static int LastIndexOfAnyExcept<T>(this Span<T> span, T value0, T value1) where T : IEquatable<T>?
-            => LastIndexOfAnyExcept((ReadOnlySpan<T>)span, value0, value1);
+        public static int LastIndexOfAnyExcept<T>(this Span<T> span, T value0, T value1) where T : IEquatable<T>? =>
+            LastIndexOfAnyExcept((ReadOnlySpan<T>)span, value0, value1);
 
         /// <summary>Searches for the last index of any value other than the specified <paramref name="value0"/>, <paramref name="value1"/>, or <paramref name="value2"/>.</summary>
         /// <typeparam name="T">The type of the span and values.</typeparam>
@@ -653,33 +732,42 @@ public static int LastIndexOfAnyExcept<T>(this Span<T> span, ReadOnlySpan<T> val
         /// If all of the values are <paramref name="value"/>, returns -1.
         /// </returns>
         public static int LastIndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value) where T : IEquatable<T>?
-            => LastIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), value, span.Length);
-
-        private static int LastIndexOfAnyExcept<T>(ref T searchSpace, T value, int length) where T : IEquatable<T>?
         {
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.LastIndexOfAnyExceptValueType(ref Unsafe.As<T, byte>(ref searchSpace), Unsafe.As<T, byte>(ref value), length);
+                    return SpanHelpers.LastIndexOfAnyExceptValueType(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, byte>(ref value),
+                        span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    return SpanHelpers.LastIndexOfAnyExceptValueType(ref Unsafe.As<T, short>(ref searchSpace), Unsafe.As<T, short>(ref value), length);
+                    return SpanHelpers.LastIndexOfAnyExceptValueType(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value),
+                        span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(int))
                 {
-                    return SpanHelpers.LastIndexOfAnyExceptValueType(ref Unsafe.As<T, int>(ref searchSpace), Unsafe.As<T, int>(ref value), length);
+                    return SpanHelpers.LastIndexOfAnyExceptValueType(
+                        ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, int>(ref value),
+                        span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(long))
                 {
-                    return SpanHelpers.LastIndexOfAnyExceptValueType(ref Unsafe.As<T, long>(ref searchSpace), Unsafe.As<T, long>(ref value), length);
+                    return SpanHelpers.LastIndexOfAnyExceptValueType(
+                        ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, long>(ref value),
+                        span.Length);
                 }
             }
 
-            for (int i = length - 1; i >= 0; i--)
+            for (int i = span.Length - 1; i >= 0; i--)
             {
-                if (!EqualityComparer<T>.Default.Equals(Unsafe.Add(ref searchSpace, i), value))
+                if (!EqualityComparer<T>.Default.Equals(span[i], value))
                 {
                     return i;
                 }
@@ -891,32 +979,23 @@ ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(other)),
         /// <param name="value">The value to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int IndexOf<T>(this ReadOnlySpan<T> span, T value) where T : IEquatable<T>?
-            => IndexOf(ref MemoryMarshal.GetReference(span), value, span.Length);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int IndexOf<T>(ref T searchSpace, T value, int length) where T : IEquatable<T>?
         {
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
-                {
-                    return SpanHelpers.IndexOfValueType(ref Unsafe.As<T, byte>(ref searchSpace), Unsafe.As<T, byte>(ref value), length);
-                }
-                else if (Unsafe.SizeOf<T>() == sizeof(short))
-                {
-                    return SpanHelpers.IndexOfValueType(ref Unsafe.As<T, short>(ref searchSpace), Unsafe.As<T, short>(ref value), length);
-                }
-                else if (Unsafe.SizeOf<T>() == sizeof(int))
-                {
-                    return SpanHelpers.IndexOfValueType(ref Unsafe.As<T, int>(ref searchSpace), Unsafe.As<T, int>(ref value), length);
-                }
-                else if (Unsafe.SizeOf<T>() == sizeof(long))
-                {
-                    return SpanHelpers.IndexOfValueType(ref Unsafe.As<T, long>(ref searchSpace), Unsafe.As<T, long>(ref value), length);
-                }
+                    return SpanHelpers.IndexOf(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, byte>(ref value),
+                        span.Length);
+
+                if (Unsafe.SizeOf<T>() == sizeof(char))
+                    return SpanHelpers.IndexOf(
+                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, char>(ref value),
+                        span.Length);
             }
 
-            return SpanHelpers.IndexOf(ref searchSpace, value, length);
+            return SpanHelpers.IndexOf(ref MemoryMarshal.GetReference(span), value, span.Length);
         }
 
         /// <summary>
@@ -926,24 +1005,25 @@ private static int IndexOf<T>(ref T searchSpace, T value, int length) where T :
         /// <param name="value">The sequence to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int IndexOf<T>(this ReadOnlySpan<T> span, ReadOnlySpan<T> value) where T : IEquatable<T>?
-            => IndexOf(ref MemoryMarshal.GetReference(span), span.Length, ref MemoryMarshal.GetReference(value), value.Length);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int IndexOf<T>(ref T searchSpace, int searchSpaceLength, ref T value, int valueLength) where T : IEquatable<T>?
         {
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
-                {
-                    return SpanHelpers.IndexOf(ref Unsafe.As<T, byte>(ref searchSpace), searchSpaceLength, ref Unsafe.As<T, byte>(ref value), valueLength);
-                }
-                else if (Unsafe.SizeOf<T>() == sizeof(char))
-                {
-                    return SpanHelpers.IndexOf(ref Unsafe.As<T, char>(ref searchSpace), searchSpaceLength, ref Unsafe.As<T, char>(ref value), valueLength);
-                }
+                    return SpanHelpers.IndexOf(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        span.Length,
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(value)),
+                        value.Length);
+
+                if (Unsafe.SizeOf<T>() == sizeof(char))
+                    return SpanHelpers.IndexOf(
+                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
+                        span.Length,
+                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(value)),
+                        value.Length);
             }
 
-            return SpanHelpers.IndexOf(ref searchSpace, searchSpaceLength, ref value, valueLength);
+            return SpanHelpers.IndexOf(ref MemoryMarshal.GetReference(span), span.Length, ref MemoryMarshal.GetReference(value), value.Length);
         }
 
         /// <summary>
@@ -953,32 +1033,40 @@ private static int IndexOf<T>(ref T searchSpace, int searchSpaceLength, ref T va
         /// <param name="value">The value to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOf<T>(this ReadOnlySpan<T> span, T value) where T : IEquatable<T>?
-            => LastIndexOf(ref MemoryMarshal.GetReference(span), value, span.Length);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int LastIndexOf<T>(ref T searchSpace, T value, int length) where T : IEquatable<T>?
         {
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.LastIndexOfValueType(ref Unsafe.As<T, byte>(ref searchSpace), Unsafe.As<T, byte>(ref value), length);
+                    return SpanHelpers.LastIndexOfValueType(
+                            ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                            Unsafe.As<T, byte>(ref value),
+                            span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    return SpanHelpers.LastIndexOfValueType(ref Unsafe.As<T, short>(ref searchSpace), Unsafe.As<T, short>(ref value), length);
+                    return SpanHelpers.LastIndexOfValueType(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value),
+                        span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(int))
                 {
-                    return SpanHelpers.LastIndexOfValueType(ref Unsafe.As<T, int>(ref searchSpace), Unsafe.As<T, int>(ref value), length);
+                    return SpanHelpers.LastIndexOfValueType(
+                        ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, int>(ref value),
+                        span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(long))
                 {
-                    return SpanHelpers.LastIndexOfValueType(ref Unsafe.As<T, long>(ref searchSpace), Unsafe.As<T, long>(ref value), length);
+                    return SpanHelpers.LastIndexOfValueType(
+                        ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, long>(ref value),
+                        span.Length);
                 }
             }
 
-            return SpanHelpers.LastIndexOf<T>(ref searchSpace, value, length);
+            return SpanHelpers.LastIndexOf<T>(ref MemoryMarshal.GetReference(span), value, span.Length);
         }
 
         /// <summary>
@@ -988,24 +1076,28 @@ private static int LastIndexOf<T>(ref T searchSpace, T value, int length) where
         /// <param name="value">The sequence to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOf<T>(this ReadOnlySpan<T> span, ReadOnlySpan<T> value) where T : IEquatable<T>?
-            => LastIndexOf(ref MemoryMarshal.GetReference(span), span.Length, ref MemoryMarshal.GetReference(value), value.Length);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int LastIndexOf<T>(ref T searchSpace, int searchSpaceLength, ref T value, int valueLength) where T : IEquatable<T>?
         {
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.LastIndexOf(ref Unsafe.As<T, byte>(ref searchSpace), searchSpaceLength, ref Unsafe.As<T, byte>(ref value), valueLength);
+                    return SpanHelpers.LastIndexOf(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        span.Length,
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(value)),
+                        value.Length);
                 }
                 if (Unsafe.SizeOf<T>() == sizeof(char))
                 {
-                    return SpanHelpers.LastIndexOf(ref Unsafe.As<T, char>(ref searchSpace), searchSpaceLength, ref Unsafe.As<T, char>(ref value), valueLength);
+                    return SpanHelpers.LastIndexOf(
+                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
+                        span.Length,
+                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(value)),
+                        value.Length);
                 }
             }
 
-            return SpanHelpers.LastIndexOf<T>(ref searchSpace, searchSpaceLength, ref value, valueLength);
+            return SpanHelpers.LastIndexOf<T>(ref MemoryMarshal.GetReference(span), span.Length, ref MemoryMarshal.GetReference(value), value.Length);
         }
 
         /// <summary>
@@ -1276,7 +1368,7 @@ private static unsafe int IndexOfAnyProbabilistic(ref char searchSpace, int sear
         /// <param name="value1">One of the values to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOfAny<T>(this Span<T> span, T value0, T value1) where T : IEquatable<T>?
-            => LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, span.Length);
+            => LastIndexOfAny((ReadOnlySpan<T>)span, value0, value1);
 
         /// <summary>
         /// Searches for the last index of any of the specified values similar to calling LastIndexOf several times with the logical OR operator. If not found, returns -1.
@@ -1287,7 +1379,7 @@ public static int LastIndexOfAny<T>(this Span<T> span, T value0, T value1) where
         /// <param name="value2">One of the values to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOfAny<T>(this Span<T> span, T value0, T value1, T value2) where T : IEquatable<T>?
-            => LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, value2, span.Length);
+            => LastIndexOfAny((ReadOnlySpan<T>)span, value0, value1, value2);
 
         /// <summary>
         /// Searches for the last index of any of the specified values similar to calling LastIndexOf several times with the logical OR operator. If not found, returns -1.
@@ -1315,32 +1407,44 @@ ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(values)),
         /// <param name="value1">One of the values to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOfAny<T>(this ReadOnlySpan<T> span, T value0, T value1) where T : IEquatable<T>?
-            => LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, span.Length);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int LastIndexOfAny<T>(ref T searchSpace, T value0, T value1, int length) where T : IEquatable<T>?
         {
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType(ref Unsafe.As<T, byte>(ref searchSpace), Unsafe.As<T, byte>(ref value0), Unsafe.As<T, byte>(ref value1), length);
+                    return SpanHelpers.LastIndexOfAnyValueType(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, byte>(ref value0),
+                        Unsafe.As<T, byte>(ref value1),
+                        span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType(ref Unsafe.As<T, short>(ref searchSpace), Unsafe.As<T, short>(ref value0), Unsafe.As<T, short>(ref value1), length);
+                    return SpanHelpers.LastIndexOfAnyValueType(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value0),
+                        Unsafe.As<T, short>(ref value1),
+                        span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(int))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType(ref Unsafe.As<T, int>(ref searchSpace), Unsafe.As<T, int>(ref value0), Unsafe.As<T, int>(ref value1), length);
+                    return SpanHelpers.LastIndexOfAnyValueType(
+                        ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, int>(ref value0),
+                        Unsafe.As<T, int>(ref value1),
+                        span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(long))
                 {
-                    return SpanHelpers.LastIndexOfAnyValueType(ref Unsafe.As<T, long>(ref searchSpace), Unsafe.As<T, long>(ref value0), Unsafe.As<T, long>(ref value1), length);
+                    return SpanHelpers.LastIndexOfAnyValueType(
+                        ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, long>(ref value0),
+                        Unsafe.As<T, long>(ref value1),
+                        span.Length);
                 }
             }
 
-            return SpanHelpers.LastIndexOfAny(ref searchSpace, value0, value1, length);
+            return SpanHelpers.LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, span.Length);
         }
 
         /// <summary>
@@ -1352,35 +1456,31 @@ private static int LastIndexOfAny<T>(ref T searchSpace, T value0, T value1, int
         /// <param name="value2">One of the values to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOfAny<T>(this ReadOnlySpan<T> span, T value0, T value1, T value2) where T : IEquatable<T>?
-            => LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, value2, span.Length);
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int LastIndexOfAny<T>(ref T searchSpace, T value0, T value1, T value2, int length) where T : IEquatable<T>?
         {
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
                     return SpanHelpers.LastIndexOfAnyValueType(
-                        ref Unsafe.As<T, byte>(ref searchSpace),
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, byte>(ref value0),
                         Unsafe.As<T, byte>(ref value1),
                         Unsafe.As<T, byte>(ref value2),
-                        length);
+                        span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
                     return SpanHelpers.LastIndexOfAnyValueType(
-                        ref Unsafe.As<T, short>(ref searchSpace),
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, short>(ref value0),
                         Unsafe.As<T, short>(ref value1),
                         Unsafe.As<T, short>(ref value2),
-                        length);
+                        span.Length);
                 }
                 // we can easily add int and long here, but there must be an evidence that shows that it's actually needed
             }
 
-            return SpanHelpers.LastIndexOfAny(ref searchSpace, value0, value1, value2, length);
+            return SpanHelpers.LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, value2, span.Length);
         }
 
         /// <summary>

From 62f16c17a9bec835f508922820f40290ac34c247 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Fri, 12 Aug 2022 18:50:41 +0200
Subject: [PATCH 21/34] add/remove special handling of some types

---
 .../src/System/MemoryExtensions.cs            | 208 ++++++++++++------
 .../src/System/SpanHelpers.Byte.cs            |   4 +-
 2 files changed, 146 insertions(+), 66 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
index de1d1f3627debb..de5b246fc3390f 100644
--- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
@@ -7,7 +7,6 @@
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 using System.Runtime.Intrinsics;
-using System.Runtime.Intrinsics.X86;
 
 namespace System
 {
@@ -265,7 +264,41 @@ public static ReadOnlyMemory<char> AsMemory(this string? text, Range range)
         /// <param name="value">The value to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static bool Contains<T>(this Span<T> span, T value) where T : IEquatable<T>?
-            => Contains((ReadOnlySpan<T>)span, value);
+        {
+            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            {
+                if (Unsafe.SizeOf<T>() == sizeof(byte))
+                {
+                    return SpanHelpers.ContainsValueType(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, byte>(ref value),
+                        span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
+                {
+                    return SpanHelpers.ContainsValueType(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value),
+                        span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(int))
+                {
+                    return SpanHelpers.ContainsValueType(
+                        ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, int>(ref value),
+                        span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(long))
+                {
+                    return SpanHelpers.ContainsValueType(
+                        ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, long>(ref value),
+                        span.Length);
+                }
+            }
+
+            return SpanHelpers.Contains(ref MemoryMarshal.GetReference(span), value, span.Length);
+        }
 
         /// <summary>
         /// Searches for the specified value and returns true if found. If not found, returns false. Values are compared using IEquatable{T}.Equals(T).
@@ -322,15 +355,15 @@ public static int IndexOf<T>(this Span<T> span, T value) where T : IEquatable<T>
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
-                    return SpanHelpers.IndexOf(
+                    return SpanHelpers.IndexOfValueType(
                         ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, byte>(ref value),
                         span.Length);
 
-                if (Unsafe.SizeOf<T>() == sizeof(char))
-                    return SpanHelpers.IndexOf(
-                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, char>(ref value),
+                if (Unsafe.SizeOf<T>() == sizeof(short))
+                    return SpanHelpers.IndexOfValueType(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value),
                         span.Length);
 
                 if (Unsafe.SizeOf<T>() == sizeof(int))
@@ -384,7 +417,41 @@ ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(value)),
         /// <param name="value">The value to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOf<T>(this Span<T> span, T value) where T : IEquatable<T>?
-            => LastIndexOf((ReadOnlySpan<T>)span, value);
+        {
+            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            {
+                if (Unsafe.SizeOf<T>() == sizeof(byte))
+                {
+                    return SpanHelpers.LastIndexOfValueType(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, byte>(ref value),
+                        span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
+                {
+                    return SpanHelpers.LastIndexOfValueType(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value),
+                        span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(int))
+                {
+                    return SpanHelpers.LastIndexOfValueType(
+                        ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, int>(ref value),
+                        span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(long))
+                {
+                    return SpanHelpers.LastIndexOfValueType(
+                        ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, long>(ref value),
+                        span.Length);
+                }
+            }
+
+            return SpanHelpers.LastIndexOf<T>(ref MemoryMarshal.GetReference(span), value, span.Length);
+        }
 
         /// <summary>
         /// Searches for the specified sequence and returns the index of its last occurrence. If not found, returns -1. Values are compared using IEquatable{T}.Equals(T).
@@ -749,20 +816,6 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, short>(ref value),
                         span.Length);
                 }
-                else if (Unsafe.SizeOf<T>() == sizeof(int))
-                {
-                    return SpanHelpers.LastIndexOfAnyExceptValueType(
-                        ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, int>(ref value),
-                        span.Length);
-                }
-                else if (Unsafe.SizeOf<T>() == sizeof(long))
-                {
-                    return SpanHelpers.LastIndexOfAnyExceptValueType(
-                        ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, long>(ref value),
-                        span.Length);
-                }
             }
 
             for (int i = span.Length - 1; i >= 0; i--)
@@ -805,22 +858,6 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, short>(ref value1),
                         span.Length);
                 }
-                else if (Unsafe.SizeOf<T>() == sizeof(int))
-                {
-                    return SpanHelpers.LastIndexOfAnyExceptValueType(
-                        ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, int>(ref value0),
-                        Unsafe.As<T, int>(ref value1),
-                        span.Length);
-                }
-                else if (Unsafe.SizeOf<T>() == sizeof(long))
-                {
-                    return SpanHelpers.LastIndexOfAnyExceptValueType(
-                        ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, long>(ref value0),
-                        Unsafe.As<T, long>(ref value1),
-                        span.Length);
-                }
             }
 
             for (int i = span.Length - 1; i >= 0; i--)
@@ -983,15 +1020,27 @@ public static int IndexOf<T>(this ReadOnlySpan<T> span, T value) where T : IEqua
             if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
-                    return SpanHelpers.IndexOf(
+                    return SpanHelpers.IndexOfValueType(
                         ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, byte>(ref value),
                         span.Length);
 
-                if (Unsafe.SizeOf<T>() == sizeof(char))
-                    return SpanHelpers.IndexOf(
-                        ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, char>(ref value),
+                if (Unsafe.SizeOf<T>() == sizeof(short))
+                    return SpanHelpers.IndexOfValueType(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value),
+                        span.Length);
+
+                if (Unsafe.SizeOf<T>() == sizeof(int))
+                    return SpanHelpers.IndexOfValueType(
+                        ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, int>(ref value),
+                        span.Length);
+
+                if (Unsafe.SizeOf<T>() == sizeof(long))
+                    return SpanHelpers.IndexOfValueType(
+                        ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, long>(ref value),
                         span.Length);
             }
 
@@ -1039,9 +1088,9 @@ public static int LastIndexOf<T>(this ReadOnlySpan<T> span, T value) where T : I
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
                     return SpanHelpers.LastIndexOfValueType(
-                            ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
-                            Unsafe.As<T, byte>(ref value),
-                            span.Length);
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, byte>(ref value),
+                        span.Length);
                 }
                 else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
@@ -1368,7 +1417,29 @@ private static unsafe int IndexOfAnyProbabilistic(ref char searchSpace, int sear
         /// <param name="value1">One of the values to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOfAny<T>(this Span<T> span, T value0, T value1) where T : IEquatable<T>?
-            => LastIndexOfAny((ReadOnlySpan<T>)span, value0, value1);
+        {
+            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            {
+                if (Unsafe.SizeOf<T>() == sizeof(byte))
+                {
+                    return SpanHelpers.LastIndexOfAnyValueType(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, byte>(ref value0),
+                        Unsafe.As<T, byte>(ref value1),
+                        span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
+                {
+                    return SpanHelpers.LastIndexOfAnyValueType(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value0),
+                        Unsafe.As<T, short>(ref value1),
+                        span.Length);
+                }
+            }
+
+            return SpanHelpers.LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, span.Length);
+        }
 
         /// <summary>
         /// Searches for the last index of any of the specified values similar to calling LastIndexOf several times with the logical OR operator. If not found, returns -1.
@@ -1379,7 +1450,32 @@ public static int LastIndexOfAny<T>(this Span<T> span, T value0, T value1) where
         /// <param name="value2">One of the values to search for.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOfAny<T>(this Span<T> span, T value0, T value1, T value2) where T : IEquatable<T>?
-            => LastIndexOfAny((ReadOnlySpan<T>)span, value0, value1, value2);
+        {
+            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            {
+                if (Unsafe.SizeOf<T>() == sizeof(byte))
+                {
+                    return SpanHelpers.LastIndexOfAnyValueType(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, byte>(ref value0),
+                        Unsafe.As<T, byte>(ref value1),
+                        Unsafe.As<T, byte>(ref value2),
+                        span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
+                {
+                    return SpanHelpers.LastIndexOfAnyValueType(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value0),
+                        Unsafe.As<T, short>(ref value1),
+                        Unsafe.As<T, short>(ref value2),
+                        span.Length);
+                }
+                // we can easily add int and long here, but there must be an evidence that shows that it's actually needed
+            }
+
+            return SpanHelpers.LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, value2, span.Length);
+        }
 
         /// <summary>
         /// Searches for the last index of any of the specified values similar to calling LastIndexOf several times with the logical OR operator. If not found, returns -1.
@@ -1426,22 +1522,6 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, short>(ref value1),
                         span.Length);
                 }
-                else if (Unsafe.SizeOf<T>() == sizeof(int))
-                {
-                    return SpanHelpers.LastIndexOfAnyValueType(
-                        ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, int>(ref value0),
-                        Unsafe.As<T, int>(ref value1),
-                        span.Length);
-                }
-                else if (Unsafe.SizeOf<T>() == sizeof(long))
-                {
-                    return SpanHelpers.LastIndexOfAnyValueType(
-                        ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
-                        Unsafe.As<T, long>(ref value0),
-                        Unsafe.As<T, long>(ref value1),
-                        span.Length);
-                }
             }
 
             return SpanHelpers.LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, span.Length);
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
index ad722fa3408bbb..820e3634ac82fa 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
@@ -196,7 +196,7 @@ public static int LastIndexOf(ref byte searchSpace, int searchSpaceLength, ref b
 
             int valueTailLength = valueLength - 1;
             if (valueTailLength == 0)
-                return LastIndexOfValueType<byte, DontNegate<byte>>(ref searchSpace, value, searchSpaceLength); // for single-byte values use plain LastIndexOf
+                return LastIndexOfValueType(ref searchSpace, value, searchSpaceLength); // for single-byte values use plain LastIndexOf
 
             int offset = 0;
             byte valueHead = value;
@@ -216,7 +216,7 @@ public static int LastIndexOf(ref byte searchSpace, int searchSpaceLength, ref b
                     break;  // The unsearched portion is now shorter than the sequence we're looking for. So it can't be there.
 
                 // Do a quick search for the first element of "value".
-                int relativeIndex = LastIndexOfValueType<byte, DontNegate<byte>>(ref searchSpace, valueHead, remainingSearchSpaceLength);
+                int relativeIndex = LastIndexOfValueType(ref searchSpace, valueHead, remainingSearchSpaceLength);
                 if (relativeIndex < 0)
                     break;
 

From 26c188dfa4abdf7b026db7d503a147048c859a6d Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Fri, 12 Aug 2022 19:23:51 +0200
Subject: [PATCH 22/34] fix NativeAOT build

---
 .../src/Internal/Runtime/CompilerHelpers/InteropHelpers.cs    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/InteropHelpers.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/InteropHelpers.cs
index 4e8bfcd492dde1..742f3bbc4991c0 100644
--- a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/InteropHelpers.cs
+++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/InteropHelpers.cs
@@ -55,7 +55,7 @@ internal static unsafe void StringToByValAnsiString(string str, byte* pNative, i
 
         public static unsafe string ByValAnsiStringToString(byte* buffer, int length)
         {
-            int end = SpanHelpers.IndexOf(ref *(byte*)buffer, 0, length);
+            int end = SpanHelpers.IndexOfValueType<byte>(ref *buffer, 0, length);
             if (end != -1)
             {
                 length = end;
@@ -77,7 +77,7 @@ internal static unsafe void StringToUnicodeFixedArray(string str, ushort* buffer
 
         internal static unsafe string UnicodeToStringFixedArray(ushort* buffer, int length)
         {
-            int end = SpanHelpers.IndexOf(ref *(char*)buffer, '\0', length);
+            int end = SpanHelpers.IndexOfChar(ref *(char*)buffer, '\0', length);
             if (end != -1)
             {
                 length = end;

From 2af01e3e530e1797493b35180867eed00e645fb5 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Sat, 13 Aug 2022 14:04:30 +0200
Subject: [PATCH 23/34] address code review feedback

---
 .../src/System/SpanHelpers.Byte.cs            |  14 ++-
 .../src/System/SpanHelpers.Char.cs            |   5 +-
 .../src/System/SpanHelpers.T.cs               | 102 ++++++++----------
 .../src/System/String.cs                      |  34 +-----
 4 files changed, 62 insertions(+), 93 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
index 820e3634ac82fa..7f76563a2f7a2c 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
 using System.Numerics;
 using System.Runtime.CompilerServices;
 using System.Runtime.Intrinsics;
@@ -332,8 +333,16 @@ ref Unsafe.Add(ref searchSpace, offset + bitPos),
             }
         }
 
+        [DoesNotReturn]
+        private static void ThrowMustBeNullTerminatedString()
+        {
+            throw new ArgumentException(SR.Arg_MustBeNullTerminatedString);
+        }
+
+        // IndexOfNullByte processes memory in aligned chunks, and thus it won't crash even if it accesses memory beyond the null terminator.
+        // This behavior is an implementation detail of the runtime and callers outside System.Private.CoreLib must not depend on it.
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        public static unsafe int IndexOfNullByte(ref byte searchSpace)
+        internal static unsafe int IndexOfNullByte(ref byte searchSpace)
         {
             const int length = int.MaxValue;
 
@@ -508,7 +517,8 @@ public static unsafe int IndexOfNullByte(ref byte searchSpace)
                     }
                 }
             }
-            return -1;
+
+            ThrowMustBeNullTerminatedString();
         Found: // Workaround for https://github.com/dotnet/runtime/issues/8795
             return (int)offset;
         Found1:
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
index 5f98699a6b955a..1a1becda8e433a 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
@@ -421,6 +421,8 @@ public static unsafe int SequenceCompareTo(ref char first, int firstLength, ref
             return lengthDelta;
         }
 
+        // IndexOfNullCharacter processes memory in aligned chunks, and thus it won't crash even if it accesses memory beyond the null terminator.
+        // This behavior is an implementation detail of the runtime and callers outside System.Private.CoreLib must not depend on it.
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
         public static unsafe int IndexOfNullCharacter(ref char searchSpace)
         {
@@ -613,7 +615,8 @@ public static unsafe int IndexOfNullCharacter(ref char searchSpace)
                     }
                 }
             }
-            return -1;
+
+            ThrowMustBeNullTerminatedString();
         Found3:
             return (int)(offset + 3);
         Found2:
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
index b9c22bbbec2cc7..496b52e69dc035 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
@@ -1278,13 +1278,11 @@ private static int IndexOfValueType<T, N>(ref T searchSpace, T value, int length
             Debug.Assert(length >= 0, "Expected non-negative length");
             Debug.Assert(value is byte or short or int or long, "Expected caller to normalize to one of these types");
 
-            N negator = default;
-
             if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
             {
                 for (int i = 0; i < length; i++)
                 {
-                    if (negator.NegateIfNeeded(Unsafe.Add(ref searchSpace, i).Equals(value)))
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, i).Equals(value)))
                     {
                         return i;
                     }
@@ -1299,7 +1297,7 @@ private static int IndexOfValueType<T, N>(ref T searchSpace, T value, int length
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
-                    equals = negator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref currentSearchSpace)));
+                    equals = N.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref currentSearchSpace)));
                     if (equals == Vector256<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector256<T>.Count);
@@ -1313,7 +1311,7 @@ private static int IndexOfValueType<T, N>(ref T searchSpace, T value, int length
                 // If any elements remain, process the last vector in the search space.
                 if ((uint)length % Vector256<T>.Count != 0)
                 {
-                    equals = negator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref oneVectorAwayFromEnd)));
+                    equals = N.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref oneVectorAwayFromEnd)));
                     if (equals != Vector256<T>.Zero)
                     {
                         return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
@@ -1329,7 +1327,7 @@ private static int IndexOfValueType<T, N>(ref T searchSpace, T value, int length
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
-                    equals = negator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref currentSearchSpace)));
+                    equals = N.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref currentSearchSpace)));
                     if (equals == Vector128<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128<T>.Count);
@@ -1343,7 +1341,7 @@ private static int IndexOfValueType<T, N>(ref T searchSpace, T value, int length
                 // If any elements remain, process the first vector in the search space.
                 if ((uint)length % Vector128<T>.Count != 0)
                 {
-                    equals = negator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref oneVectorAwayFromEnd)));
+                    equals = N.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref oneVectorAwayFromEnd)));
                     if (equals != Vector128<T>.Zero)
                     {
                         return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
@@ -1374,14 +1372,12 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
             Debug.Assert(length >= 0, "Expected non-negative length");
             Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types");
 
-            N negator = default;
-
             if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
             {
                 for (int i = 0; i < length; i++)
                 {
                     T current = Unsafe.Add(ref searchSpace, i);
-                    if (negator.NegateIfNeeded(current.Equals(value0) | current.Equals(value1)))
+                    if (N.NegateIfNeeded(current.Equals(value0) | current.Equals(value1)))
                     {
                         return i;
                     }
@@ -1397,7 +1393,7 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
                 do
                 {
                     current = Vector256.LoadUnsafe(ref currentSearchSpace);
-                    equals = negator.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current));
+                    equals = N.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current));
                     if (equals == Vector256<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector256<T>.Count);
@@ -1412,7 +1408,7 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
                 if ((uint)length % Vector256<T>.Count != 0)
                 {
                     current = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd);
-                    equals = negator.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current));
+                    equals = N.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current));
                     if (equals != Vector256<T>.Zero)
                     {
                         return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
@@ -1429,7 +1425,7 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
                 do
                 {
                     current = Vector128.LoadUnsafe(ref currentSearchSpace);
-                    equals = negator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current));
+                    equals = N.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current));
                     if (equals == Vector128<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128<T>.Count);
@@ -1444,7 +1440,7 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
                 if ((uint)length % Vector128<T>.Count != 0)
                 {
                     current = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd);
-                    equals = negator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current));
+                    equals = N.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current));
                     if (equals != Vector128<T>.Zero)
                     {
                         return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
@@ -1471,14 +1467,12 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
             Debug.Assert(length >= 0, "Expected non-negative length");
             Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types");
 
-            N negator = default;
-
             if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
             {
                 for (int i = 0; i < length; i++)
                 {
                     T current = Unsafe.Add(ref searchSpace, i);
-                    if (negator.NegateIfNeeded(current.Equals(value0) | current.Equals(value1) | current.Equals(value2)))
+                    if (N.NegateIfNeeded(current.Equals(value0) | current.Equals(value1) | current.Equals(value2)))
                     {
                         return i;
                     }
@@ -1494,7 +1488,7 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
                 do
                 {
                     current = Vector256.LoadUnsafe(ref currentSearchSpace);
-                    equals = negator.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current) | Vector256.Equals(values2, current));
+                    equals = N.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current) | Vector256.Equals(values2, current));
                     if (equals == Vector256<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector256<T>.Count);
@@ -1509,7 +1503,7 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
                 if ((uint)length % Vector256<T>.Count != 0)
                 {
                     current = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd);
-                    equals = negator.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current) | Vector256.Equals(values2, current));
+                    equals = N.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current) | Vector256.Equals(values2, current));
                     if (equals != Vector256<T>.Zero)
                     {
                         return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
@@ -1526,7 +1520,7 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
                 do
                 {
                     current = Vector128.LoadUnsafe(ref currentSearchSpace);
-                    equals = negator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current) | Vector128.Equals(values2, current));
+                    equals = N.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current) | Vector128.Equals(values2, current));
                     if (equals == Vector128<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128<T>.Count);
@@ -1541,7 +1535,7 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
                 if ((uint)length % Vector128<T>.Count != 0)
                 {
                     current = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd);
-                    equals = negator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current) | Vector128.Equals(values2, current));
+                    equals = N.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current) | Vector128.Equals(values2, current));
                     if (equals != Vector128<T>.Zero)
                     {
                         return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
@@ -1568,14 +1562,12 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
             Debug.Assert(length >= 0, "Expected non-negative length");
             Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types");
 
-            N negator = default;
-
             if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
             {
                 for (int i = 0; i < length; i++)
                 {
                     T current = Unsafe.Add(ref searchSpace, i);
-                    if (negator.NegateIfNeeded(current.Equals(value0) | current.Equals(value1) | current.Equals(value2) | current.Equals(value3)))
+                    if (N.NegateIfNeeded(current.Equals(value0) | current.Equals(value1) | current.Equals(value2) | current.Equals(value3)))
                     {
                         return i;
                     }
@@ -1591,7 +1583,7 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
                 do
                 {
                     current = Vector256.LoadUnsafe(ref currentSearchSpace);
-                    equals = negator.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current)
+                    equals = N.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current)
                         | Vector256.Equals(values2, current) | Vector256.Equals(values3, current));
                     if (equals == Vector256<T>.Zero)
                     {
@@ -1607,7 +1599,7 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
                 if ((uint)length % Vector256<T>.Count != 0)
                 {
                     current = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd);
-                    equals = negator.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current)
+                    equals = N.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current)
                         | Vector256.Equals(values2, current) | Vector256.Equals(values3, current));
                     if (equals != Vector256<T>.Zero)
                     {
@@ -1625,7 +1617,7 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
                 do
                 {
                     current = Vector128.LoadUnsafe(ref currentSearchSpace);
-                    equals = negator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current)
+                    equals = N.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current)
                         | Vector128.Equals(values2, current) | Vector128.Equals(values3, current));
                     if (equals == Vector128<T>.Zero)
                     {
@@ -1641,7 +1633,7 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
                 if ((uint)length % Vector128<T>.Count != 0)
                 {
                     current = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd);
-                    equals = negator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current)
+                    equals = N.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current)
                         | Vector128.Equals(values2, current) | Vector128.Equals(values3, current));
                     if (equals != Vector128<T>.Zero)
                     {
@@ -1761,13 +1753,11 @@ private static int LastIndexOfValueType<T, N>(ref T searchSpace, T value, int le
             Debug.Assert(length >= 0, "Expected non-negative length");
             Debug.Assert(value is byte or short or int or long, "Expected caller to normalize to one of these types");
 
-            N negator = default;
-
             if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
             {
                 for (int i = length - 1; i >= 0; i--)
                 {
-                    if (negator.NegateIfNeeded(Unsafe.Add(ref searchSpace, i).Equals(value)))
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, i).Equals(value)))
                     {
                         return i;
                     }
@@ -1781,7 +1771,7 @@ private static int LastIndexOfValueType<T, N>(ref T searchSpace, T value, int le
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
-                    equals = negator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref currentSearchSpace)));
+                    equals = N.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref currentSearchSpace)));
                     if (equals == Vector256<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<T>.Count);
@@ -1795,7 +1785,7 @@ private static int LastIndexOfValueType<T, N>(ref T searchSpace, T value, int le
                 // If any elements remain, process the first vector in the search space.
                 if ((uint)length % Vector256<T>.Count != 0)
                 {
-                    equals = negator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref searchSpace)));
+                    equals = N.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref searchSpace)));
                     if (equals != Vector256<T>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
@@ -1810,7 +1800,7 @@ private static int LastIndexOfValueType<T, N>(ref T searchSpace, T value, int le
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
-                    equals = negator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref currentSearchSpace)));
+                    equals = N.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref currentSearchSpace)));
                     if (equals == Vector128<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<T>.Count);
@@ -1824,7 +1814,7 @@ private static int LastIndexOfValueType<T, N>(ref T searchSpace, T value, int le
                 // If any elements remain, process the first vector in the search space.
                 if ((uint)length % Vector128<T>.Count != 0)
                 {
-                    equals = negator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref searchSpace)));
+                    equals = N.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref searchSpace)));
                     if (equals != Vector128<T>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
@@ -1851,14 +1841,12 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
             Debug.Assert(length >= 0, "Expected non-negative length");
             Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types");
 
-            N negator = default;
-
             if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
             {
                 for (int i = length - 1; i >= 0; i--)
                 {
                     T current = Unsafe.Add(ref searchSpace, i);
-                    if (negator.NegateIfNeeded(current.Equals(value0) || current.Equals(value1)))
+                    if (N.NegateIfNeeded(current.Equals(value0) || current.Equals(value1)))
                     {
                         return i;
                     }
@@ -1873,7 +1861,7 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
                 do
                 {
                     current = Vector256.LoadUnsafe(ref currentSearchSpace);
-                    equals = negator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1));
+                    equals = N.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1));
                     if (equals == Vector256<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<T>.Count);
@@ -1888,7 +1876,7 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
                 if ((uint)length % Vector256<T>.Count != 0)
                 {
                     current = Vector256.LoadUnsafe(ref searchSpace);
-                    equals = negator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1));
+                    equals = N.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1));
                     if (equals != Vector256<T>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
@@ -1904,7 +1892,7 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
                 do
                 {
                     current = Vector128.LoadUnsafe(ref currentSearchSpace);
-                    equals = negator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1));
+                    equals = N.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1));
                     if (equals == Vector128<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<T>.Count);
@@ -1919,7 +1907,7 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
                 if ((uint)length % Vector128<T>.Count != 0)
                 {
                     current = Vector128.LoadUnsafe(ref searchSpace);
-                    equals = negator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1));
+                    equals = N.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1));
                     if (equals != Vector128<T>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
@@ -1946,14 +1934,12 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
             Debug.Assert(length >= 0, "Expected non-negative length");
             Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types");
 
-            N negator = default;
-
             if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
             {
                 for (int i = length - 1; i >= 0; i--)
                 {
                     T current = Unsafe.Add(ref searchSpace, i);
-                    if (negator.NegateIfNeeded(current.Equals(value0) || current.Equals(value1) || current.Equals(value2)))
+                    if (N.NegateIfNeeded(current.Equals(value0) || current.Equals(value1) || current.Equals(value2)))
                     {
                         return i;
                     }
@@ -1968,7 +1954,7 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
                 do
                 {
                     current = Vector256.LoadUnsafe(ref currentSearchSpace);
-                    equals = negator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) | Vector256.Equals(current, values2));
+                    equals = N.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) | Vector256.Equals(current, values2));
                     if (equals == Vector256<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<T>.Count);
@@ -1983,7 +1969,7 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
                 if ((uint)length % Vector256<T>.Count != 0)
                 {
                     current = Vector256.LoadUnsafe(ref searchSpace);
-                    equals = negator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) | Vector256.Equals(current, values2));
+                    equals = N.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) | Vector256.Equals(current, values2));
                     if (equals != Vector256<T>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
@@ -1999,7 +1985,7 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
                 do
                 {
                     current = Vector128.LoadUnsafe(ref currentSearchSpace);
-                    equals = negator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) | Vector128.Equals(current, values2));
+                    equals = N.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) | Vector128.Equals(current, values2));
                     if (equals == Vector128<T>.Zero)
                     {
                         currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<T>.Count);
@@ -2014,7 +2000,7 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
                 if ((uint)length % Vector128<T>.Count != 0)
                 {
                     current = Vector128.LoadUnsafe(ref searchSpace);
-                    equals = negator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) | Vector128.Equals(current, values2));
+                    equals = N.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) | Vector128.Equals(current, values2));
                     if (equals != Vector128<T>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
@@ -2059,23 +2045,23 @@ private static int ComputeLastIndex<T>(ref T searchSpace, ref T current, Vector2
 
         private interface INegator<T> where T : struct, IEquatable<T>
         {
-            bool NegateIfNeeded(bool equals);
-            Vector128<T> NegateIfNeeded(Vector128<T> equals);
-            Vector256<T> NegateIfNeeded(Vector256<T> equals);
+            static abstract bool NegateIfNeeded(bool equals);
+            static abstract Vector128<T> NegateIfNeeded(Vector128<T> equals);
+            static abstract Vector256<T> NegateIfNeeded(Vector256<T> equals);
         }
 
         private readonly struct DontNegate<T> : INegator<T> where T : struct, IEquatable<T>
         {
-            public bool NegateIfNeeded(bool equals) => equals;
-            public Vector128<T> NegateIfNeeded(Vector128<T> equals) => equals;
-            public Vector256<T> NegateIfNeeded(Vector256<T> equals) => equals;
+            public static bool NegateIfNeeded(bool equals) => equals;
+            public static Vector128<T> NegateIfNeeded(Vector128<T> equals) => equals;
+            public static Vector256<T> NegateIfNeeded(Vector256<T> equals) => equals;
         }
 
         private readonly struct Negate<T> : INegator<T> where T : struct, IEquatable<T>
         {
-            public bool NegateIfNeeded(bool equals) => !equals;
-            public Vector128<T> NegateIfNeeded(Vector128<T> equals) => ~equals;
-            public Vector256<T> NegateIfNeeded(Vector256<T> equals) => ~equals;
+            public static bool NegateIfNeeded(bool equals) => !equals;
+            public static Vector128<T> NegateIfNeeded(Vector128<T> equals) => ~equals;
+            public static Vector256<T> NegateIfNeeded(Vector256<T> equals) => ~equals;
         }
     }
 }
diff --git a/src/libraries/System.Private.CoreLib/src/System/String.cs b/src/libraries/System.Private.CoreLib/src/System/String.cs
index 765571c03592af..8cc627f38084f5 100644
--- a/src/libraries/System.Private.CoreLib/src/System/String.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/String.cs
@@ -593,39 +593,9 @@ public StringRuneEnumerator EnumerateRunes()
             return new StringRuneEnumerator(this);
         }
 
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        internal static unsafe int wcslen(char* ptr)
-        {
-            // IndexOf processes memory in aligned chunks, and thus it won't crash even if it accesses memory beyond the null terminator.
-            // This IndexOf behavior is an implementation detail of the runtime and callers outside System.Private.CoreLib must not depend on it.
-            int length = SpanHelpers.IndexOfNullCharacter(ref *ptr);
-            if (length < 0)
-            {
-                ThrowMustBeNullTerminatedString();
-            }
+        internal static unsafe int wcslen(char* ptr) => SpanHelpers.IndexOfNullCharacter(ref *ptr);
 
-            return length;
-        }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        internal static unsafe int strlen(byte* ptr)
-        {
-            // IndexOf processes memory in aligned chunks, and thus it won't crash even if it accesses memory beyond the null terminator.
-            // This IndexOf behavior is an implementation detail of the runtime and callers outside System.Private.CoreLib must not depend on it.
-            int length = SpanHelpers.IndexOfNullByte(ref *ptr);
-            if (length < 0)
-            {
-                ThrowMustBeNullTerminatedString();
-            }
-
-            return length;
-        }
-
-        [DoesNotReturn]
-        private static void ThrowMustBeNullTerminatedString()
-        {
-            throw new ArgumentException(SR.Arg_MustBeNullTerminatedString);
-        }
+        internal static unsafe int strlen(byte* ptr) => SpanHelpers.IndexOfNullByte(ref *ptr);
 
         //
         // IConvertible implementation

From b305c54ca153e8950874b45fa89478b79702b249 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Sat, 13 Aug 2022 14:42:57 +0200
Subject: [PATCH 24/34] perform manual loop unrolling in order to avoid perf
 regression for small inputs

use INumber instead of IEquatable to generate better codegen when comparing multiple values
---
 .../src/System/SpanHelpers.T.cs               | 367 +++++++++++++++---
 1 file changed, 308 insertions(+), 59 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
index 496b52e69dc035..792cdf18dba67f 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
@@ -1179,19 +1179,56 @@ public static int SequenceCompareTo<T>(ref T first, int firstLength, ref T secon
         }
 
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        internal static bool ContainsValueType<T>(ref T searchSpace, T value, int length) where T : struct, IEquatable<T>
+        internal static bool ContainsValueType<T>(ref T searchSpace, T value, int length) where T : struct, INumber<T>
         {
             Debug.Assert(length >= 0, "Expected non-negative length");
             Debug.Assert(value is byte or short or int or long, "Expected caller to normalize to one of these types");
 
             if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
             {
-                for (int i = 0; i < length; i++)
+                nuint offset = 0;
+
+                while (length >= 8)
                 {
-                    if (Unsafe.Add(ref searchSpace, i).Equals(value))
+                    length -= 8;
+
+                    if (Unsafe.Add(ref searchSpace, offset) == value
+                     || Unsafe.Add(ref searchSpace, offset + 1) == value
+                     || Unsafe.Add(ref searchSpace, offset + 2) == value
+                     || Unsafe.Add(ref searchSpace, offset + 3) == value
+                     || Unsafe.Add(ref searchSpace, offset + 4) == value
+                     || Unsafe.Add(ref searchSpace, offset + 5) == value
+                     || Unsafe.Add(ref searchSpace, offset + 6) == value
+                     || Unsafe.Add(ref searchSpace, offset + 7) == value)
                     {
                         return true;
                     }
+
+                    offset += 8;
+                }
+
+                if (length >= 4)
+                {
+                    length -= 4;
+
+                    if (Unsafe.Add(ref searchSpace, offset) == value
+                     || Unsafe.Add(ref searchSpace, offset + 1) == value
+                     || Unsafe.Add(ref searchSpace, offset + 2) == value
+                     || Unsafe.Add(ref searchSpace, offset + 3) == value)
+                    {
+                        return true;
+                    }
+
+                    offset += 4;
+                }
+
+                while (length > 0)
+                {
+                    length -= 1;
+
+                    if (Unsafe.Add(ref searchSpace, offset) == value) return true;
+
+                    offset += 1;
                 }
             }
             else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
@@ -1263,16 +1300,16 @@ internal static int IndexOfChar(ref char searchSpace, char value, int length)
             => IndexOfValueType(ref Unsafe.As<char, short>(ref searchSpace), (short)value, length);
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        internal static int IndexOfValueType<T>(ref T searchSpace, T value, int length) where T : struct, IEquatable<T>
+        internal static int IndexOfValueType<T>(ref T searchSpace, T value, int length) where T : struct, INumber<T>
             => IndexOfValueType<T, DontNegate<T>>(ref searchSpace, value, length);
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        internal static int IndexOfAnyExceptValueType<T>(ref T searchSpace, T value, int length) where T : struct, IEquatable<T>
+        internal static int IndexOfAnyExceptValueType<T>(ref T searchSpace, T value, int length) where T : struct, INumber<T>
             => IndexOfValueType<T, Negate<T>>(ref searchSpace, value, length);
 
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
         private static int IndexOfValueType<T, N>(ref T searchSpace, T value, int length)
-            where T : struct, IEquatable<T>
+            where T : struct, INumber<T>
             where N : struct, INegator<T>
         {
             Debug.Assert(length >= 0, "Expected non-negative length");
@@ -1280,12 +1317,43 @@ private static int IndexOfValueType<T, N>(ref T searchSpace, T value, int length
 
             if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
             {
-                for (int i = 0; i < length; i++)
+                nuint offset = 0;
+
+                while (length >= 8)
                 {
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, i).Equals(value)))
-                    {
-                        return i;
-                    }
+                    length -= 8;
+
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 0) == value)) return (int)offset;
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 1) == value)) return (int)offset + 1;
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 2) == value)) return (int)offset + 2;
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 3) == value)) return (int)offset + 3;
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 4) == value)) return (int)offset + 4;
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 5) == value)) return (int)offset + 5;
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 6) == value)) return (int)offset + 6;
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 7) == value)) return (int)offset + 7;
+
+                    offset += 8;
+                }
+
+                if (length >= 4)
+                {
+                    length -= 4;
+
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 0) == value)) return (int)offset;
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 1) == value)) return (int)offset + 1;
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 2) == value)) return (int)offset + 2;
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 3) == value)) return (int)offset + 3;
+
+                    offset += 4;
+                }
+
+                while (length > 0)
+                {
+                    length -= 1;
+
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset).Equals(value))) return (int)offset;
+
+                    offset += 1;
                 }
             }
             else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
@@ -1357,16 +1425,17 @@ internal static int IndexOfAnyChar(ref char searchSpace, char value0, char value
             => IndexOfAnyValueType(ref Unsafe.As<char, short>(ref searchSpace), (short)value0, (short)value1, length);
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        internal static int IndexOfAnyValueType<T>(ref T searchSpace, T value0, T value1, int length) where T : struct, IEquatable<T>
+        internal static int IndexOfAnyValueType<T>(ref T searchSpace, T value0, T value1, int length) where T : struct, INumber<T>
             => IndexOfAnyValueType<T, DontNegate<T>>(ref searchSpace, value0, value1, length);
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        internal static int IndexOfAnyExceptValueType<T>(ref T searchSpace, T value0, T value1, int length) where T : struct, IEquatable<T>
+        internal static int IndexOfAnyExceptValueType<T>(ref T searchSpace, T value0, T value1, int length) where T : struct, INumber<T>
             => IndexOfAnyValueType<T, Negate<T>>(ref searchSpace, value0, value1, length);
 
+        // having INumber<T> constraint here allows to use == operator and get better perf compared to .Equals
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
         private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T value1, int length)
-            where T : struct, IEquatable<T>
+            where T : struct, INumber<T>
             where N : struct, INegator<T>
         {
             Debug.Assert(length >= 0, "Expected non-negative length");
@@ -1374,14 +1443,63 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
 
             if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
             {
-                for (int i = 0; i < length; i++)
+                nuint offset = 0;
+                T lookUp;
+
+                if (typeof(T) == typeof(byte)) // this optimization is beneficial only to byte
                 {
-                    T current = Unsafe.Add(ref searchSpace, i);
-                    if (N.NegateIfNeeded(current.Equals(value0) | current.Equals(value1)))
+                    while (length >= 8)
                     {
-                        return i;
+                        length -= 8;
+
+                        ref T current = ref Unsafe.Add(ref searchSpace, offset);
+                        lookUp = current;
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset;
+                        lookUp = Unsafe.Add(ref current, 1);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 1;
+                        lookUp = Unsafe.Add(ref current, 2);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 2;
+                        lookUp = Unsafe.Add(ref current, 3);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 3;
+                        lookUp = Unsafe.Add(ref current, 4);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 4;
+                        lookUp = Unsafe.Add(ref current, 5);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 5;
+                        lookUp = Unsafe.Add(ref current, 6);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 6;
+                        lookUp = Unsafe.Add(ref current, 7);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 7;
+
+                        offset += 8;
                     }
                 }
+
+                while (length >= 4)
+                {
+                    length -= 4;
+
+                    ref T current = ref Unsafe.Add(ref searchSpace, offset);
+                    lookUp = current;
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset;
+                    lookUp = Unsafe.Add(ref current, 1);
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 1;
+                    lookUp = Unsafe.Add(ref current, 2);
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 2;
+                    lookUp = Unsafe.Add(ref current, 3);
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 3;
+
+                    offset += 4;
+                }
+
+                while (length > 0)
+                {
+                    length -= 1;
+
+                    lookUp = Unsafe.Add(ref searchSpace, offset);
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset;
+
+                    offset += 1;
+                }
             }
             else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
             {
@@ -1452,16 +1570,16 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        internal static int IndexOfAnyValueType<T>(ref T searchSpace, T value0, T value1, T value2, int length) where T : struct, IEquatable<T>
+        internal static int IndexOfAnyValueType<T>(ref T searchSpace, T value0, T value1, T value2, int length) where T : struct, INumber<T>
             => IndexOfAnyValueType<T, DontNegate<T>>(ref searchSpace, value0, value1, value2, length);
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        internal static int IndexOfAnyExceptValueType<T>(ref T searchSpace, T value0, T value1, T value2, int length) where T : struct, IEquatable<T>
+        internal static int IndexOfAnyExceptValueType<T>(ref T searchSpace, T value0, T value1, T value2, int length) where T : struct, INumber<T>
             => IndexOfAnyValueType<T, Negate<T>>(ref searchSpace, value0, value1, value2, length);
 
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
         private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T value1, T value2, int length)
-            where T : struct, IEquatable<T>
+            where T : struct, INumber<T>
             where N : struct, INegator<T>
         {
             Debug.Assert(length >= 0, "Expected non-negative length");
@@ -1469,14 +1587,63 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
 
             if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
             {
-                for (int i = 0; i < length; i++)
+                nuint offset = 0;
+                T lookUp;
+
+                if (typeof(T) == typeof(byte)) // this optimization is beneficial only to byte
                 {
-                    T current = Unsafe.Add(ref searchSpace, i);
-                    if (N.NegateIfNeeded(current.Equals(value0) | current.Equals(value1) | current.Equals(value2)))
+                    while (length >= 8)
                     {
-                        return i;
+                        length -= 8;
+
+                        ref T current = ref Unsafe.Add(ref searchSpace, offset);
+                        lookUp = current;
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset;
+                        lookUp = Unsafe.Add(ref current, 1);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 1;
+                        lookUp = Unsafe.Add(ref current, 2);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 2;
+                        lookUp = Unsafe.Add(ref current, 3);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 3;
+                        lookUp = Unsafe.Add(ref current, 4);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 4;
+                        lookUp = Unsafe.Add(ref current, 5);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 5;
+                        lookUp = Unsafe.Add(ref current, 6);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 6;
+                        lookUp = Unsafe.Add(ref current, 7);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 7;
+
+                        offset += 8;
                     }
                 }
+
+                while (length >= 4)
+                {
+                    length -= 4;
+
+                    ref T current = ref Unsafe.Add(ref searchSpace, offset);
+                    lookUp = current;
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset;
+                    lookUp = Unsafe.Add(ref current, 1);
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 1;
+                    lookUp = Unsafe.Add(ref current, 2);
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 2;
+                    lookUp = Unsafe.Add(ref current, 3);
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 3;
+
+                    offset += 4;
+                }
+
+                while (length > 0)
+                {
+                    length -= 1;
+
+                    lookUp = Unsafe.Add(ref searchSpace, offset);
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset;
+
+                    offset += 1;
+                }
             }
             else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
             {
@@ -1547,16 +1714,16 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        internal static int IndexOfAnyValueType<T>(ref T searchSpace, T value0, T value1, T value2, T value3, int length) where T : struct, IEquatable<T>
+        internal static int IndexOfAnyValueType<T>(ref T searchSpace, T value0, T value1, T value2, T value3, int length) where T : struct, INumber<T>
             => IndexOfAnyValueType<T, DontNegate<T>>(ref searchSpace, value0, value1, value2, value3, length);
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        internal static int IndexOfAnyExceptValueType<T>(ref T searchSpace, T value0, T value1, T value2, T value3, int length) where T : struct, IEquatable<T>
+        internal static int IndexOfAnyExceptValueType<T>(ref T searchSpace, T value0, T value1, T value2, T value3, int length) where T : struct, INumber<T>
             => IndexOfAnyValueType<T, Negate<T>>(ref searchSpace, value0, value1, value2, value3, length);
 
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
         private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T value1, T value2, T value3, int length)
-            where T : struct, IEquatable<T>
+            where T : struct, INumber<T>
             where N : struct, INegator<T>
         {
             Debug.Assert(length >= 0, "Expected non-negative length");
@@ -1564,13 +1731,34 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
 
             if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
             {
-                for (int i = 0; i < length; i++)
+                nuint offset = 0;
+                T lookUp;
+
+                while (length >= 4)
                 {
-                    T current = Unsafe.Add(ref searchSpace, i);
-                    if (N.NegateIfNeeded(current.Equals(value0) | current.Equals(value1) | current.Equals(value2) | current.Equals(value3)))
-                    {
-                        return i;
-                    }
+                    length -= 4;
+
+                    ref T current = ref Unsafe.Add(ref searchSpace, offset);
+                    lookUp = current;
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset;
+                    lookUp = Unsafe.Add(ref current, 1);
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset + 1;
+                    lookUp = Unsafe.Add(ref current, 2);
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset + 2;
+                    lookUp = Unsafe.Add(ref current, 3);
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset + 3;
+
+                    offset += 4;
+                }
+
+                while (length > 0)
+                {
+                    length -= 1;
+
+                    lookUp = Unsafe.Add(ref searchSpace, offset);
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset;
+
+                    offset += 1;
                 }
             }
             else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
@@ -1647,20 +1835,41 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
 
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
         internal static int IndexOfAnyValueType<T>(ref T searchSpace, T value0, T value1, T value2, T value3, T value4, int length)
-            where T : struct, IEquatable<T>
+            where T : struct, INumber<T>
         {
             Debug.Assert(length >= 0, "Expected non-negative length");
             Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types");
 
             if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
             {
-                for (int i = 0; i < length; i++)
+                nuint offset = 0;
+                T lookUp;
+
+                while (length >= 4)
                 {
-                    T current = Unsafe.Add(ref searchSpace, i);
-                    if (current.Equals(value0) | current.Equals(value1) | current.Equals(value2) | current.Equals(value3) | current.Equals(value4))
-                    {
-                        return i;
-                    }
+                    length -= 4;
+
+                    ref T current = ref Unsafe.Add(ref searchSpace, offset);
+                    lookUp = current;
+                    if (lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3 || lookUp == value4) return (int)offset;
+                    lookUp = Unsafe.Add(ref current, 1);
+                    if (lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3 || lookUp == value4) return (int)offset + 1;
+                    lookUp = Unsafe.Add(ref current, 2);
+                    if (lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3 || lookUp == value4) return (int)offset + 2;
+                    lookUp = Unsafe.Add(ref current, 3);
+                    if (lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3 || lookUp == value4) return (int)offset + 3;
+
+                    offset += 4;
+                }
+
+                while (length > 0)
+                {
+                    length -= 1;
+
+                    lookUp = Unsafe.Add(ref searchSpace, offset);
+                    if (lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3 || lookUp == value4) return (int)offset;
+
+                    offset += 1;
                 }
             }
             else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
@@ -1738,16 +1947,16 @@ internal static int IndexOfAnyValueType<T>(ref T searchSpace, T value0, T value1
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        internal static int LastIndexOfValueType<T>(ref T searchSpace, T value, int length) where T : struct, IEquatable<T>
+        internal static int LastIndexOfValueType<T>(ref T searchSpace, T value, int length) where T : struct, INumber<T>
             => LastIndexOfValueType<T, DontNegate<T>>(ref searchSpace, value, length);
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        internal static int LastIndexOfAnyExceptValueType<T>(ref T searchSpace, T value, int length) where T : struct, IEquatable<T>
+        internal static int LastIndexOfAnyExceptValueType<T>(ref T searchSpace, T value, int length) where T : struct, INumber<T>
             => LastIndexOfValueType<T, Negate<T>>(ref searchSpace, value, length);
 
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
         private static int LastIndexOfValueType<T, N>(ref T searchSpace, T value, int length)
-            where T : struct, IEquatable<T>
+            where T : struct, INumber<T>
             where N : struct, INegator<T>
         {
             Debug.Assert(length >= 0, "Expected non-negative length");
@@ -1755,6 +1964,46 @@ private static int LastIndexOfValueType<T, N>(ref T searchSpace, T value, int le
 
             if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
             {
+                nuint offset = (nuint)length - 1;
+
+                while (length >= 8)
+                {
+                    length -= 8;
+
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 0) == value)) return (int)offset - 0;
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 1) == value)) return (int)offset - 1;
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 2) == value)) return (int)offset - 2;
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 3) == value)) return (int)offset - 3;
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 4) == value)) return (int)offset - 4;
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 5) == value)) return (int)offset - 5;
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 6) == value)) return (int)offset - 6;
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 7) == value)) return (int)offset - 7;
+
+                    offset -= 8;
+                }
+
+                if (length >= 4)
+                {
+                    length -= 4;
+
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset) == value)) return (int)offset;
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 1) == value)) return (int)offset - 1;
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 2) == value)) return (int)offset - 2;
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 3) == value)) return (int)offset - 3;
+
+                    offset -= 4;
+                }
+
+                while (length > 0)
+                {
+                    length -= 1;
+
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset).Equals(value))) return (int)offset;
+
+                    offset -= 1;
+                }
+
+
                 for (int i = length - 1; i >= 0; i--)
                 {
                     if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, i).Equals(value)))
@@ -1826,16 +2075,16 @@ private static int LastIndexOfValueType<T, N>(ref T searchSpace, T value, int le
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        internal static int LastIndexOfAnyValueType<T>(ref T searchSpace, T value0, T value1, int length) where T : struct, IEquatable<T>
+        internal static int LastIndexOfAnyValueType<T>(ref T searchSpace, T value0, T value1, int length) where T : struct, INumber<T>
             => LastIndexOfAnyValueType<T, DontNegate<T>>(ref searchSpace, value0, value1, length);
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        internal static int LastIndexOfAnyExceptValueType<T>(ref T searchSpace, T value0, T value1, int length) where T : struct, IEquatable<T>
+        internal static int LastIndexOfAnyExceptValueType<T>(ref T searchSpace, T value0, T value1, int length) where T : struct, INumber<T>
             => LastIndexOfAnyValueType<T, Negate<T>>(ref searchSpace, value0, value1, length);
 
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
         private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T value1, int length)
-            where T : struct, IEquatable<T>
+            where T : struct, INumber<T>
             where N : struct, INegator<T>
         {
             Debug.Assert(length >= 0, "Expected non-negative length");
@@ -1846,7 +2095,7 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
                 for (int i = length - 1; i >= 0; i--)
                 {
                     T current = Unsafe.Add(ref searchSpace, i);
-                    if (N.NegateIfNeeded(current.Equals(value0) || current.Equals(value1)))
+                    if (N.NegateIfNeeded(current == value0 || current == value1))
                     {
                         return i;
                     }
@@ -1919,16 +2168,16 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        internal static int LastIndexOfAnyValueType<T>(ref T searchSpace, T value0, T value1, T value2, int length) where T : struct, IEquatable<T>
+        internal static int LastIndexOfAnyValueType<T>(ref T searchSpace, T value0, T value1, T value2, int length) where T : struct, INumber<T>
             => LastIndexOfAnyValueType<T, DontNegate<T>>(ref searchSpace, value0, value1, value2, length);
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        internal static int LastIndexOfAnyExceptValueType<T>(ref T searchSpace, T value0, T value1, T value2, int length) where T : struct, IEquatable<T>
+        internal static int LastIndexOfAnyExceptValueType<T>(ref T searchSpace, T value0, T value1, T value2, int length) where T : struct, INumber<T>
             => LastIndexOfAnyValueType<T, Negate<T>>(ref searchSpace, value0, value1, value2, length);
 
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
         private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T value1, T value2, int length)
-            where T : struct, IEquatable<T>
+            where T : struct, INumber<T>
             where N : struct, INegator<T>
         {
             Debug.Assert(length >= 0, "Expected non-negative length");
@@ -1939,7 +2188,7 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
                 for (int i = length - 1; i >= 0; i--)
                 {
                     T current = Unsafe.Add(ref searchSpace, i);
-                    if (N.NegateIfNeeded(current.Equals(value0) || current.Equals(value1) || current.Equals(value2)))
+                    if (N.NegateIfNeeded(current == value0 || current == value1 || current == value2))
                     {
                         return i;
                     }
@@ -2012,7 +2261,7 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int ComputeFirstIndex<T>(ref T searchSpace, ref T current, Vector128<T> equals) where T : struct, IEquatable<T>
+        private static int ComputeFirstIndex<T>(ref T searchSpace, ref T current, Vector128<T> equals) where T : struct
         {
             uint notEqualsElements = equals.ExtractMostSignificantBits();
             int index = BitOperations.TrailingZeroCount(notEqualsElements);
@@ -2020,7 +2269,7 @@ private static int ComputeFirstIndex<T>(ref T searchSpace, ref T current, Vector
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int ComputeFirstIndex<T>(ref T searchSpace, ref T current, Vector256<T> equals) where T : struct, IEquatable<T>
+        private static int ComputeFirstIndex<T>(ref T searchSpace, ref T current, Vector256<T> equals) where T : struct
         {
             uint notEqualsElements = equals.ExtractMostSignificantBits();
             int index = BitOperations.TrailingZeroCount(notEqualsElements);
@@ -2028,7 +2277,7 @@ private static int ComputeFirstIndex<T>(ref T searchSpace, ref T current, Vector
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int ComputeLastIndex<T>(ref T searchSpace, ref T current, Vector128<T> equals) where T : struct, IEquatable<T>
+        private static int ComputeLastIndex<T>(ref T searchSpace, ref T current, Vector128<T> equals) where T : struct
         {
             uint notEqualsElements = equals.ExtractMostSignificantBits();
             int index = 31 - BitOperations.LeadingZeroCount(notEqualsElements); // 31 = 32 (bits in Int32) - 1 (indexing from zero)
@@ -2036,28 +2285,28 @@ private static int ComputeLastIndex<T>(ref T searchSpace, ref T current, Vector1
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static int ComputeLastIndex<T>(ref T searchSpace, ref T current, Vector256<T> equals) where T : struct, IEquatable<T>
+        private static int ComputeLastIndex<T>(ref T searchSpace, ref T current, Vector256<T> equals) where T : struct
         {
             uint notEqualsElements = equals.ExtractMostSignificantBits();
             int index = 31 - BitOperations.LeadingZeroCount(notEqualsElements); // 31 = 32 (bits in Int32) - 1 (indexing from zero)
             return (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / Unsafe.SizeOf<T>()) + index;
         }
 
-        private interface INegator<T> where T : struct, IEquatable<T>
+        private interface INegator<T> where T : struct
         {
             static abstract bool NegateIfNeeded(bool equals);
             static abstract Vector128<T> NegateIfNeeded(Vector128<T> equals);
             static abstract Vector256<T> NegateIfNeeded(Vector256<T> equals);
         }
 
-        private readonly struct DontNegate<T> : INegator<T> where T : struct, IEquatable<T>
+        private readonly struct DontNegate<T> : INegator<T> where T : struct
         {
             public static bool NegateIfNeeded(bool equals) => equals;
             public static Vector128<T> NegateIfNeeded(Vector128<T> equals) => equals;
             public static Vector256<T> NegateIfNeeded(Vector256<T> equals) => equals;
         }
 
-        private readonly struct Negate<T> : INegator<T> where T : struct, IEquatable<T>
+        private readonly struct Negate<T> : INegator<T> where T : struct
         {
             public static bool NegateIfNeeded(bool equals) => !equals;
             public static Vector128<T> NegateIfNeeded(Vector128<T> equals) => ~equals;

From 846e4fc0bcee76c86240f391e565bf48038af869 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Sun, 14 Aug 2022 11:01:16 +0200
Subject: [PATCH 25/34] restore Vector<T> code path as Vector128 is not
 accelerated by Mono x64 (it is on arm64)

---
 .../src/System/SpanHelpers.Byte.cs            | 84 +++++++++++++++++++
 .../src/System/SpanHelpers.Char.cs            | 71 ++++++++++++++++
 2 files changed, 155 insertions(+)

diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
index 7f76563a2f7a2c..a66042e0302972 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
@@ -517,6 +517,32 @@ internal static unsafe int IndexOfNullByte(ref byte searchSpace)
                     }
                 }
             }
+            else if (Vector.IsHardwareAccelerated)
+            {
+                if (offset < (nuint)(uint)length)
+                {
+                    lengthToExamine = GetByteVectorSpanLength(offset, length);
+
+                    while (lengthToExamine > offset)
+                    {
+                        var matches = Vector.Equals(Vector<byte>.Zero, LoadVector(ref searchSpace, offset));
+                        if (Vector<byte>.Zero.Equals(matches))
+                        {
+                            offset += (nuint)Vector<byte>.Count;
+                            continue;
+                        }
+
+                        // Find offset of first match and add to current offset
+                        return (int)offset + LocateFirstFoundByte(matches);
+                    }
+
+                    if (offset < (nuint)(uint)length)
+                    {
+                        lengthToExamine = ((nuint)(uint)length - offset);
+                        goto SequentialScan;
+                    }
+                }
+            }
 
             ThrowMustBeNullTerminatedString();
         Found: // Workaround for https://github.com/dotnet/runtime/issues/8795
@@ -740,6 +766,27 @@ public static unsafe bool SequenceEqual(ref byte first, ref byte second, nuint l
             return false;
         }
 
+        // Vector sub-search adapted from https://github.com/aspnet/KestrelHttpServer/pull/1138
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static int LocateFirstFoundByte(Vector<byte> match)
+        {
+            var vector64 = Vector.AsVectorUInt64(match);
+            ulong candidate = 0;
+            int i = 0;
+            // Pattern unrolled by jit https://github.com/dotnet/coreclr/pull/8001
+            for (; i < Vector<ulong>.Count; i++)
+            {
+                candidate = vector64[i];
+                if (candidate != 0)
+                {
+                    break;
+                }
+            }
+
+            // Single LEA instruction with jitted const (using function result)
+            return i * 8 + LocateFirstFoundByte(candidate);
+        }
+
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
         public static unsafe int SequenceCompareTo(ref byte first, int firstLength, ref byte second, int secondLength)
         {
@@ -1015,6 +1062,39 @@ public static nuint CommonPrefixLength(ref byte first, ref byte second, nuint le
             return i + uint.TrailingZeroCount(mask);
         }
 
+        // Vector sub-search adapted from https://github.com/aspnet/KestrelHttpServer/pull/1138
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static int LocateLastFoundByte(Vector<byte> match)
+        {
+            var vector64 = Vector.AsVectorUInt64(match);
+            ulong candidate = 0;
+            int i = Vector<ulong>.Count - 1;
+
+            // This pattern is only unrolled by the Jit if the limit is Vector<T>.Count
+            // As such, we need a dummy iteration variable for that condition to be satisfied
+            for (int j = 0; j < Vector<ulong>.Count; j++)
+            {
+                candidate = vector64[i];
+                if (candidate != 0)
+                {
+                    break;
+                }
+
+                i--;
+            }
+
+            // Single LEA instruction with jitted const (using function result)
+            return i * 8 + LocateLastFoundByte(candidate);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static int LocateFirstFoundByte(ulong match)
+            => BitOperations.TrailingZeroCount(match) >> 3;
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static int LocateLastFoundByte(ulong match)
+            => BitOperations.Log2(match) >> 3;
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static ushort LoadUShort(ref byte start)
             => Unsafe.ReadUnaligned<ushort>(ref start);
@@ -1047,6 +1127,10 @@ private static Vector128<byte> LoadVector128(ref byte start, nuint offset)
         private static Vector256<byte> LoadVector256(ref byte start, nuint offset)
             => Unsafe.ReadUnaligned<Vector256<byte>>(ref Unsafe.AddByteOffset(ref start, offset));
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static nuint GetByteVectorSpanLength(nuint offset, int length)
+            => (nuint)(uint)((length - (int)offset) & ~(Vector<byte>.Count - 1));
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static nuint GetByteVector128SpanLength(nuint offset, int length)
             => (nuint)(uint)((length - (int)offset) & ~(Vector128<byte>.Count - 1));
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
index 1a1becda8e433a..8a44db8cc3aed8 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
@@ -615,6 +615,40 @@ public static unsafe int IndexOfNullCharacter(ref char searchSpace)
                     }
                 }
             }
+            else if (Vector.IsHardwareAccelerated)
+            {
+                if (offset < length)
+                {
+                    Debug.Assert(length - offset >= Vector<ushort>.Count);
+
+                    lengthToExamine = GetCharVectorSpanLength(offset, length);
+
+                    if (lengthToExamine > 0)
+                    {
+                        do
+                        {
+                            Debug.Assert(lengthToExamine >= Vector<ushort>.Count);
+
+                            var matches = Vector.Equals(Vector<ushort>.Zero, LoadVector(ref searchSpace, offset));
+                            if (Vector<ushort>.Zero.Equals(matches))
+                            {
+                                offset += Vector<ushort>.Count;
+                                lengthToExamine -= Vector<ushort>.Count;
+                                continue;
+                            }
+
+                            // Find offset of first match
+                            return (int)(offset + LocateFirstFoundChar(matches));
+                        } while (lengthToExamine > 0);
+                    }
+
+                    if (offset < length)
+                    {
+                        lengthToExamine = length - offset;
+                        goto SequentialScan;
+                    }
+                }
+            }
 
             ThrowMustBeNullTerminatedString();
         Found3:
@@ -627,6 +661,43 @@ public static unsafe int IndexOfNullCharacter(ref char searchSpace)
             return (int)(offset);
         }
 
+        // Vector sub-search adapted from https://github.com/aspnet/KestrelHttpServer/pull/1138
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static int LocateFirstFoundChar(Vector<ushort> match)
+        {
+            var vector64 = Vector.AsVectorUInt64(match);
+            ulong candidate = 0;
+            int i = 0;
+            // Pattern unrolled by jit https://github.com/dotnet/coreclr/pull/8001
+            for (; i < Vector<ulong>.Count; i++)
+            {
+                candidate = vector64[i];
+                if (candidate != 0)
+                {
+                    break;
+                }
+            }
+
+            // Single LEA instruction with jitted const (using function result)
+            return i * 4 + LocateFirstFoundChar(candidate);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static int LocateFirstFoundChar(ulong match)
+            => BitOperations.TrailingZeroCount(match) >> 4;
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static Vector<ushort> LoadVector(ref char start, nint offset)
+            => Unsafe.ReadUnaligned<Vector<ushort>>(ref Unsafe.As<char, byte>(ref Unsafe.Add(ref start, offset)));
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static Vector<ushort> LoadVector(ref char start, nuint offset)
+            => Unsafe.ReadUnaligned<Vector<ushort>>(ref Unsafe.As<char, byte>(ref Unsafe.Add(ref start, (nint)offset)));
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static nint GetCharVectorSpanLength(nint offset, nint length)
+            => (length - offset) & ~(Vector<ushort>.Count - 1);
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static nint GetCharVector128SpanLength(nint offset, nint length)
             => (length - offset) & ~(Vector128<ushort>.Count - 1);

From 8a74f28712293f670b688e8bf4c291ea22a72c55 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Sun, 14 Aug 2022 12:04:36 +0200
Subject: [PATCH 26/34] remove redundant for loop that was left here by mistake

---
 .../System.Private.CoreLib/src/System/SpanHelpers.T.cs   | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
index 792cdf18dba67f..9697e9504c5837 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
@@ -2002,15 +2002,6 @@ private static int LastIndexOfValueType<T, N>(ref T searchSpace, T value, int le
 
                     offset -= 1;
                 }
-
-
-                for (int i = length - 1; i >= 0; i--)
-                {
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, i).Equals(value)))
-                    {
-                        return i;
-                    }
-                }
             }
             else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
             {

From fab3143063d76aaea8078c7ca36baa99fea02b73 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Mon, 15 Aug 2022 17:43:01 +0200
Subject: [PATCH 27/34] apply manual loop unrolling to LastIndexOfAny methods

---
 .../src/System/SpanHelpers.T.cs               | 120 ++++++++++++++++--
 1 file changed, 109 insertions(+), 11 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
index 9697e9504c5837..d879d3236175d5 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
@@ -1323,7 +1323,7 @@ private static int IndexOfValueType<T, N>(ref T searchSpace, T value, int length
                 {
                     length -= 8;
 
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 0) == value)) return (int)offset;
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset) == value)) return (int)offset;
                     if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 1) == value)) return (int)offset + 1;
                     if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 2) == value)) return (int)offset + 2;
                     if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 3) == value)) return (int)offset + 3;
@@ -1339,7 +1339,7 @@ private static int IndexOfValueType<T, N>(ref T searchSpace, T value, int length
                 {
                     length -= 4;
 
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 0) == value)) return (int)offset;
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset) == value)) return (int)offset;
                     if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 1) == value)) return (int)offset + 1;
                     if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 2) == value)) return (int)offset + 2;
                     if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 3) == value)) return (int)offset + 3;
@@ -1970,7 +1970,7 @@ private static int LastIndexOfValueType<T, N>(ref T searchSpace, T value, int le
                 {
                     length -= 8;
 
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 0) == value)) return (int)offset - 0;
+                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset) == value)) return (int)offset;
                     if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 1) == value)) return (int)offset - 1;
                     if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 2) == value)) return (int)offset - 2;
                     if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 3) == value)) return (int)offset - 3;
@@ -2083,14 +2083,63 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
 
             if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
             {
-                for (int i = length - 1; i >= 0; i--)
+                nuint offset = (nuint)length - 1;
+                T lookUp;
+
+                if (typeof(T) == typeof(byte)) // this optimization is beneficial only to byte
                 {
-                    T current = Unsafe.Add(ref searchSpace, i);
-                    if (N.NegateIfNeeded(current == value0 || current == value1))
+                    while (length >= 8)
                     {
-                        return i;
+                        length -= 8;
+
+                        ref T current = ref Unsafe.Add(ref searchSpace, offset);
+                        lookUp = current;
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset;
+                        lookUp = Unsafe.Add(ref current, -1);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 1;
+                        lookUp = Unsafe.Add(ref current, -2);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 2;
+                        lookUp = Unsafe.Add(ref current, -3);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 3;
+                        lookUp = Unsafe.Add(ref current, -4);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 4;
+                        lookUp = Unsafe.Add(ref current, -5);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 5;
+                        lookUp = Unsafe.Add(ref current, -6);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 6;
+                        lookUp = Unsafe.Add(ref current, -7);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 7;
+
+                        offset -= 8;
                     }
                 }
+
+                while (length >= 4)
+                {
+                    length -= 4;
+
+                    ref T current = ref Unsafe.Add(ref searchSpace, offset);
+                    lookUp = current;
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset;
+                    lookUp = Unsafe.Add(ref current, -1);
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 1;
+                    lookUp = Unsafe.Add(ref current, -2);
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 2;
+                    lookUp = Unsafe.Add(ref current, -3);
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 3;
+
+                    offset -= 4;
+                }
+
+                while (length > 0)
+                {
+                    length -= 1;
+
+                    lookUp = Unsafe.Add(ref searchSpace, offset);
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset;
+
+                    offset -= 1;
+                }
             }
             else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
             {
@@ -2176,14 +2225,63 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
 
             if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
             {
-                for (int i = length - 1; i >= 0; i--)
+                nuint offset = (nuint)length - 1;
+                T lookUp;
+
+                if (typeof(T) == typeof(byte)) // this optimization is beneficial only to byte
                 {
-                    T current = Unsafe.Add(ref searchSpace, i);
-                    if (N.NegateIfNeeded(current == value0 || current == value1 || current == value2))
+                    while (length >= 8)
                     {
-                        return i;
+                        length -= 8;
+
+                        ref T current = ref Unsafe.Add(ref searchSpace, offset);
+                        lookUp = current;
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset;
+                        lookUp = Unsafe.Add(ref current, -1);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 1;
+                        lookUp = Unsafe.Add(ref current, -2);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 2;
+                        lookUp = Unsafe.Add(ref current, -3);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 3;
+                        lookUp = Unsafe.Add(ref current, -4);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 4;
+                        lookUp = Unsafe.Add(ref current, -5);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 5;
+                        lookUp = Unsafe.Add(ref current, -6);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 6;
+                        lookUp = Unsafe.Add(ref current, -7);
+                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 7;
+
+                        offset -= 8;
                     }
                 }
+
+                while (length >= 4)
+                {
+                    length -= 4;
+
+                    ref T current = ref Unsafe.Add(ref searchSpace, offset);
+                    lookUp = current;
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset;
+                    lookUp = Unsafe.Add(ref current, -1);
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 1;
+                    lookUp = Unsafe.Add(ref current, -2);
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 2;
+                    lookUp = Unsafe.Add(ref current, -3);
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 3;
+
+                    offset -= 4;
+                }
+
+                while (length > 0)
+                {
+                    length -= 1;
+
+                    lookUp = Unsafe.Add(ref searchSpace, offset);
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset;
+
+                    offset -= 1;
+                }
             }
             else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
             {

From 4274f54108d86d21978b2e9c6471387d8f7ba4f2 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Mon, 15 Aug 2022 18:06:48 +0200
Subject: [PATCH 28/34] vectorize LastIndexOf(4) and LastIndexOfAnyExcept(4)

---
 .../src/System/MemoryExtensions.cs            | 117 ++++++++++++++---
 .../src/System/SpanHelpers.T.cs               | 119 ++++++++++++++++++
 2 files changed, 217 insertions(+), 19 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
index de5b246fc3390f..b190977c878b45 100644
--- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
@@ -920,6 +920,47 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
             return -1;
         }
 
+        private static int LastIndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value0, T value1, T value2, T value3) where T : IEquatable<T>?
+        {
+            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            {
+                if (Unsafe.SizeOf<T>() == sizeof(byte))
+                {
+                    return SpanHelpers.LastIndexOfAnyExceptValueType(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, byte>(ref value0),
+                        Unsafe.As<T, byte>(ref value1),
+                        Unsafe.As<T, byte>(ref value2),
+                        Unsafe.As<T, byte>(ref value3),
+                        span.Length);
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
+                {
+                    return SpanHelpers.LastIndexOfAnyExceptValueType(
+                        ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, short>(ref value0),
+                        Unsafe.As<T, short>(ref value1),
+                        Unsafe.As<T, short>(ref value2),
+                        Unsafe.As<T, short>(ref value3),
+                        span.Length);
+                }
+                // we can easily add int and long here, but there must be an evidence that shows that it's actually needed
+            }
+
+            for (int i = span.Length - 1; i >= 0; i--)
+            {
+                if (!EqualityComparer<T>.Default.Equals(span[i], value0) &&
+                    !EqualityComparer<T>.Default.Equals(span[i], value1) &&
+                    !EqualityComparer<T>.Default.Equals(span[i], value2) &&
+                    !EqualityComparer<T>.Default.Equals(span[i], value3))
+                {
+                    return i;
+                }
+            }
+
+            return -1;
+        }
+
         /// <summary>Searches for the last index of any value other than the specified <paramref name="values"/>.</summary>
         /// <typeparam name="T">The type of the span and values.</typeparam>
         /// <param name="span">The span to search.</param>
@@ -948,6 +989,9 @@ public static int LastIndexOfAnyExcept<T>(this ReadOnlySpan<T> span, ReadOnlySpa
                 case 3:
                     return LastIndexOfAnyExcept(span, values[0], values[1], values[2]);
 
+                case 4: // common for searching ASCII whitespaces (" \t\r\n")
+                    return LastIndexOfAnyExcept(span, values[0], values[1], values[2], values[3]);
+
                 default:
                     for (int i = span.Length - 1; i >= 0; i--)
                     {
@@ -1571,31 +1615,66 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOfAny<T>(this ReadOnlySpan<T> span, ReadOnlySpan<T> values) where T : IEquatable<T>?
         {
-            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            if (Unsafe.SizeOf<T>() == sizeof(byte))
             {
-                if (Unsafe.SizeOf<T>() == sizeof(char))
+                ref byte valueRef = ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(values));
+                if (values.Length == 2)
                 {
-                    switch (values.Length)
-                    {
-                        case 0:
-                            return -1;
+                    return SpanHelpers.LastIndexOfAnyValueType(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        valueRef,
+                        Unsafe.Add(ref valueRef, 1),
+                        span.Length);
+                }
+                else if (values.Length == 3)
+                {
+                    return SpanHelpers.LastIndexOfAnyValueType(
+                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
+                        valueRef,
+                        Unsafe.Add(ref valueRef, 1),
+                        Unsafe.Add(ref valueRef, 2),
+                        span.Length);
+                }
+            }
 
-                        case 1:
-                            return LastIndexOf(span, values[0]);
+            if (Unsafe.SizeOf<T>() == sizeof(short))
+            {
+                ref short spanRef = ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span));
+                ref short valueRef = ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(values));
+                switch (values.Length)
+                {
+                    case 0:
+                        return -1;
 
-                        case 2:
-                            return LastIndexOfAny(span, values[0], values[1]);
+                    case 1:
+                        return SpanHelpers.LastIndexOfValueType(ref spanRef, valueRef, span.Length);
 
-                        case 3:
-                            return LastIndexOfAny(span, values[0], values[1], values[2]);
+                    case 2:
+                        return SpanHelpers.LastIndexOfAnyValueType(
+                            ref spanRef,
+                            valueRef,
+                            Unsafe.Add(ref valueRef, 1),
+                            span.Length);
 
-                        default:
-                            return LastIndexOfAnyProbabilistic(
-                                ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(span)),
-                                span.Length,
-                                ref Unsafe.As<T, char>(ref MemoryMarshal.GetReference(values)),
-                                values.Length);
-                    }
+                    case 3:
+                        return SpanHelpers.LastIndexOfAnyValueType(
+                            ref spanRef,
+                            valueRef,
+                            Unsafe.Add(ref valueRef, 1),
+                            Unsafe.Add(ref valueRef, 2),
+                            span.Length);
+
+                    case 4:
+                        return SpanHelpers.LastIndexOfAnyValueType(
+                            ref spanRef,
+                            valueRef,
+                            Unsafe.Add(ref valueRef, 1),
+                            Unsafe.Add(ref valueRef, 2),
+                            Unsafe.Add(ref valueRef, 3),
+                            span.Length);
+
+                    default:
+                        return LastIndexOfAnyProbabilistic(ref Unsafe.As<short, char>(ref spanRef), span.Length, ref Unsafe.As<short, char>(ref valueRef), values.Length);
                 }
             }
 
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
index d879d3236175d5..cc4b006983e3ea 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
@@ -2349,6 +2349,125 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
             return -1;
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static int LastIndexOfAnyValueType<T>(ref T searchSpace, T value0, T value1, T value2, T value3, int length) where T : struct, INumber<T>
+            => LastIndexOfAnyValueType<T, DontNegate<T>>(ref searchSpace, value0, value1, value2, value3, length);
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static int LastIndexOfAnyExceptValueType<T>(ref T searchSpace, T value0, T value1, T value2, T value3, int length) where T : struct, INumber<T>
+            => LastIndexOfAnyValueType<T, Negate<T>>(ref searchSpace, value0, value1, value2, value3, length);
+
+        [MethodImpl(MethodImplOptions.AggressiveOptimization)]
+        private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T value1, T value2, T value3, int length)
+            where T : struct, INumber<T>
+            where N : struct, INegator<T>
+        {
+            Debug.Assert(length >= 0, "Expected non-negative length");
+            Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types");
+
+            if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
+            {
+                nuint offset = (nuint)length - 1;
+                T lookUp;
+
+                while (length >= 4)
+                {
+                    length -= 4;
+
+                    ref T current = ref Unsafe.Add(ref searchSpace, offset);
+                    lookUp = current;
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset;
+                    lookUp = Unsafe.Add(ref current, -1);
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset - 1;
+                    lookUp = Unsafe.Add(ref current, -2);
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset - 2;
+                    lookUp = Unsafe.Add(ref current, -3);
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset - 3;
+
+                    offset -= 4;
+                }
+
+                while (length > 0)
+                {
+                    length -= 1;
+
+                    lookUp = Unsafe.Add(ref searchSpace, offset);
+                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset;
+
+                    offset -= 1;
+                }
+            }
+            else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
+            {
+                Vector256<T> equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1), values2 = Vector256.Create(value2), values3 = Vector256.Create(value3);
+                ref T currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256<T>.Count);
+
+                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
+                do
+                {
+                    current = Vector256.LoadUnsafe(ref currentSearchSpace);
+                    equals = N.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1)
+                                            | Vector256.Equals(current, values2) | Vector256.Equals(current, values3));
+                    if (equals == Vector256<T>.Zero)
+                    {
+                        currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<T>.Count);
+                        continue;
+                    }
+
+                    return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals);
+                }
+                while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace));
+
+                // If any elements remain, process the first vector in the search space.
+                if ((uint)length % Vector256<T>.Count != 0)
+                {
+                    current = Vector256.LoadUnsafe(ref searchSpace);
+                    equals = N.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1)
+                                            | Vector256.Equals(current, values2) | Vector256.Equals(current, values3));
+                    if (equals != Vector256<T>.Zero)
+                    {
+                        return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
+                    }
+                }
+            }
+            else
+            {
+                Vector128<T> equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1), values2 = Vector128.Create(value2), values3 = Vector128.Create(value3);
+                ref T currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128<T>.Count);
+
+                // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
+                do
+                {
+                    current = Vector128.LoadUnsafe(ref currentSearchSpace);
+                    equals = N.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1)
+                                            | Vector128.Equals(current, values2) | Vector128.Equals(current, values3));
+                    if (equals == Vector128<T>.Zero)
+                    {
+                        currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<T>.Count);
+                        continue;
+                    }
+
+                    return ComputeLastIndex(ref searchSpace, ref currentSearchSpace, equals);
+                }
+                while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace));
+
+                // If any elements remain, process the first vector in the search space.
+                if ((uint)length % Vector128<T>.Count != 0)
+                {
+                    current = Vector128.LoadUnsafe(ref searchSpace);
+                    equals = N.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1)
+                                            | Vector128.Equals(current, values2) | Vector128.Equals(current, values3));
+                    if (equals != Vector128<T>.Zero)
+                    {
+                        return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
+                    }
+                }
+            }
+
+            return -1;
+        }
+
+
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static int ComputeFirstIndex<T>(ref T searchSpace, ref T current, Vector128<T> equals) where T : struct
         {

From 59c867fe17ef3fa3b4aadb8d1d7c6a5012c8596d Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Mon, 15 Aug 2022 18:16:14 +0200
Subject: [PATCH 29/34] Apply suggestions from code review

Co-authored-by: Jan Kotas <jkotas@microsoft.com>
---
 .../System.Private.CoreLib/src/System/StubHelpers.cs      | 4 ++--
 .../Internal/Runtime/CompilerHelpers/InteropHelpers.cs    | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs b/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs
index b0277baa2d56df..fa2735164b88fc 100644
--- a/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs
+++ b/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs
@@ -176,7 +176,7 @@ internal static unsafe void ConvertFixedToNative(int flags, string strManaged, I
 
         internal static unsafe string ConvertFixedToManaged(IntPtr cstr, int length)
         {
-            int end = SpanHelpers.IndexOfValueType(ref *(byte*)cstr, (byte)0, length);
+            int end = new ReadOnlySpan<byte>(cstr, length).IndexOf(0);
             if (end >= 0)
             {
                 length = end;
@@ -450,7 +450,7 @@ internal static unsafe void ConvertToNative(string? strManaged, IntPtr nativeHom
 
         internal static unsafe string ConvertToManaged(IntPtr nativeHome, int length)
         {
-            int end = SpanHelpers.IndexOfValueType(ref *(short*)nativeHome, (short)'\0', length);
+            int end = new ReadOnlySpan<char>(nativeHome, length).IndexOf(`\0`);
             if (end >= 0)
             {
                 length = end;
diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/InteropHelpers.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/InteropHelpers.cs
index 742f3bbc4991c0..d7843557835300 100644
--- a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/InteropHelpers.cs
+++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/InteropHelpers.cs
@@ -55,8 +55,8 @@ internal static unsafe void StringToByValAnsiString(string str, byte* pNative, i
 
         public static unsafe string ByValAnsiStringToString(byte* buffer, int length)
         {
-            int end = SpanHelpers.IndexOfValueType<byte>(ref *buffer, 0, length);
-            if (end != -1)
+            int end = new ReadOnlySpan<byte>(buffer, length).IndexOf(0);
+            if (end >= 0)
             {
                 length = end;
             }
@@ -77,8 +77,8 @@ internal static unsafe void StringToUnicodeFixedArray(string str, ushort* buffer
 
         internal static unsafe string UnicodeToStringFixedArray(ushort* buffer, int length)
         {
-            int end = SpanHelpers.IndexOfChar(ref *(char*)buffer, '\0', length);
-            if (end != -1)
+            int end = new ReadOnlySpan<char>(buffer, length).IndexOf('\0');
+            if (end >= 0)
             {
                 length = end;
             }

From afc09bd42433744a9d3bd2a94505e47f1b235928 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Mon, 15 Aug 2022 18:23:34 +0200
Subject: [PATCH 30/34] fix the build

---
 src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs b/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs
index fa2735164b88fc..bc731b674d2e97 100644
--- a/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs
+++ b/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs
@@ -176,7 +176,7 @@ internal static unsafe void ConvertFixedToNative(int flags, string strManaged, I
 
         internal static unsafe string ConvertFixedToManaged(IntPtr cstr, int length)
         {
-            int end = new ReadOnlySpan<byte>(cstr, length).IndexOf(0);
+            int end = new ReadOnlySpan<byte>((byte*)cstr, length).IndexOf((byte)0);
             if (end >= 0)
             {
                 length = end;
@@ -450,7 +450,7 @@ internal static unsafe void ConvertToNative(string? strManaged, IntPtr nativeHom
 
         internal static unsafe string ConvertToManaged(IntPtr nativeHome, int length)
         {
-            int end = new ReadOnlySpan<char>(nativeHome, length).IndexOf(`\0`);
+            int end = new ReadOnlySpan<char>((char*)nativeHome, length).IndexOf('\0');
             if (end >= 0)
             {
                 length = end;

From efdff0e5f569639965c29f06ace6168cbf35f6b4 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Mon, 15 Aug 2022 20:37:29 +0200
Subject: [PATCH 31/34] fix the NativeAOT build

---
 .../src/Internal/Runtime/CompilerHelpers/InteropHelpers.cs      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/InteropHelpers.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/InteropHelpers.cs
index d7843557835300..8e383b44d85033 100644
--- a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/InteropHelpers.cs
+++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/InteropHelpers.cs
@@ -55,7 +55,7 @@ internal static unsafe void StringToByValAnsiString(string str, byte* pNative, i
 
         public static unsafe string ByValAnsiStringToString(byte* buffer, int length)
         {
-            int end = new ReadOnlySpan<byte>(buffer, length).IndexOf(0);
+            int end = new ReadOnlySpan<byte>(buffer, length).IndexOf((byte)0);
             if (end >= 0)
             {
                 length = end;

From 7f003daf69bc6c886d2ee01b639f8ddce3fffefc Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Tue, 16 Aug 2022 19:06:03 +0200
Subject: [PATCH 32/34] optimize Except methods for small inputs

---
 .../src/System/MemoryExtensions.cs            | 143 ++++++-----------
 .../src/System/SpanHelpers.T.cs               | 149 +++++++++++++++++-
 2 files changed, 196 insertions(+), 96 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
index b190977c878b45..a4e8f803b8cb39 100644
--- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
@@ -529,7 +529,7 @@ public static int IndexOfAnyExcept<T>(this Span<T> span, ReadOnlySpan<T> values)
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int IndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value) where T : IEquatable<T>?
         {
-            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            if (SpanHelpers.CanVectorizeAndBenefit<T>(span.Length))
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
@@ -538,36 +538,34 @@ ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, byte>(ref value),
                         span.Length);
                 }
-
-                if (Unsafe.SizeOf<T>() == sizeof(short))
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
                 {
                     return SpanHelpers.IndexOfAnyExceptValueType(
                         ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, short>(ref value),
                         span.Length);
                 }
-
-                if (Unsafe.SizeOf<T>() == sizeof(int))
+                else if (Unsafe.SizeOf<T>() == sizeof(int))
                 {
                     return SpanHelpers.IndexOfAnyExceptValueType(
                         ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, int>(ref value),
                         span.Length);
                 }
-
-                if (Unsafe.SizeOf<T>() == sizeof(long))
+                else
                 {
+                    Debug.Assert(Unsafe.SizeOf<T>() == sizeof(long));
+
                     return SpanHelpers.IndexOfAnyExceptValueType(
                         ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, long>(ref value),
                         span.Length);
                 }
             }
-
-            return SpanHelpers.IndexOfAnyExcept(
-                ref MemoryMarshal.GetReference(span),
-                value,
-                span.Length);
+            else
+            {
+                return SpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), value, span.Length);
+            }
         }
 
         /// <summary>Searches for the first index of any value other than the specified <paramref name="value0"/> or <paramref name="value1"/>.</summary>
@@ -579,9 +577,10 @@ ref MemoryMarshal.GetReference(span),
         /// The index in the span of the first occurrence of any value other than <paramref name="value0"/> and <paramref name="value1"/>.
         /// If all of the values are <paramref name="value0"/> or <paramref name="value1"/>, returns -1.
         /// </returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int IndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value0, T value1) where T : IEquatable<T>?
         {
-            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            if (SpanHelpers.CanVectorizeAndBenefit<T>(span.Length))
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
@@ -601,16 +600,7 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                 }
             }
 
-            for (int i = 0; i < span.Length; i++)
-            {
-                if (!EqualityComparer<T>.Default.Equals(span[i], value0) &&
-                    !EqualityComparer<T>.Default.Equals(span[i], value1))
-                {
-                    return i;
-                }
-            }
-
-            return -1;
+            return SpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), value0, value1, span.Length);
         }
 
         /// <summary>Searches for the first index of any value other than the specified <paramref name="value0"/>, <paramref name="value1"/>, or <paramref name="value2"/>.</summary>
@@ -623,9 +613,10 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
         /// The index in the span of the first occurrence of any value other than <paramref name="value0"/>, <paramref name="value1"/>, and <paramref name="value2"/>.
         /// If all of the values are <paramref name="value0"/>, <paramref name="value1"/>, and <paramref name="value2"/>, returns -1.
         /// </returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int IndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value0, T value1, T value2) where T : IEquatable<T>?
         {
-            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            if (SpanHelpers.CanVectorizeAndBenefit<T>(span.Length))
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
@@ -647,22 +638,13 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                 }
             }
 
-            for (int i = 0; i < span.Length; i++)
-            {
-                if (!EqualityComparer<T>.Default.Equals(span[i], value0) &&
-                    !EqualityComparer<T>.Default.Equals(span[i], value1) &&
-                    !EqualityComparer<T>.Default.Equals(span[i], value2))
-                {
-                    return i;
-                }
-            }
-
-            return -1;
+            return SpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), value0, value1, value2, span.Length);
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static int IndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value0, T value1, T value2, T value3) where T : IEquatable<T>?
         {
-            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            if (SpanHelpers.CanVectorizeAndBenefit<T>(span.Length))
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
@@ -686,18 +668,7 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                 }
             }
 
-            for (int i = 0; i < span.Length; i++)
-            {
-                if (!EqualityComparer<T>.Default.Equals(span[i], value0) &&
-                    !EqualityComparer<T>.Default.Equals(span[i], value1) &&
-                    !EqualityComparer<T>.Default.Equals(span[i], value2) &&
-                    !EqualityComparer<T>.Default.Equals(span[i], value3))
-                {
-                    return i;
-                }
-            }
-
-            return -1;
+            return SpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), value0, value1, value2, value3, span.Length);
         }
 
         /// <summary>Searches for the first index of any value other than the specified <paramref name="values"/>.</summary>
@@ -798,9 +769,10 @@ public static int LastIndexOfAnyExcept<T>(this Span<T> span, ReadOnlySpan<T> val
         /// The index in the span of the last occurrence of any value other than <paramref name="value"/>.
         /// If all of the values are <paramref name="value"/>, returns -1.
         /// </returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value) where T : IEquatable<T>?
         {
-            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            if (SpanHelpers.CanVectorizeAndBenefit<T>(span.Length))
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
@@ -816,17 +788,27 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, short>(ref value),
                         span.Length);
                 }
-            }
-
-            for (int i = span.Length - 1; i >= 0; i--)
-            {
-                if (!EqualityComparer<T>.Default.Equals(span[i], value))
+                else if (Unsafe.SizeOf<T>() == sizeof(int))
                 {
-                    return i;
+                    return SpanHelpers.LastIndexOfAnyExceptValueType(
+                        ref Unsafe.As<T, int>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, int>(ref value),
+                        span.Length);
                 }
-            }
+                else
+                {
+                    Debug.Assert(Unsafe.SizeOf<T>() == sizeof(long));
 
-            return -1;
+                    return SpanHelpers.LastIndexOfAnyExceptValueType(
+                        ref Unsafe.As<T, long>(ref MemoryMarshal.GetReference(span)),
+                        Unsafe.As<T, long>(ref value),
+                        span.Length);
+                }
+            }
+            else
+            {
+                return SpanHelpers.LastIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), value, span.Length);
+            }
         }
 
         /// <summary>Searches for the last index of any value other than the specified <paramref name="value0"/> or <paramref name="value1"/>.</summary>
@@ -838,9 +820,10 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
         /// The index in the span of the last occurrence of any value other than <paramref name="value0"/> and <paramref name="value1"/>.
         /// If all of the values are <paramref name="value0"/> or <paramref name="value1"/>, returns -1.
         /// </returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value0, T value1) where T : IEquatable<T>?
         {
-            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            if (SpanHelpers.CanVectorizeAndBenefit<T>(span.Length))
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
@@ -860,16 +843,7 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                 }
             }
 
-            for (int i = span.Length - 1; i >= 0; i--)
-            {
-                if (!EqualityComparer<T>.Default.Equals(span[i], value0) &&
-                    !EqualityComparer<T>.Default.Equals(span[i], value1))
-                {
-                    return i;
-                }
-            }
-
-            return -1;
+            return SpanHelpers.LastIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), value0, value1, span.Length);
         }
 
         /// <summary>Searches for the last index of any value other than the specified <paramref name="value0"/>, <paramref name="value1"/>, or <paramref name="value2"/>.</summary>
@@ -882,9 +856,10 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
         /// The index in the span of the last occurrence of any value other than <paramref name="value0"/>, <paramref name="value1"/>, and <paramref name="value2"/>.
         /// If all of the values are <paramref name="value0"/>, <paramref name="value1"/>, and <paramref name="value2"/>, returns -1.
         /// </returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value0, T value1, T value2) where T : IEquatable<T>?
         {
-            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            if (SpanHelpers.CanVectorizeAndBenefit<T>(span.Length))
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
@@ -904,25 +879,15 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, short>(ref value2),
                         span.Length);
                 }
-                // we can easily add int and long here, but there must be an evidence that shows that it's actually needed
-            }
-
-            for (int i = span.Length - 1; i >= 0; i--)
-            {
-                if (!EqualityComparer<T>.Default.Equals(span[i], value0) &&
-                    !EqualityComparer<T>.Default.Equals(span[i], value1) &&
-                    !EqualityComparer<T>.Default.Equals(span[i], value2))
-                {
-                    return i;
-                }
             }
 
-            return -1;
+            return SpanHelpers.LastIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), value0, value1, value2, span.Length);
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static int LastIndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value0, T value1, T value2, T value3) where T : IEquatable<T>?
         {
-            if (RuntimeHelpers.IsBitwiseEquatable<T>())
+            if (SpanHelpers.CanVectorizeAndBenefit<T>(span.Length))
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
@@ -944,21 +909,9 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, short>(ref value3),
                         span.Length);
                 }
-                // we can easily add int and long here, but there must be an evidence that shows that it's actually needed
-            }
-
-            for (int i = span.Length - 1; i >= 0; i--)
-            {
-                if (!EqualityComparer<T>.Default.Equals(span[i], value0) &&
-                    !EqualityComparer<T>.Default.Equals(span[i], value1) &&
-                    !EqualityComparer<T>.Default.Equals(span[i], value2) &&
-                    !EqualityComparer<T>.Default.Equals(span[i], value3))
-                {
-                    return i;
-                }
             }
 
-            return -1;
+            return SpanHelpers.LastIndexOfAnyExcept(ref MemoryMarshal.GetReference(span), value0, value1, value2, value3, span.Length);
         }
 
         /// <summary>Searches for the last index of any value other than the specified <paramref name="values"/>.</summary>
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
index cc4b006983e3ea..feb698c5e00a99 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
@@ -1053,7 +1053,7 @@ public static int LastIndexOfAny<T>(ref T searchSpace, int searchSpaceLength, re
             return -1; // not found
         }
 
-        public static int IndexOfAnyExcept<T>(ref T searchSpace, T value0, int length)
+        internal static int IndexOfAnyExcept<T>(ref T searchSpace, T value0, int length)
         {
             Debug.Assert(length >= 0, "Expected non-negative length");
 
@@ -1068,6 +1068,127 @@ public static int IndexOfAnyExcept<T>(ref T searchSpace, T value0, int length)
             return -1;
         }
 
+        internal static int LastIndexOfAnyExcept<T>(ref T searchSpace, T value0, int length)
+        {
+            Debug.Assert(length >= 0, "Expected non-negative length");
+
+            for (int i = length -1; i >= 0; i--)
+            {
+                if (!EqualityComparer<T>.Default.Equals(Unsafe.Add(ref searchSpace, i), value0))
+                {
+                    return i;
+                }
+            }
+
+            return -1;
+        }
+
+        internal static int IndexOfAnyExcept<T>(ref T searchSpace, T value0, T value1, int length)
+        {
+            Debug.Assert(length >= 0, "Expected non-negative length");
+
+            for (int i = 0; i < length; i++)
+            {
+                ref T current = ref Unsafe.Add(ref searchSpace, i);
+                if (!EqualityComparer<T>.Default.Equals(current, value0) && !EqualityComparer<T>.Default.Equals(current, value1))
+                {
+                    return i;
+                }
+            }
+
+            return -1;
+        }
+
+        internal static int LastIndexOfAnyExcept<T>(ref T searchSpace, T value0, T value1, int length)
+        {
+            Debug.Assert(length >= 0, "Expected non-negative length");
+
+            for (int i = length - 1; i >= 0; i--)
+            {
+                ref T current = ref Unsafe.Add(ref searchSpace, i);
+                if (!EqualityComparer<T>.Default.Equals(current, value0) && !EqualityComparer<T>.Default.Equals(current, value1))
+                {
+                    return i;
+                }
+            }
+
+            return -1;
+        }
+
+        internal static int IndexOfAnyExcept<T>(ref T searchSpace, T value0, T value1, T value2, int length)
+        {
+            Debug.Assert(length >= 0, "Expected non-negative length");
+
+            for (int i = 0; i < length; i++)
+            {
+                ref T current = ref Unsafe.Add(ref searchSpace, i);
+                if (!EqualityComparer<T>.Default.Equals(current, value0)
+                    && !EqualityComparer<T>.Default.Equals(current, value1)
+                    && !EqualityComparer<T>.Default.Equals(current, value2))
+                {
+                    return i;
+                }
+            }
+
+            return -1;
+        }
+
+        internal static int LastIndexOfAnyExcept<T>(ref T searchSpace, T value0, T value1, T value2, int length)
+        {
+            Debug.Assert(length >= 0, "Expected non-negative length");
+
+            for (int i = length - 1; i >= 0; i--)
+            {
+                ref T current = ref Unsafe.Add(ref searchSpace, i);
+                if (!EqualityComparer<T>.Default.Equals(current, value0)
+                    && !EqualityComparer<T>.Default.Equals(current, value1)
+                    && !EqualityComparer<T>.Default.Equals(current, value2))
+                {
+                    return i;
+                }
+            }
+
+            return -1;
+        }
+
+        internal static int IndexOfAnyExcept<T>(ref T searchSpace, T value0, T value1, T value2, T value3, int length)
+        {
+            Debug.Assert(length >= 0, "Expected non-negative length");
+
+            for (int i = 0; i < length; i++)
+            {
+                ref T current = ref Unsafe.Add(ref searchSpace, i);
+                if (!EqualityComparer<T>.Default.Equals(current, value0)
+                    && !EqualityComparer<T>.Default.Equals(current, value1)
+                    && !EqualityComparer<T>.Default.Equals(current, value2)
+                    && !EqualityComparer<T>.Default.Equals(current, value3))
+                {
+                    return i;
+                }
+            }
+
+            return -1;
+        }
+
+        internal static int LastIndexOfAnyExcept<T>(ref T searchSpace, T value0, T value1, T value2, T value3, int length)
+        {
+            Debug.Assert(length >= 0, "Expected non-negative length");
+
+            for (int i = length - 1; i >= 0; i--)
+            {
+                ref T current = ref Unsafe.Add(ref searchSpace, i);
+                if (!EqualityComparer<T>.Default.Equals(current, value0)
+                    && !EqualityComparer<T>.Default.Equals(current, value1)
+                    && !EqualityComparer<T>.Default.Equals(current, value2)
+                    && !EqualityComparer<T>.Default.Equals(current, value3))
+                {
+                    return i;
+                }
+            }
+
+            return -1;
+        }
+
         public static bool SequenceEqual<T>(ref T first, ref T second, int length) where T : IEquatable<T>?
         {
             Debug.Assert(length >= 0);
@@ -1178,6 +1299,32 @@ public static int SequenceCompareTo<T>(ref T first, int firstLength, ref T secon
             return firstLength.CompareTo(secondLength);
         }
 
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        internal static bool CanVectorizeAndBenefit<T>(int length) where T : IEquatable<T>?
+        {
+            if (Vector128.IsHardwareAccelerated && RuntimeHelpers.IsBitwiseEquatable<T>())
+            {
+                if (Unsafe.SizeOf<T>() == sizeof(byte))
+                {
+                    return length >= Vector128<byte>.Count;
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(short))
+                {
+                    return length >= Vector128<short>.Count;
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(int))
+                {
+                    return length >= Vector128<int>.Count;
+                }
+                else if (Unsafe.SizeOf<T>() == sizeof(long))
+                {
+                    return length >= Vector128<long>.Count;
+                }
+            }
+
+            return false;
+        }
+
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
         internal static bool ContainsValueType<T>(ref T searchSpace, T value, int length) where T : struct, INumber<T>
         {

From 0dc257ff0e827bb9bb864098ecc99e8b52760063 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Wed, 17 Aug 2022 09:00:10 +0200
Subject: [PATCH 33/34] address code review feedback:

- add a failing test and fix the bug
- remove comments
- rename generic arguments to follow existing convention
- rename constant to start with uppercase character
---
 .../System.Memory/tests/Span/IndexOf.T.cs     |  55 ++
 .../src/System/MemoryExtensions.cs            | 103 ++--
 .../src/System/SpanHelpers.Byte.cs            |  30 +-
 .../src/System/SpanHelpers.T.cs               | 569 +++++++++---------
 4 files changed, 406 insertions(+), 351 deletions(-)

diff --git a/src/libraries/System.Memory/tests/Span/IndexOf.T.cs b/src/libraries/System.Memory/tests/Span/IndexOf.T.cs
index 67b6e896bd59fa..e2fb8a0e64e48a 100644
--- a/src/libraries/System.Memory/tests/Span/IndexOf.T.cs
+++ b/src/libraries/System.Memory/tests/Span/IndexOf.T.cs
@@ -192,5 +192,60 @@ public static void IndexOfNull_String(string[] spanInput, int expected)
             Span<string> theStrings = spanInput;
             Assert.Equal(expected, theStrings.IndexOf((string)null));
         }
+
+        [Fact]
+        public static void NotBitwiseEquatableUsesCustomIEquatableImplementationForActualComparison()
+        {
+            const byte Ten = 10, NotTen = 11;
+            for (int length = 1; length < 100; length++)
+            {
+                TwoBytes[] array = new TwoBytes[length];
+                for (int i = 0; i < length; i++)
+                {
+                    array[i] = new TwoBytes(Ten, (byte)i);
+                }
+
+                Span<TwoBytes> span = new Span<TwoBytes>(array);
+                ReadOnlySpan<TwoBytes> ros = new ReadOnlySpan<TwoBytes>(array);
+
+                ReadOnlySpan<TwoBytes> noMatch2 = new TwoBytes[2] { new TwoBytes(10, NotTen), new TwoBytes(10, NotTen) };
+                Assert.Equal(-1, span.IndexOfAny(noMatch2));
+                Assert.Equal(-1, ros.IndexOfAny(noMatch2));
+                Assert.Equal(-1, span.LastIndexOfAny(noMatch2));
+                Assert.Equal(-1, ros.LastIndexOfAny(noMatch2));
+
+                ReadOnlySpan<TwoBytes> noMatch3 = new TwoBytes[3] { new TwoBytes(10, NotTen), new TwoBytes(10, NotTen), new TwoBytes(10, NotTen) };
+                Assert.Equal(-1, span.IndexOfAny(noMatch3));
+                Assert.Equal(-1, ros.IndexOfAny(noMatch3));
+                Assert.Equal(-1, span.LastIndexOfAny(noMatch3));
+                Assert.Equal(-1, ros.LastIndexOfAny(noMatch3));
+
+                ReadOnlySpan<TwoBytes> match2 = new TwoBytes[2] { new TwoBytes(0, Ten), new TwoBytes(0, Ten) };
+                Assert.Equal(0, span.IndexOfAny(match2));
+                Assert.Equal(0, ros.IndexOfAny(match2));
+                Assert.Equal(0, span.LastIndexOfAny(match2));
+                Assert.Equal(0, ros.LastIndexOfAny(match2));
+
+                ReadOnlySpan<TwoBytes> match3 = new TwoBytes[3] { new TwoBytes(0, Ten), new TwoBytes(0, Ten), new TwoBytes(0, Ten) };
+                Assert.Equal(0, span.IndexOfAny(match3));
+                Assert.Equal(0, ros.IndexOfAny(match3));
+                Assert.Equal(0, span.LastIndexOfAny(match3));
+                Assert.Equal(0, ros.LastIndexOfAny(match3));
+            }
+        }
+
+        private readonly struct TwoBytes : IEquatable<TwoBytes>
+        {
+            private readonly byte _first, _second;
+
+            public TwoBytes(byte first, byte second)
+            {
+                _first = first;
+                _second = second;
+            }
+
+            // it compares different fields on purpose
+            public bool Equals(TwoBytes other) => _first == other._second && _second == other._first;
+        }
     }
 }
diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
index a4e8f803b8cb39..dcdddefa9defe6 100644
--- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
@@ -698,7 +698,7 @@ public static int IndexOfAnyExcept<T>(this ReadOnlySpan<T> span, ReadOnlySpan<T>
                 case 3:
                     return IndexOfAnyExcept(span, values[0], values[1], values[2]);
 
-                case 4: // used to search for non whitespaces " \t\r\n"
+                case 4:
                     return IndexOfAnyExcept(span, values[0], values[1], values[2], values[3]);
 
                 default:
@@ -942,7 +942,7 @@ public static int LastIndexOfAnyExcept<T>(this ReadOnlySpan<T> span, ReadOnlySpa
                 case 3:
                     return LastIndexOfAnyExcept(span, values[0], values[1], values[2]);
 
-                case 4: // common for searching ASCII whitespaces (" \t\r\n")
+                case 4:
                     return LastIndexOfAnyExcept(span, values[0], values[1], values[2], values[3]);
 
                 default:
@@ -1468,7 +1468,6 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, short>(ref value2),
                         span.Length);
                 }
-                // we can easily add int and long here, but there must be an evidence that shows that it's actually needed
             }
 
             return SpanHelpers.LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, value2, span.Length);
@@ -1554,7 +1553,6 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
                         Unsafe.As<T, short>(ref value2),
                         span.Length);
                 }
-                // we can easily add int and long here, but there must be an evidence that shows that it's actually needed
             }
 
             return SpanHelpers.LastIndexOfAny(ref MemoryMarshal.GetReference(span), value0, value1, value2, span.Length);
@@ -1568,66 +1566,69 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOfAny<T>(this ReadOnlySpan<T> span, ReadOnlySpan<T> values) where T : IEquatable<T>?
         {
-            if (Unsafe.SizeOf<T>() == sizeof(byte))
-            {
-                ref byte valueRef = ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(values));
-                if (values.Length == 2)
-                {
-                    return SpanHelpers.LastIndexOfAnyValueType(
-                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
-                        valueRef,
-                        Unsafe.Add(ref valueRef, 1),
-                        span.Length);
-                }
-                else if (values.Length == 3)
-                {
-                    return SpanHelpers.LastIndexOfAnyValueType(
-                        ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
-                        valueRef,
-                        Unsafe.Add(ref valueRef, 1),
-                        Unsafe.Add(ref valueRef, 2),
-                        span.Length);
-                }
-            }
-
-            if (Unsafe.SizeOf<T>() == sizeof(short))
+            if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
-                ref short spanRef = ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span));
-                ref short valueRef = ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(values));
-                switch (values.Length)
+                if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
-                    case 0:
-                        return -1;
-
-                    case 1:
-                        return SpanHelpers.LastIndexOfValueType(ref spanRef, valueRef, span.Length);
-
-                    case 2:
+                    ref byte valueRef = ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(values));
+                    if (values.Length == 2)
+                    {
                         return SpanHelpers.LastIndexOfAnyValueType(
-                            ref spanRef,
+                            ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                             valueRef,
                             Unsafe.Add(ref valueRef, 1),
                             span.Length);
-
-                    case 3:
+                    }
+                    else if (values.Length == 3)
+                    {
                         return SpanHelpers.LastIndexOfAnyValueType(
-                            ref spanRef,
+                            ref Unsafe.As<T, byte>(ref MemoryMarshal.GetReference(span)),
                             valueRef,
                             Unsafe.Add(ref valueRef, 1),
                             Unsafe.Add(ref valueRef, 2),
                             span.Length);
+                    }
+                }
 
-                    case 4:
-                        return SpanHelpers.LastIndexOfAnyValueType(
-                            ref spanRef,
-                            valueRef,
-                            Unsafe.Add(ref valueRef, 1),
-                            Unsafe.Add(ref valueRef, 2),
-                            Unsafe.Add(ref valueRef, 3),
-                            span.Length);
+                if (Unsafe.SizeOf<T>() == sizeof(short))
+                {
+                    ref short spanRef = ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span));
+                    ref short valueRef = ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(values));
+                    switch (values.Length)
+                    {
+                        case 0:
+                            return -1;
 
-                    default:
-                        return LastIndexOfAnyProbabilistic(ref Unsafe.As<short, char>(ref spanRef), span.Length, ref Unsafe.As<short, char>(ref valueRef), values.Length);
+                        case 1:
+                            return SpanHelpers.LastIndexOfValueType(ref spanRef, valueRef, span.Length);
+
+                        case 2:
+                            return SpanHelpers.LastIndexOfAnyValueType(
+                                ref spanRef,
+                                valueRef,
+                                Unsafe.Add(ref valueRef, 1),
+                                span.Length);
+
+                        case 3:
+                            return SpanHelpers.LastIndexOfAnyValueType(
+                                ref spanRef,
+                                valueRef,
+                                Unsafe.Add(ref valueRef, 1),
+                                Unsafe.Add(ref valueRef, 2),
+                                span.Length);
+
+                        case 4:
+                            return SpanHelpers.LastIndexOfAnyValueType(
+                                ref spanRef,
+                                valueRef,
+                                Unsafe.Add(ref valueRef, 1),
+                                Unsafe.Add(ref valueRef, 2),
+                                Unsafe.Add(ref valueRef, 3),
+                                span.Length);
+
+                        default:
+                            return LastIndexOfAnyProbabilistic(ref Unsafe.As<short, char>(ref spanRef), span.Length, ref Unsafe.As<short, char>(ref valueRef), values.Length);
+                    }
                 }
             }
 
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
index a66042e0302972..61565cfd27db38 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Byte.cs
@@ -344,11 +344,11 @@ private static void ThrowMustBeNullTerminatedString()
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
         internal static unsafe int IndexOfNullByte(ref byte searchSpace)
         {
-            const int length = int.MaxValue;
+            const int Length = int.MaxValue;
 
             const uint uValue = 0; // Use uint for comparisons to avoid unnecessary 8->32 extensions
             nuint offset = 0; // Use nuint for arithmetic to avoid unnecessary 64->32->64 truncations
-            nuint lengthToExamine = (nuint)(uint)length;
+            nuint lengthToExamine = (nuint)(uint)Length;
 
             if (Vector128.IsHardwareAccelerated)
             {
@@ -415,7 +415,7 @@ internal static unsafe int IndexOfNullByte(ref byte searchSpace)
             // have hardware accelerated. After processing Vector lengths we return to SequentialScan to finish any remaining.
             if (Vector256.IsHardwareAccelerated)
             {
-                if (offset < (nuint)(uint)length)
+                if (offset < (nuint)(uint)Length)
                 {
                     if ((((nuint)(uint)Unsafe.AsPointer(ref searchSpace) + offset) & (nuint)(Vector256<byte>.Count - 1)) != 0)
                     {
@@ -439,7 +439,7 @@ internal static unsafe int IndexOfNullByte(ref byte searchSpace)
                         }
                     }
 
-                    lengthToExamine = GetByteVector256SpanLength(offset, length);
+                    lengthToExamine = GetByteVector256SpanLength(offset, Length);
                     if (lengthToExamine > offset)
                     {
                         do
@@ -460,7 +460,7 @@ internal static unsafe int IndexOfNullByte(ref byte searchSpace)
                         } while (lengthToExamine > offset);
                     }
 
-                    lengthToExamine = GetByteVector128SpanLength(offset, length);
+                    lengthToExamine = GetByteVector128SpanLength(offset, Length);
                     if (lengthToExamine > offset)
                     {
                         Vector128<byte> search = Vector128.LoadUnsafe(ref searchSpace, offset);
@@ -479,18 +479,18 @@ internal static unsafe int IndexOfNullByte(ref byte searchSpace)
                         }
                     }
 
-                    if (offset < (nuint)(uint)length)
+                    if (offset < (nuint)(uint)Length)
                     {
-                        lengthToExamine = ((nuint)(uint)length - offset);
+                        lengthToExamine = ((nuint)(uint)Length - offset);
                         goto SequentialScan;
                     }
                 }
             }
             else if (Vector128.IsHardwareAccelerated)
             {
-                if (offset < (nuint)(uint)length)
+                if (offset < (nuint)(uint)Length)
                 {
-                    lengthToExamine = GetByteVector128SpanLength(offset, length);
+                    lengthToExamine = GetByteVector128SpanLength(offset, Length);
 
                     while (lengthToExamine > offset)
                     {
@@ -510,18 +510,18 @@ internal static unsafe int IndexOfNullByte(ref byte searchSpace)
                         return (int)(offset + (uint)BitOperations.TrailingZeroCount(matches));
                     }
 
-                    if (offset < (nuint)(uint)length)
+                    if (offset < (nuint)(uint)Length)
                     {
-                        lengthToExamine = ((nuint)(uint)length - offset);
+                        lengthToExamine = ((nuint)(uint)Length - offset);
                         goto SequentialScan;
                     }
                 }
             }
             else if (Vector.IsHardwareAccelerated)
             {
-                if (offset < (nuint)(uint)length)
+                if (offset < (nuint)(uint)Length)
                 {
-                    lengthToExamine = GetByteVectorSpanLength(offset, length);
+                    lengthToExamine = GetByteVectorSpanLength(offset, Length);
 
                     while (lengthToExamine > offset)
                     {
@@ -536,9 +536,9 @@ internal static unsafe int IndexOfNullByte(ref byte searchSpace)
                         return (int)offset + LocateFirstFoundByte(matches);
                     }
 
-                    if (offset < (nuint)(uint)length)
+                    if (offset < (nuint)(uint)Length)
                     {
-                        lengthToExamine = ((nuint)(uint)length - offset);
+                        lengthToExamine = ((nuint)(uint)Length - offset);
                         goto SequentialScan;
                     }
                 }
diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
index feb698c5e00a99..b093cb11a29e6d 100644
--- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs
@@ -1455,14 +1455,14 @@ internal static int IndexOfAnyExceptValueType<T>(ref T searchSpace, T value, int
             => IndexOfValueType<T, Negate<T>>(ref searchSpace, value, length);
 
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        private static int IndexOfValueType<T, N>(ref T searchSpace, T value, int length)
-            where T : struct, INumber<T>
-            where N : struct, INegator<T>
+        private static int IndexOfValueType<TValue, TNegator>(ref TValue searchSpace, TValue value, int length)
+            where TValue : struct, INumber<TValue>
+            where TNegator : struct, INegator<TValue>
         {
             Debug.Assert(length >= 0, "Expected non-negative length");
             Debug.Assert(value is byte or short or int or long, "Expected caller to normalize to one of these types");
 
-            if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
+            if (!Vector128.IsHardwareAccelerated || length < Vector128<TValue>.Count)
             {
                 nuint offset = 0;
 
@@ -1470,14 +1470,14 @@ private static int IndexOfValueType<T, N>(ref T searchSpace, T value, int length
                 {
                     length -= 8;
 
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset) == value)) return (int)offset;
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 1) == value)) return (int)offset + 1;
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 2) == value)) return (int)offset + 2;
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 3) == value)) return (int)offset + 3;
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 4) == value)) return (int)offset + 4;
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 5) == value)) return (int)offset + 5;
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 6) == value)) return (int)offset + 6;
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 7) == value)) return (int)offset + 7;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset) == value)) return (int)offset;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 1) == value)) return (int)offset + 1;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 2) == value)) return (int)offset + 2;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 3) == value)) return (int)offset + 3;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 4) == value)) return (int)offset + 4;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 5) == value)) return (int)offset + 5;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 6) == value)) return (int)offset + 6;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 7) == value)) return (int)offset + 7;
 
                     offset += 8;
                 }
@@ -1486,10 +1486,10 @@ private static int IndexOfValueType<T, N>(ref T searchSpace, T value, int length
                 {
                     length -= 4;
 
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset) == value)) return (int)offset;
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 1) == value)) return (int)offset + 1;
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 2) == value)) return (int)offset + 2;
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 3) == value)) return (int)offset + 3;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset) == value)) return (int)offset;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 1) == value)) return (int)offset + 1;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 2) == value)) return (int)offset + 2;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset + 3) == value)) return (int)offset + 3;
 
                     offset += 4;
                 }
@@ -1498,24 +1498,24 @@ private static int IndexOfValueType<T, N>(ref T searchSpace, T value, int length
                 {
                     length -= 1;
 
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset).Equals(value))) return (int)offset;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset).Equals(value))) return (int)offset;
 
                     offset += 1;
                 }
             }
-            else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
+            else if (Vector256.IsHardwareAccelerated && length >= Vector256<TValue>.Count)
             {
-                Vector256<T> equals, values = Vector256.Create(value);
-                ref T currentSearchSpace = ref searchSpace;
-                ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256<T>.Count);
+                Vector256<TValue> equals, values = Vector256.Create(value);
+                ref TValue currentSearchSpace = ref searchSpace;
+                ref TValue oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256<TValue>.Count);
 
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
-                    equals = N.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref currentSearchSpace)));
-                    if (equals == Vector256<T>.Zero)
+                    equals = TNegator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref currentSearchSpace)));
+                    if (equals == Vector256<TValue>.Zero)
                     {
-                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector256<T>.Count);
+                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector256<TValue>.Count);
                         continue;
                     }
 
@@ -1524,10 +1524,10 @@ private static int IndexOfValueType<T, N>(ref T searchSpace, T value, int length
                 while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd));
 
                 // If any elements remain, process the last vector in the search space.
-                if ((uint)length % Vector256<T>.Count != 0)
+                if ((uint)length % Vector256<TValue>.Count != 0)
                 {
-                    equals = N.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref oneVectorAwayFromEnd)));
-                    if (equals != Vector256<T>.Zero)
+                    equals = TNegator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref oneVectorAwayFromEnd)));
+                    if (equals != Vector256<TValue>.Zero)
                     {
                         return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
                     }
@@ -1535,17 +1535,17 @@ private static int IndexOfValueType<T, N>(ref T searchSpace, T value, int length
             }
             else
             {
-                Vector128<T> equals, values = Vector128.Create(value);
-                ref T currentSearchSpace = ref searchSpace;
-                ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128<T>.Count);
+                Vector128<TValue> equals, values = Vector128.Create(value);
+                ref TValue currentSearchSpace = ref searchSpace;
+                ref TValue oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128<TValue>.Count);
 
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
-                    equals = N.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref currentSearchSpace)));
-                    if (equals == Vector128<T>.Zero)
+                    equals = TNegator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref currentSearchSpace)));
+                    if (equals == Vector128<TValue>.Zero)
                     {
-                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128<T>.Count);
+                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128<TValue>.Count);
                         continue;
                     }
 
@@ -1554,10 +1554,10 @@ private static int IndexOfValueType<T, N>(ref T searchSpace, T value, int length
                 while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd));
 
                 // If any elements remain, process the first vector in the search space.
-                if ((uint)length % Vector128<T>.Count != 0)
+                if ((uint)length % Vector128<TValue>.Count != 0)
                 {
-                    equals = N.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref oneVectorAwayFromEnd)));
-                    if (equals != Vector128<T>.Zero)
+                    equals = TNegator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref oneVectorAwayFromEnd)));
+                    if (equals != Vector128<TValue>.Zero)
                     {
                         return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
                     }
@@ -1581,41 +1581,41 @@ internal static int IndexOfAnyExceptValueType<T>(ref T searchSpace, T value0, T
 
         // having INumber<T> constraint here allows to use == operator and get better perf compared to .Equals
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T value1, int length)
-            where T : struct, INumber<T>
-            where N : struct, INegator<T>
+        private static int IndexOfAnyValueType<TValue, TNegator>(ref TValue searchSpace, TValue value0, TValue value1, int length)
+            where TValue : struct, INumber<TValue>
+            where TNegator : struct, INegator<TValue>
         {
             Debug.Assert(length >= 0, "Expected non-negative length");
             Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types");
 
-            if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
+            if (!Vector128.IsHardwareAccelerated || length < Vector128<TValue>.Count)
             {
                 nuint offset = 0;
-                T lookUp;
+                TValue lookUp;
 
-                if (typeof(T) == typeof(byte)) // this optimization is beneficial only to byte
+                if (typeof(TValue) == typeof(byte)) // this optimization is beneficial only to byte
                 {
                     while (length >= 8)
                     {
                         length -= 8;
 
-                        ref T current = ref Unsafe.Add(ref searchSpace, offset);
+                        ref TValue current = ref Unsafe.Add(ref searchSpace, offset);
                         lookUp = current;
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset;
                         lookUp = Unsafe.Add(ref current, 1);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 1;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 1;
                         lookUp = Unsafe.Add(ref current, 2);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 2;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 2;
                         lookUp = Unsafe.Add(ref current, 3);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 3;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 3;
                         lookUp = Unsafe.Add(ref current, 4);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 4;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 4;
                         lookUp = Unsafe.Add(ref current, 5);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 5;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 5;
                         lookUp = Unsafe.Add(ref current, 6);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 6;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 6;
                         lookUp = Unsafe.Add(ref current, 7);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 7;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 7;
 
                         offset += 8;
                     }
@@ -1625,15 +1625,15 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
                 {
                     length -= 4;
 
-                    ref T current = ref Unsafe.Add(ref searchSpace, offset);
+                    ref TValue current = ref Unsafe.Add(ref searchSpace, offset);
                     lookUp = current;
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset;
                     lookUp = Unsafe.Add(ref current, 1);
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 1;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 1;
                     lookUp = Unsafe.Add(ref current, 2);
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 2;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 2;
                     lookUp = Unsafe.Add(ref current, 3);
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 3;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset + 3;
 
                     offset += 4;
                 }
@@ -1643,25 +1643,25 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
                     length -= 1;
 
                     lookUp = Unsafe.Add(ref searchSpace, offset);
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset;
 
                     offset += 1;
                 }
             }
-            else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
+            else if (Vector256.IsHardwareAccelerated && length >= Vector256<TValue>.Count)
             {
-                Vector256<T> equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1);
-                ref T currentSearchSpace = ref searchSpace;
-                ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256<T>.Count);
+                Vector256<TValue> equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1);
+                ref TValue currentSearchSpace = ref searchSpace;
+                ref TValue oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256<TValue>.Count);
 
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
                     current = Vector256.LoadUnsafe(ref currentSearchSpace);
-                    equals = N.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current));
-                    if (equals == Vector256<T>.Zero)
+                    equals = TNegator.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current));
+                    if (equals == Vector256<TValue>.Zero)
                     {
-                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector256<T>.Count);
+                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector256<TValue>.Count);
                         continue;
                     }
 
@@ -1670,11 +1670,11 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
                 while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd));
 
                 // If any elements remain, process the last vector in the search space.
-                if ((uint)length % Vector256<T>.Count != 0)
+                if ((uint)length % Vector256<TValue>.Count != 0)
                 {
                     current = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd);
-                    equals = N.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current));
-                    if (equals != Vector256<T>.Zero)
+                    equals = TNegator.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current));
+                    if (equals != Vector256<TValue>.Zero)
                     {
                         return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
                     }
@@ -1682,18 +1682,18 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
             }
             else
             {
-                Vector128<T> equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1);
-                ref T currentSearchSpace = ref searchSpace;
-                ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128<T>.Count);
+                Vector128<TValue> equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1);
+                ref TValue currentSearchSpace = ref searchSpace;
+                ref TValue oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128<TValue>.Count);
 
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
                     current = Vector128.LoadUnsafe(ref currentSearchSpace);
-                    equals = N.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current));
-                    if (equals == Vector128<T>.Zero)
+                    equals = TNegator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current));
+                    if (equals == Vector128<TValue>.Zero)
                     {
-                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128<T>.Count);
+                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128<TValue>.Count);
                         continue;
                     }
 
@@ -1702,11 +1702,11 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
                 while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd));
 
                 // If any elements remain, process the first vector in the search space.
-                if ((uint)length % Vector128<T>.Count != 0)
+                if ((uint)length % Vector128<TValue>.Count != 0)
                 {
                     current = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd);
-                    equals = N.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current));
-                    if (equals != Vector128<T>.Zero)
+                    equals = TNegator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current));
+                    if (equals != Vector128<TValue>.Zero)
                     {
                         return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
                     }
@@ -1725,41 +1725,41 @@ internal static int IndexOfAnyExceptValueType<T>(ref T searchSpace, T value0, T
             => IndexOfAnyValueType<T, Negate<T>>(ref searchSpace, value0, value1, value2, length);
 
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T value1, T value2, int length)
-            where T : struct, INumber<T>
-            where N : struct, INegator<T>
+        private static int IndexOfAnyValueType<TValue, TNegator>(ref TValue searchSpace, TValue value0, TValue value1, TValue value2, int length)
+            where TValue : struct, INumber<TValue>
+            where TNegator : struct, INegator<TValue>
         {
             Debug.Assert(length >= 0, "Expected non-negative length");
             Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types");
 
-            if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
+            if (!Vector128.IsHardwareAccelerated || length < Vector128<TValue>.Count)
             {
                 nuint offset = 0;
-                T lookUp;
+                TValue lookUp;
 
-                if (typeof(T) == typeof(byte)) // this optimization is beneficial only to byte
+                if (typeof(TValue) == typeof(byte)) // this optimization is beneficial only to byte
                 {
                     while (length >= 8)
                     {
                         length -= 8;
 
-                        ref T current = ref Unsafe.Add(ref searchSpace, offset);
+                        ref TValue current = ref Unsafe.Add(ref searchSpace, offset);
                         lookUp = current;
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset;
                         lookUp = Unsafe.Add(ref current, 1);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 1;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 1;
                         lookUp = Unsafe.Add(ref current, 2);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 2;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 2;
                         lookUp = Unsafe.Add(ref current, 3);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 3;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 3;
                         lookUp = Unsafe.Add(ref current, 4);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 4;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 4;
                         lookUp = Unsafe.Add(ref current, 5);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 5;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 5;
                         lookUp = Unsafe.Add(ref current, 6);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 6;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 6;
                         lookUp = Unsafe.Add(ref current, 7);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 7;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 7;
 
                         offset += 8;
                     }
@@ -1769,15 +1769,15 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
                 {
                     length -= 4;
 
-                    ref T current = ref Unsafe.Add(ref searchSpace, offset);
+                    ref TValue current = ref Unsafe.Add(ref searchSpace, offset);
                     lookUp = current;
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset;
                     lookUp = Unsafe.Add(ref current, 1);
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 1;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 1;
                     lookUp = Unsafe.Add(ref current, 2);
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 2;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 2;
                     lookUp = Unsafe.Add(ref current, 3);
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 3;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset + 3;
 
                     offset += 4;
                 }
@@ -1787,25 +1787,25 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
                     length -= 1;
 
                     lookUp = Unsafe.Add(ref searchSpace, offset);
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset;
 
                     offset += 1;
                 }
             }
-            else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
+            else if (Vector256.IsHardwareAccelerated && length >= Vector256<TValue>.Count)
             {
-                Vector256<T> equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1), values2 = Vector256.Create(value2);
-                ref T currentSearchSpace = ref searchSpace;
-                ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256<T>.Count);
+                Vector256<TValue> equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1), values2 = Vector256.Create(value2);
+                ref TValue currentSearchSpace = ref searchSpace;
+                ref TValue oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256<TValue>.Count);
 
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
                     current = Vector256.LoadUnsafe(ref currentSearchSpace);
-                    equals = N.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current) | Vector256.Equals(values2, current));
-                    if (equals == Vector256<T>.Zero)
+                    equals = TNegator.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current) | Vector256.Equals(values2, current));
+                    if (equals == Vector256<TValue>.Zero)
                     {
-                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector256<T>.Count);
+                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector256<TValue>.Count);
                         continue;
                     }
 
@@ -1814,11 +1814,11 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
                 while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd));
 
                 // If any elements remain, process the last vector in the search space.
-                if ((uint)length % Vector256<T>.Count != 0)
+                if ((uint)length % Vector256<TValue>.Count != 0)
                 {
                     current = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd);
-                    equals = N.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current) | Vector256.Equals(values2, current));
-                    if (equals != Vector256<T>.Zero)
+                    equals = TNegator.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current) | Vector256.Equals(values2, current));
+                    if (equals != Vector256<TValue>.Zero)
                     {
                         return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
                     }
@@ -1826,18 +1826,18 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
             }
             else
             {
-                Vector128<T> equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1), values2 = Vector128.Create(value2);
-                ref T currentSearchSpace = ref searchSpace;
-                ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128<T>.Count);
+                Vector128<TValue> equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1), values2 = Vector128.Create(value2);
+                ref TValue currentSearchSpace = ref searchSpace;
+                ref TValue oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128<TValue>.Count);
 
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
                     current = Vector128.LoadUnsafe(ref currentSearchSpace);
-                    equals = N.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current) | Vector128.Equals(values2, current));
-                    if (equals == Vector128<T>.Zero)
+                    equals = TNegator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current) | Vector128.Equals(values2, current));
+                    if (equals == Vector128<TValue>.Zero)
                     {
-                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128<T>.Count);
+                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128<TValue>.Count);
                         continue;
                     }
 
@@ -1846,11 +1846,11 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
                 while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd));
 
                 // If any elements remain, process the first vector in the search space.
-                if ((uint)length % Vector128<T>.Count != 0)
+                if ((uint)length % Vector128<TValue>.Count != 0)
                 {
                     current = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd);
-                    equals = N.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current) | Vector128.Equals(values2, current));
-                    if (equals != Vector128<T>.Zero)
+                    equals = TNegator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current) | Vector128.Equals(values2, current));
+                    if (equals != Vector128<TValue>.Zero)
                     {
                         return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
                     }
@@ -1869,31 +1869,31 @@ internal static int IndexOfAnyExceptValueType<T>(ref T searchSpace, T value0, T
             => IndexOfAnyValueType<T, Negate<T>>(ref searchSpace, value0, value1, value2, value3, length);
 
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T value1, T value2, T value3, int length)
-            where T : struct, INumber<T>
-            where N : struct, INegator<T>
+        private static int IndexOfAnyValueType<TValue, TNegator>(ref TValue searchSpace, TValue value0, TValue value1, TValue value2, TValue value3, int length)
+            where TValue : struct, INumber<TValue>
+            where TNegator : struct, INegator<TValue>
         {
             Debug.Assert(length >= 0, "Expected non-negative length");
             Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types");
 
-            if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
+            if (!Vector128.IsHardwareAccelerated || length < Vector128<TValue>.Count)
             {
                 nuint offset = 0;
-                T lookUp;
+                TValue lookUp;
 
                 while (length >= 4)
                 {
                     length -= 4;
 
-                    ref T current = ref Unsafe.Add(ref searchSpace, offset);
+                    ref TValue current = ref Unsafe.Add(ref searchSpace, offset);
                     lookUp = current;
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset;
                     lookUp = Unsafe.Add(ref current, 1);
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset + 1;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset + 1;
                     lookUp = Unsafe.Add(ref current, 2);
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset + 2;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset + 2;
                     lookUp = Unsafe.Add(ref current, 3);
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset + 3;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset + 3;
 
                     offset += 4;
                 }
@@ -1903,26 +1903,26 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
                     length -= 1;
 
                     lookUp = Unsafe.Add(ref searchSpace, offset);
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset;
 
                     offset += 1;
                 }
             }
-            else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
+            else if (Vector256.IsHardwareAccelerated && length >= Vector256<TValue>.Count)
             {
-                Vector256<T> equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1), values2 = Vector256.Create(value2), values3 = Vector256.Create(value3);
-                ref T currentSearchSpace = ref searchSpace;
-                ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256<T>.Count);
+                Vector256<TValue> equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1), values2 = Vector256.Create(value2), values3 = Vector256.Create(value3);
+                ref TValue currentSearchSpace = ref searchSpace;
+                ref TValue oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector256<TValue>.Count);
 
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
                     current = Vector256.LoadUnsafe(ref currentSearchSpace);
-                    equals = N.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current)
+                    equals = TNegator.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current)
                         | Vector256.Equals(values2, current) | Vector256.Equals(values3, current));
-                    if (equals == Vector256<T>.Zero)
+                    if (equals == Vector256<TValue>.Zero)
                     {
-                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector256<T>.Count);
+                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector256<TValue>.Count);
                         continue;
                     }
 
@@ -1931,12 +1931,12 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
                 while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd));
 
                 // If any elements remain, process the last vector in the search space.
-                if ((uint)length % Vector256<T>.Count != 0)
+                if ((uint)length % Vector256<TValue>.Count != 0)
                 {
                     current = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd);
-                    equals = N.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current)
+                    equals = TNegator.NegateIfNeeded(Vector256.Equals(values0, current) | Vector256.Equals(values1, current)
                         | Vector256.Equals(values2, current) | Vector256.Equals(values3, current));
-                    if (equals != Vector256<T>.Zero)
+                    if (equals != Vector256<TValue>.Zero)
                     {
                         return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
                     }
@@ -1944,19 +1944,19 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
             }
             else
             {
-                Vector128<T> equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1), values2 = Vector128.Create(value2), values3 = Vector128.Create(value3);
-                ref T currentSearchSpace = ref searchSpace;
-                ref T oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128<T>.Count);
+                Vector128<TValue> equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1), values2 = Vector128.Create(value2), values3 = Vector128.Create(value3);
+                ref TValue currentSearchSpace = ref searchSpace;
+                ref TValue oneVectorAwayFromEnd = ref Unsafe.Add(ref searchSpace, length - Vector128<TValue>.Count);
 
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
                     current = Vector128.LoadUnsafe(ref currentSearchSpace);
-                    equals = N.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current)
+                    equals = TNegator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current)
                         | Vector128.Equals(values2, current) | Vector128.Equals(values3, current));
-                    if (equals == Vector128<T>.Zero)
+                    if (equals == Vector128<TValue>.Zero)
                     {
-                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128<T>.Count);
+                        currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128<TValue>.Count);
                         continue;
                     }
 
@@ -1965,12 +1965,12 @@ private static int IndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T valu
                 while (!Unsafe.IsAddressGreaterThan(ref currentSearchSpace, ref oneVectorAwayFromEnd));
 
                 // If any elements remain, process the first vector in the search space.
-                if ((uint)length % Vector128<T>.Count != 0)
+                if ((uint)length % Vector128<TValue>.Count != 0)
                 {
                     current = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd);
-                    equals = N.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current)
+                    equals = TNegator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current)
                         | Vector128.Equals(values2, current) | Vector128.Equals(values3, current));
-                    if (equals != Vector128<T>.Zero)
+                    if (equals != Vector128<TValue>.Zero)
                     {
                         return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
                     }
@@ -2102,14 +2102,14 @@ internal static int LastIndexOfAnyExceptValueType<T>(ref T searchSpace, T value,
             => LastIndexOfValueType<T, Negate<T>>(ref searchSpace, value, length);
 
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        private static int LastIndexOfValueType<T, N>(ref T searchSpace, T value, int length)
-            where T : struct, INumber<T>
-            where N : struct, INegator<T>
+        private static int LastIndexOfValueType<TValue, TNegator>(ref TValue searchSpace, TValue value, int length)
+            where TValue : struct, INumber<TValue>
+            where TNegator : struct, INegator<TValue>
         {
             Debug.Assert(length >= 0, "Expected non-negative length");
             Debug.Assert(value is byte or short or int or long, "Expected caller to normalize to one of these types");
 
-            if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
+            if (!Vector128.IsHardwareAccelerated || length < Vector128<TValue>.Count)
             {
                 nuint offset = (nuint)length - 1;
 
@@ -2117,14 +2117,14 @@ private static int LastIndexOfValueType<T, N>(ref T searchSpace, T value, int le
                 {
                     length -= 8;
 
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset) == value)) return (int)offset;
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 1) == value)) return (int)offset - 1;
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 2) == value)) return (int)offset - 2;
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 3) == value)) return (int)offset - 3;
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 4) == value)) return (int)offset - 4;
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 5) == value)) return (int)offset - 5;
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 6) == value)) return (int)offset - 6;
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 7) == value)) return (int)offset - 7;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset) == value)) return (int)offset;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 1) == value)) return (int)offset - 1;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 2) == value)) return (int)offset - 2;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 3) == value)) return (int)offset - 3;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 4) == value)) return (int)offset - 4;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 5) == value)) return (int)offset - 5;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 6) == value)) return (int)offset - 6;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 7) == value)) return (int)offset - 7;
 
                     offset -= 8;
                 }
@@ -2133,10 +2133,10 @@ private static int LastIndexOfValueType<T, N>(ref T searchSpace, T value, int le
                 {
                     length -= 4;
 
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset) == value)) return (int)offset;
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 1) == value)) return (int)offset - 1;
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 2) == value)) return (int)offset - 2;
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 3) == value)) return (int)offset - 3;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset) == value)) return (int)offset;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 1) == value)) return (int)offset - 1;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 2) == value)) return (int)offset - 2;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset - 3) == value)) return (int)offset - 3;
 
                     offset -= 4;
                 }
@@ -2145,23 +2145,23 @@ private static int LastIndexOfValueType<T, N>(ref T searchSpace, T value, int le
                 {
                     length -= 1;
 
-                    if (N.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset).Equals(value))) return (int)offset;
+                    if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset).Equals(value))) return (int)offset;
 
                     offset -= 1;
                 }
             }
-            else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
+            else if (Vector256.IsHardwareAccelerated && length >= Vector256<TValue>.Count)
             {
-                Vector256<T> equals, values = Vector256.Create(value);
-                ref T currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256<T>.Count);
+                Vector256<TValue> equals, values = Vector256.Create(value);
+                ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256<TValue>.Count);
 
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
-                    equals = N.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref currentSearchSpace)));
-                    if (equals == Vector256<T>.Zero)
+                    equals = TNegator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref currentSearchSpace)));
+                    if (equals == Vector256<TValue>.Zero)
                     {
-                        currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<T>.Count);
+                        currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<TValue>.Count);
                         continue;
                     }
 
@@ -2170,10 +2170,10 @@ private static int LastIndexOfValueType<T, N>(ref T searchSpace, T value, int le
                 while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace));
 
                 // If any elements remain, process the first vector in the search space.
-                if ((uint)length % Vector256<T>.Count != 0)
+                if ((uint)length % Vector256<TValue>.Count != 0)
                 {
-                    equals = N.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref searchSpace)));
-                    if (equals != Vector256<T>.Zero)
+                    equals = TNegator.NegateIfNeeded(Vector256.Equals(values, Vector256.LoadUnsafe(ref searchSpace)));
+                    if (equals != Vector256<TValue>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
                     }
@@ -2181,16 +2181,16 @@ private static int LastIndexOfValueType<T, N>(ref T searchSpace, T value, int le
             }
             else
             {
-                Vector128<T> equals, values = Vector128.Create(value);
-                ref T currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128<T>.Count);
+                Vector128<TValue> equals, values = Vector128.Create(value);
+                ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128<TValue>.Count);
 
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
-                    equals = N.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref currentSearchSpace)));
-                    if (equals == Vector128<T>.Zero)
+                    equals = TNegator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref currentSearchSpace)));
+                    if (equals == Vector128<TValue>.Zero)
                     {
-                        currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<T>.Count);
+                        currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<TValue>.Count);
                         continue;
                     }
 
@@ -2199,10 +2199,10 @@ private static int LastIndexOfValueType<T, N>(ref T searchSpace, T value, int le
                 while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace));
 
                 // If any elements remain, process the first vector in the search space.
-                if ((uint)length % Vector128<T>.Count != 0)
+                if ((uint)length % Vector128<TValue>.Count != 0)
                 {
-                    equals = N.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref searchSpace)));
-                    if (equals != Vector128<T>.Zero)
+                    equals = TNegator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref searchSpace)));
+                    if (equals != Vector128<TValue>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
                     }
@@ -2221,41 +2221,41 @@ internal static int LastIndexOfAnyExceptValueType<T>(ref T searchSpace, T value0
             => LastIndexOfAnyValueType<T, Negate<T>>(ref searchSpace, value0, value1, length);
 
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T value1, int length)
-            where T : struct, INumber<T>
-            where N : struct, INegator<T>
+        private static int LastIndexOfAnyValueType<TValue, TNegator>(ref TValue searchSpace, TValue value0, TValue value1, int length)
+            where TValue : struct, INumber<TValue>
+            where TNegator : struct, INegator<TValue>
         {
             Debug.Assert(length >= 0, "Expected non-negative length");
             Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types");
 
-            if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
+            if (!Vector128.IsHardwareAccelerated || length < Vector128<TValue>.Count)
             {
                 nuint offset = (nuint)length - 1;
-                T lookUp;
+                TValue lookUp;
 
-                if (typeof(T) == typeof(byte)) // this optimization is beneficial only to byte
+                if (typeof(TValue) == typeof(byte)) // this optimization is beneficial only to byte
                 {
                     while (length >= 8)
                     {
                         length -= 8;
 
-                        ref T current = ref Unsafe.Add(ref searchSpace, offset);
+                        ref TValue current = ref Unsafe.Add(ref searchSpace, offset);
                         lookUp = current;
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset;
                         lookUp = Unsafe.Add(ref current, -1);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 1;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 1;
                         lookUp = Unsafe.Add(ref current, -2);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 2;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 2;
                         lookUp = Unsafe.Add(ref current, -3);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 3;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 3;
                         lookUp = Unsafe.Add(ref current, -4);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 4;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 4;
                         lookUp = Unsafe.Add(ref current, -5);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 5;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 5;
                         lookUp = Unsafe.Add(ref current, -6);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 6;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 6;
                         lookUp = Unsafe.Add(ref current, -7);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 7;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 7;
 
                         offset -= 8;
                     }
@@ -2265,15 +2265,15 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
                 {
                     length -= 4;
 
-                    ref T current = ref Unsafe.Add(ref searchSpace, offset);
+                    ref TValue current = ref Unsafe.Add(ref searchSpace, offset);
                     lookUp = current;
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset;
                     lookUp = Unsafe.Add(ref current, -1);
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 1;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 1;
                     lookUp = Unsafe.Add(ref current, -2);
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 2;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 2;
                     lookUp = Unsafe.Add(ref current, -3);
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 3;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset - 3;
 
                     offset -= 4;
                 }
@@ -2283,24 +2283,24 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
                     length -= 1;
 
                     lookUp = Unsafe.Add(ref searchSpace, offset);
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset;
 
                     offset -= 1;
                 }
             }
-            else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
+            else if (Vector256.IsHardwareAccelerated && length >= Vector256<TValue>.Count)
             {
-                Vector256<T> equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1);
-                ref T currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256<T>.Count);
+                Vector256<TValue> equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1);
+                ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256<TValue>.Count);
 
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
                     current = Vector256.LoadUnsafe(ref currentSearchSpace);
-                    equals = N.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1));
-                    if (equals == Vector256<T>.Zero)
+                    equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1));
+                    if (equals == Vector256<TValue>.Zero)
                     {
-                        currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<T>.Count);
+                        currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<TValue>.Count);
                         continue;
                     }
 
@@ -2309,11 +2309,11 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
                 while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace));
 
                 // If any elements remain, process the first vector in the search space.
-                if ((uint)length % Vector256<T>.Count != 0)
+                if ((uint)length % Vector256<TValue>.Count != 0)
                 {
                     current = Vector256.LoadUnsafe(ref searchSpace);
-                    equals = N.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1));
-                    if (equals != Vector256<T>.Zero)
+                    equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1));
+                    if (equals != Vector256<TValue>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
                     }
@@ -2321,17 +2321,17 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
             }
             else
             {
-                Vector128<T> equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1);
-                ref T currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128<T>.Count);
+                Vector128<TValue> equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1);
+                ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128<TValue>.Count);
 
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
                     current = Vector128.LoadUnsafe(ref currentSearchSpace);
-                    equals = N.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1));
-                    if (equals == Vector128<T>.Zero)
+                    equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1));
+                    if (equals == Vector128<TValue>.Zero)
                     {
-                        currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<T>.Count);
+                        currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<TValue>.Count);
                         continue;
                     }
 
@@ -2340,11 +2340,11 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
                 while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace));
 
                 // If any elements remain, process the first vector in the search space.
-                if ((uint)length % Vector128<T>.Count != 0)
+                if ((uint)length % Vector128<TValue>.Count != 0)
                 {
                     current = Vector128.LoadUnsafe(ref searchSpace);
-                    equals = N.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1));
-                    if (equals != Vector128<T>.Zero)
+                    equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1));
+                    if (equals != Vector128<TValue>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
                     }
@@ -2363,41 +2363,41 @@ internal static int LastIndexOfAnyExceptValueType<T>(ref T searchSpace, T value0
             => LastIndexOfAnyValueType<T, Negate<T>>(ref searchSpace, value0, value1, value2, length);
 
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T value1, T value2, int length)
-            where T : struct, INumber<T>
-            where N : struct, INegator<T>
+        private static int LastIndexOfAnyValueType<TValue, TNegator>(ref TValue searchSpace, TValue value0, TValue value1, TValue value2, int length)
+            where TValue : struct, INumber<TValue>
+            where TNegator : struct, INegator<TValue>
         {
             Debug.Assert(length >= 0, "Expected non-negative length");
             Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types");
 
-            if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
+            if (!Vector128.IsHardwareAccelerated || length < Vector128<TValue>.Count)
             {
                 nuint offset = (nuint)length - 1;
-                T lookUp;
+                TValue lookUp;
 
-                if (typeof(T) == typeof(byte)) // this optimization is beneficial only to byte
+                if (typeof(TValue) == typeof(byte)) // this optimization is beneficial only to byte
                 {
                     while (length >= 8)
                     {
                         length -= 8;
 
-                        ref T current = ref Unsafe.Add(ref searchSpace, offset);
+                        ref TValue current = ref Unsafe.Add(ref searchSpace, offset);
                         lookUp = current;
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset;
                         lookUp = Unsafe.Add(ref current, -1);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 1;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 1;
                         lookUp = Unsafe.Add(ref current, -2);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 2;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 2;
                         lookUp = Unsafe.Add(ref current, -3);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 3;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 3;
                         lookUp = Unsafe.Add(ref current, -4);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 4;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 4;
                         lookUp = Unsafe.Add(ref current, -5);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 5;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 5;
                         lookUp = Unsafe.Add(ref current, -6);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 6;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 6;
                         lookUp = Unsafe.Add(ref current, -7);
-                        if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 7;
+                        if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 7;
 
                         offset -= 8;
                     }
@@ -2407,15 +2407,15 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
                 {
                     length -= 4;
 
-                    ref T current = ref Unsafe.Add(ref searchSpace, offset);
+                    ref TValue current = ref Unsafe.Add(ref searchSpace, offset);
                     lookUp = current;
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset;
                     lookUp = Unsafe.Add(ref current, -1);
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 1;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 1;
                     lookUp = Unsafe.Add(ref current, -2);
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 2;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 2;
                     lookUp = Unsafe.Add(ref current, -3);
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 3;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset - 3;
 
                     offset -= 4;
                 }
@@ -2425,24 +2425,24 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
                     length -= 1;
 
                     lookUp = Unsafe.Add(ref searchSpace, offset);
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset;
 
                     offset -= 1;
                 }
             }
-            else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
+            else if (Vector256.IsHardwareAccelerated && length >= Vector256<TValue>.Count)
             {
-                Vector256<T> equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1), values2 = Vector256.Create(value2);
-                ref T currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256<T>.Count);
+                Vector256<TValue> equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1), values2 = Vector256.Create(value2);
+                ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256<TValue>.Count);
 
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
                     current = Vector256.LoadUnsafe(ref currentSearchSpace);
-                    equals = N.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) | Vector256.Equals(current, values2));
-                    if (equals == Vector256<T>.Zero)
+                    equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) | Vector256.Equals(current, values2));
+                    if (equals == Vector256<TValue>.Zero)
                     {
-                        currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<T>.Count);
+                        currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<TValue>.Count);
                         continue;
                     }
 
@@ -2451,11 +2451,11 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
                 while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace));
 
                 // If any elements remain, process the first vector in the search space.
-                if ((uint)length % Vector256<T>.Count != 0)
+                if ((uint)length % Vector256<TValue>.Count != 0)
                 {
                     current = Vector256.LoadUnsafe(ref searchSpace);
-                    equals = N.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) | Vector256.Equals(current, values2));
-                    if (equals != Vector256<T>.Zero)
+                    equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1) | Vector256.Equals(current, values2));
+                    if (equals != Vector256<TValue>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
                     }
@@ -2463,17 +2463,17 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
             }
             else
             {
-                Vector128<T> equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1), values2 = Vector128.Create(value2);
-                ref T currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128<T>.Count);
+                Vector128<TValue> equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1), values2 = Vector128.Create(value2);
+                ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128<TValue>.Count);
 
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
                     current = Vector128.LoadUnsafe(ref currentSearchSpace);
-                    equals = N.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) | Vector128.Equals(current, values2));
-                    if (equals == Vector128<T>.Zero)
+                    equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) | Vector128.Equals(current, values2));
+                    if (equals == Vector128<TValue>.Zero)
                     {
-                        currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<T>.Count);
+                        currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<TValue>.Count);
                         continue;
                     }
 
@@ -2482,11 +2482,11 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
                 while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace));
 
                 // If any elements remain, process the first vector in the search space.
-                if ((uint)length % Vector128<T>.Count != 0)
+                if ((uint)length % Vector128<TValue>.Count != 0)
                 {
                     current = Vector128.LoadUnsafe(ref searchSpace);
-                    equals = N.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) | Vector128.Equals(current, values2));
-                    if (equals != Vector128<T>.Zero)
+                    equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1) | Vector128.Equals(current, values2));
+                    if (equals != Vector128<TValue>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
                     }
@@ -2505,31 +2505,31 @@ internal static int LastIndexOfAnyExceptValueType<T>(ref T searchSpace, T value0
             => LastIndexOfAnyValueType<T, Negate<T>>(ref searchSpace, value0, value1, value2, value3, length);
 
         [MethodImpl(MethodImplOptions.AggressiveOptimization)]
-        private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T value1, T value2, T value3, int length)
-            where T : struct, INumber<T>
-            where N : struct, INegator<T>
+        private static int LastIndexOfAnyValueType<TValue, TNegator>(ref TValue searchSpace, TValue value0, TValue value1, TValue value2, TValue value3, int length)
+            where TValue : struct, INumber<TValue>
+            where TNegator : struct, INegator<TValue>
         {
             Debug.Assert(length >= 0, "Expected non-negative length");
             Debug.Assert(value0 is byte or short or int or long, "Expected caller to normalize to one of these types");
 
-            if (!Vector128.IsHardwareAccelerated || length < Vector128<T>.Count)
+            if (!Vector128.IsHardwareAccelerated || length < Vector128<TValue>.Count)
             {
                 nuint offset = (nuint)length - 1;
-                T lookUp;
+                TValue lookUp;
 
                 while (length >= 4)
                 {
                     length -= 4;
 
-                    ref T current = ref Unsafe.Add(ref searchSpace, offset);
+                    ref TValue current = ref Unsafe.Add(ref searchSpace, offset);
                     lookUp = current;
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset;
                     lookUp = Unsafe.Add(ref current, -1);
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset - 1;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset - 1;
                     lookUp = Unsafe.Add(ref current, -2);
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset - 2;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset - 2;
                     lookUp = Unsafe.Add(ref current, -3);
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset - 3;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset - 3;
 
                     offset -= 4;
                 }
@@ -2539,25 +2539,25 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
                     length -= 1;
 
                     lookUp = Unsafe.Add(ref searchSpace, offset);
-                    if (N.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset;
+                    if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2 || lookUp == value3)) return (int)offset;
 
                     offset -= 1;
                 }
             }
-            else if (Vector256.IsHardwareAccelerated && length >= Vector256<T>.Count)
+            else if (Vector256.IsHardwareAccelerated && length >= Vector256<TValue>.Count)
             {
-                Vector256<T> equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1), values2 = Vector256.Create(value2), values3 = Vector256.Create(value3);
-                ref T currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256<T>.Count);
+                Vector256<TValue> equals, current, values0 = Vector256.Create(value0), values1 = Vector256.Create(value1), values2 = Vector256.Create(value2), values3 = Vector256.Create(value3);
+                ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector256<TValue>.Count);
 
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
                     current = Vector256.LoadUnsafe(ref currentSearchSpace);
-                    equals = N.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1)
+                    equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1)
                                             | Vector256.Equals(current, values2) | Vector256.Equals(current, values3));
-                    if (equals == Vector256<T>.Zero)
+                    if (equals == Vector256<TValue>.Zero)
                     {
-                        currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<T>.Count);
+                        currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector256<TValue>.Count);
                         continue;
                     }
 
@@ -2566,12 +2566,12 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
                 while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace));
 
                 // If any elements remain, process the first vector in the search space.
-                if ((uint)length % Vector256<T>.Count != 0)
+                if ((uint)length % Vector256<TValue>.Count != 0)
                 {
                     current = Vector256.LoadUnsafe(ref searchSpace);
-                    equals = N.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1)
+                    equals = TNegator.NegateIfNeeded(Vector256.Equals(current, values0) | Vector256.Equals(current, values1)
                                             | Vector256.Equals(current, values2) | Vector256.Equals(current, values3));
-                    if (equals != Vector256<T>.Zero)
+                    if (equals != Vector256<TValue>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
                     }
@@ -2579,18 +2579,18 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
             }
             else
             {
-                Vector128<T> equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1), values2 = Vector128.Create(value2), values3 = Vector128.Create(value3);
-                ref T currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128<T>.Count);
+                Vector128<TValue> equals, current, values0 = Vector128.Create(value0), values1 = Vector128.Create(value1), values2 = Vector128.Create(value2), values3 = Vector128.Create(value3);
+                ref TValue currentSearchSpace = ref Unsafe.Add(ref searchSpace, length - Vector128<TValue>.Count);
 
                 // Loop until either we've finished all elements or there's less than a vector's-worth remaining.
                 do
                 {
                     current = Vector128.LoadUnsafe(ref currentSearchSpace);
-                    equals = N.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1)
+                    equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1)
                                             | Vector128.Equals(current, values2) | Vector128.Equals(current, values3));
-                    if (equals == Vector128<T>.Zero)
+                    if (equals == Vector128<TValue>.Zero)
                     {
-                        currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<T>.Count);
+                        currentSearchSpace = ref Unsafe.Subtract(ref currentSearchSpace, Vector128<TValue>.Count);
                         continue;
                     }
 
@@ -2599,12 +2599,12 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
                 while (!Unsafe.IsAddressLessThan(ref currentSearchSpace, ref searchSpace));
 
                 // If any elements remain, process the first vector in the search space.
-                if ((uint)length % Vector128<T>.Count != 0)
+                if ((uint)length % Vector128<TValue>.Count != 0)
                 {
                     current = Vector128.LoadUnsafe(ref searchSpace);
-                    equals = N.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1)
+                    equals = TNegator.NegateIfNeeded(Vector128.Equals(current, values0) | Vector128.Equals(current, values1)
                                             | Vector128.Equals(current, values2) | Vector128.Equals(current, values3));
-                    if (equals != Vector128<T>.Zero)
+                    if (equals != Vector128<TValue>.Zero)
                     {
                         return ComputeLastIndex(ref searchSpace, ref searchSpace, equals);
                     }
@@ -2614,7 +2614,6 @@ private static int LastIndexOfAnyValueType<T, N>(ref T searchSpace, T value0, T
             return -1;
         }
 
-
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private static int ComputeFirstIndex<T>(ref T searchSpace, ref T current, Vector128<T> equals) where T : struct
         {

From 0c6b01d85a236ce1fce9f3bd374fd6c11a1a1979 Mon Sep 17 00:00:00 2001
From: Adam Sitnik <adam.sitnik@gmail.com>
Date: Wed, 17 Aug 2022 10:26:26 +0200
Subject: [PATCH 34/34] reduce the regression for IndexOfAnyExcept(3) and
 LastIndexOfAnyExcept(3) for small inputs by always calling the heavily
 optimized SpanHelpers.IndexOfAnyExceptValueType

---
 .../System.Private.CoreLib/src/System/MemoryExtensions.cs     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
index dcdddefa9defe6..5ae66529eb20db 100644
--- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
+++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs
@@ -616,7 +616,7 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int IndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value0, T value1, T value2) where T : IEquatable<T>?
         {
-            if (SpanHelpers.CanVectorizeAndBenefit<T>(span.Length))
+            if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {
@@ -859,7 +859,7 @@ ref Unsafe.As<T, short>(ref MemoryMarshal.GetReference(span)),
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static int LastIndexOfAnyExcept<T>(this ReadOnlySpan<T> span, T value0, T value1, T value2) where T : IEquatable<T>?
         {
-            if (SpanHelpers.CanVectorizeAndBenefit<T>(span.Length))
+            if (RuntimeHelpers.IsBitwiseEquatable<T>())
             {
                 if (Unsafe.SizeOf<T>() == sizeof(byte))
                 {