Improve aarch64 crc32c on short-input

Summary:
We are introducing a specialized crc32c routine for small inputs

Similar to D95478675 but for crc32c

Before:

crc32c_8                                                    2.78ns   359.63M
crc32c_16                                                   2.74ns   364.88M
crc32c_32                                                   2.75ns   363.37M
crc32c_64                                                   3.34ns   299.11M
crc32c_128                                                  5.77ns   173.31M
crc32c_256                                                 10.93ns    91.49M
crc32c_384                                                 15.80ns    63.31M
crc32c_512                                                 20.65ns    48.42M
crc32c_1024                                                40.15ns    24.91M

After:

crc32c_8                                                    2.65ns   377.68M
crc32c_16                                                   2.72ns   366.98M
crc32c_32                                                   2.72ns   367.79M
crc32c_64                                                   3.32ns   300.90M
crc32c_128                                                  4.53ns   220.59M
crc32c_256                                                  9.37ns   106.76M
crc32c_384                                                 14.20ns    70.43M
crc32c_512                                                 18.53ns    53.96M
crc32c_1024                                                34.27ns    29.18M

Reviewed By: yfeldblum

Differential Revision: D95633651

fbshipit-source-id: 4b783de6478b0ec4069eff0055607e07559221f1

Author: Nicoshev

folly/external/fast-crc32/neon_eor3_crc32c_v8s2x4e_s2x1.cpp

@@ -25,6 +25,10 @@ CRC_EXPORT uint32_t neon_eor3_crc32c_v8s2x4e_s2x1(const uint8_t*, size_t, uint32
   abort(); // not implemented on this platform
 }
 
+CRC_EXPORT uint32_t neon_eor3_crc32c_small(const uint8_t*, size_t, uint32_t) {
+  abort(); // not implemented on this platform
+}
+
 CRC_EXPORT bool has_neon_eor3_crc32c_v8s2x4e_s2x1() {
   return false;
 }
@@ -85,6 +89,75 @@ CRC_EXPORT bool has_neon_eor3_crc32c_v8s2x4e_s2x1() {
       caps.aarch64_crc32() && caps.aarch64_sha3();
 }
 
+// Mix v2s1x2 and s1x2
+CRC_EXPORT uint32_t neon_eor3_crc32c_small(const uint8_t* buf, size_t len, uint32_t crc0) {
+  for (; len && ((uintptr_t)buf & 7); --len) {
+    crc0 = __crc32cb(crc0, *buf++);
+  }
+  if (len > 384) {
+    if (((uintptr_t)buf & 8) && len >= 8) {
+      crc0 = __crc32cd(crc0, *(const uint64_t*)buf);
+      buf += 8;
+      len -= 8;
+    }
+    size_t blk = (len - 0) / 48;
+    size_t klen = blk * 16;
+    const uint8_t* buf2 = buf + klen;
+    uint64x2_t vc0;
+    uint64_t vc;
+    /* First vector chunk. */
+    uint64x2_t x0 = vld1q_u64((const uint64_t*)buf2), y0;
+    uint64x2_t x1 = vld1q_u64((const uint64_t*)(buf2 + 16)), y1;
+    uint64x2_t k;
+    { static const uint64_t CRC_ALIGN(16) k_[] = {0x3da6d0cb, 0xba4fc28e}; k = vld1q_u64(k_); }
+    buf2 += 32;
+    len -= 48;
+    /* Main loop. */
+    while (len >= 48) {
+      y0 = clmul_lo(x0, k), x0 = clmul_hi(x0, k);
+      y1 = clmul_lo(x1, k), x1 = clmul_hi(x1, k);
+      x0 = veor3q_u64(x0, y0, vld1q_u64((const uint64_t*)buf2));
+      x1 = veor3q_u64(x1, y1, vld1q_u64((const uint64_t*)(buf2 + 16)));
+      crc0 = __crc32cd(crc0, *(const uint64_t*)buf);
+      crc0 = __crc32cd(crc0, *(const uint64_t*)(buf + 8));
+      buf += 16;
+      buf2 += 32;
+      len -= 48;
+    }
+    /* Reduce x0 ... x1 to just x0. */
+    { static const uint64_t CRC_ALIGN(16) k_[] = {0xf20c0dfe, 0x493c7d27}; k = vld1q_u64(k_); }
+    y0 = clmul_lo(x0, k), x0 = clmul_hi(x0, k);
+    x0 = veor3q_u64(x0, y0, x1);
+    /* Final scalar chunk. */
+    crc0 = __crc32cd(crc0, *(const uint64_t*)buf);
+    crc0 = __crc32cd(crc0, *(const uint64_t*)(buf + 8));
+    vc0 = crc_shift(crc0, 0 + blk * 32);
+    vc = vgetq_lane_u64(vc0, 0);
+    /* Reduce 128 bits to 32 bits, and multiply by x^32. */
+    crc0 = __crc32cd(0, vgetq_lane_u64(x0, 0));
+    crc0 = __crc32cd(crc0, vc ^ vgetq_lane_u64(x0, 1));
+    buf = buf2;
+  }
+  if (len >= 16) {
+    /* Main loop. */
+    do {
+      crc0 = __crc32cd(crc0, *(const uint64_t*)buf);
+      crc0 = __crc32cd(crc0, *(const uint64_t*)(buf + 8));
+      buf += 16;
+      len -= 16;
+    } while (len >= 16);
+  }
+  if (len >= 8) {
+    crc0 = __crc32cd(crc0, *(const uint64_t*)buf);
+    len -= 8;
+    buf += 8;
+  }
+  for (; len; --len) {
+    crc0 = __crc32cb(crc0, *buf++);
+  }
+  return crc0;
+}
+
 CRC_EXPORT uint32_t neon_eor3_crc32c_v8s2x4e_s2x1(const uint8_t* buf, size_t len, uint32_t crc0) {
   for (; len && ((uintptr_t)buf & 7); --len) {
     crc0 = __crc32cb(crc0, *buf++);

folly/external/fast-crc32/neon_eor3_crc32c_v8s2x4e_s2x1.h

@@ -4,5 +4,6 @@
 
 namespace folly::detail {
 uint32_t neon_eor3_crc32c_v8s2x4e_s2x1(const uint8_t* buf, size_t len, uint32_t crc0);
+uint32_t neon_eor3_crc32c_small(const uint8_t* buf, size_t len, uint32_t crc0);
 bool has_neon_eor3_crc32c_v8s2x4e_s2x1();
 }

folly/hash/Checksum.cpp

@@ -221,9 +221,13 @@ uint32_t crc32c(const uint8_t* data, size_t nbytes, uint32_t startingChecksum) {
 #endif
 
 #if FOLLY_AARCH64
-  if (nbytes >= 2048 && detail::crc32c_hw_supported_neon_eor3_sha3()) {
-    return detail::neon_eor3_crc32c_v8s2x4e_s2x1(
-        data, nbytes, startingChecksum);
+  if (detail::crc32c_hw_supported_neon_eor3_sha3()) {
+    if (nbytes < 1536) {
+      return detail::neon_eor3_crc32c_small(data, nbytes, startingChecksum);
+    } else {
+      return detail::neon_eor3_crc32c_v8s2x4e_s2x1(
+          data, nbytes, startingChecksum);
+    }
   }
 
   if (nbytes >= 4096 && detail::crc32c_hw_supported_neon()) {