refactor, build: Upstream fixes for the /crypto files. Implement Keccak and SHA3

configure.ac

@@ -702,7 +702,21 @@ AC_CHECK_DECLS([bswap_16, bswap_32, bswap_64],,,
                  #include <byteswap.h>
                  #endif])
 
-AC_CHECK_DECLS([__builtin_clz, __builtin_clzl, __builtin_clzll])
+AC_MSG_CHECKING(for __builtin_clzl)
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[
+ (void) __builtin_clzl(0);
+  ]])],
+ [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_BUILTIN_CLZL, 1, [Define this symbol if you have __builtin_clzl])],
+ [ AC_MSG_RESULT(no)]
+)
+
+AC_MSG_CHECKING(for __builtin_clzll)
+AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[
+  (void) __builtin_clzll(0);
+  ]])],
+ [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_BUILTIN_CLZLL, 1, [Define this symbol if you have __builtin_clzll])],
+ [ AC_MSG_RESULT(no)]
+)
 
 dnl Check for MSG_NOSIGNAL
 AC_MSG_CHECKING(for MSG_NOSIGNAL)

src/Makefile.am

@@ -76,6 +76,7 @@ GRIDCOIN_CORE_H = \
     compat.h \
     compat/assumptions.h \
     compat/byteswap.h \
+    compat/cpuid.h \
     compat/endian.h \
     consensus/consensus.h \
     consensus/merkle.h \
@@ -321,16 +322,22 @@ crypto_libgridcoin_crypto_base_a_SOURCES = \
     crypto/common.h \
     crypto/hmac_sha256.cpp \
     crypto/hmac_sha256.h \
+    crypto/sha3.cpp \
+    crypto/sha3.h \
     crypto/hmac_sha512.cpp \
     crypto/hmac_sha512.h \
+    crypto/poly1305.h \
+    crypto/poly1305.cpp \
     crypto/ripemd160.cpp \
     crypto/ripemd160.h \
     crypto/sha1.cpp \
     crypto/sha1.h \
     crypto/sha256.cpp \
     crypto/sha256.h \
     crypto/sha512.cpp \
-    crypto/sha512.h
+    crypto/sha512.h \
+    crypto/siphash.cpp \
+    crypto/siphash.h
 
 if USE_ASM
 crypto_libgridcoin_crypto_base_a_SOURCES += crypto/sha256_sse4.cpp

src/compat/cpuid.h

@@ -0,0 +1,24 @@
+// Copyright (c) 2017-2019 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_COMPAT_CPUID_H
+#define BITCOIN_COMPAT_CPUID_H
+
+#if defined(__x86_64__) || defined(__amd64__) || defined(__i386__)
+#define HAVE_GETCPUID
+
+#include <cpuid.h>
+
+// We can't use cpuid.h's __get_cpuid as it does not support subleafs.
+void static inline GetCPUID(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d)
+{
+#ifdef __GNUC__
+    __cpuid_count(leaf, subleaf, a, b, c, d);
+#else
+  __asm__ ("cpuid" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(leaf), "2"(subleaf));
+#endif
+}
+
+#endif // defined(__x86_64__) || defined(__amd64__) || defined(__i386__)
+#endif // BITCOIN_COMPAT_CPUID_H

src/crypto/chacha20.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2017 The Bitcoin Core developers
+// Copyright (c) 2017-2019 The Bitcoin Core developers
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or https://opensource.org/licenses/mit-license.php.
 
@@ -71,7 +71,7 @@ void ChaCha20::Seek(uint64_t pos)
     input[13] = pos >> 32;
 }
 
-void ChaCha20::Output(unsigned char* c, size_t bytes)
+void ChaCha20::Keystream(unsigned char* c, size_t bytes)
 {
     uint32_t x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15;
     uint32_t j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15;
@@ -178,3 +178,133 @@ void ChaCha20::Output(unsigned char* c, size_t bytes)
         c += 64;
     }
 }
+
+void ChaCha20::Crypt(const unsigned char* m, unsigned char* c, size_t bytes)
+{
+    uint32_t x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15;
+    uint32_t j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15;
+    unsigned char *ctarget = nullptr;
+    unsigned char tmp[64];
+    unsigned int i;
+
+    if (!bytes) return;
+
+    j0 = input[0];
+    j1 = input[1];
+    j2 = input[2];
+    j3 = input[3];
+    j4 = input[4];
+    j5 = input[5];
+    j6 = input[6];
+    j7 = input[7];
+    j8 = input[8];
+    j9 = input[9];
+    j10 = input[10];
+    j11 = input[11];
+    j12 = input[12];
+    j13 = input[13];
+    j14 = input[14];
+    j15 = input[15];
+
+    for (;;) {
+        if (bytes < 64) {
+            // if m has fewer than 64 bytes available, copy m to tmp and
+            // read from tmp instead
+            for (i = 0;i < bytes;++i) tmp[i] = m[i];
+            m = tmp;
+            ctarget = c;
+            c = tmp;
+        }
+        x0 = j0;
+        x1 = j1;
+        x2 = j2;
+        x3 = j3;
+        x4 = j4;
+        x5 = j5;
+        x6 = j6;
+        x7 = j7;
+        x8 = j8;
+        x9 = j9;
+        x10 = j10;
+        x11 = j11;
+        x12 = j12;
+        x13 = j13;
+        x14 = j14;
+        x15 = j15;
+        for (i = 20;i > 0;i -= 2) {
+            QUARTERROUND( x0, x4, x8,x12)
+            QUARTERROUND( x1, x5, x9,x13)
+            QUARTERROUND( x2, x6,x10,x14)
+            QUARTERROUND( x3, x7,x11,x15)
+            QUARTERROUND( x0, x5,x10,x15)
+            QUARTERROUND( x1, x6,x11,x12)
+            QUARTERROUND( x2, x7, x8,x13)
+            QUARTERROUND( x3, x4, x9,x14)
+        }
+        x0 += j0;
+        x1 += j1;
+        x2 += j2;
+        x3 += j3;
+        x4 += j4;
+        x5 += j5;
+        x6 += j6;
+        x7 += j7;
+        x8 += j8;
+        x9 += j9;
+        x10 += j10;
+        x11 += j11;
+        x12 += j12;
+        x13 += j13;
+        x14 += j14;
+        x15 += j15;
+
+        x0 ^= ReadLE32(m + 0);
+        x1 ^= ReadLE32(m + 4);
+        x2 ^= ReadLE32(m + 8);
+        x3 ^= ReadLE32(m + 12);
+        x4 ^= ReadLE32(m + 16);
+        x5 ^= ReadLE32(m + 20);
+        x6 ^= ReadLE32(m + 24);
+        x7 ^= ReadLE32(m + 28);
+        x8 ^= ReadLE32(m + 32);
+        x9 ^= ReadLE32(m + 36);
+        x10 ^= ReadLE32(m + 40);
+        x11 ^= ReadLE32(m + 44);
+        x12 ^= ReadLE32(m + 48);
+        x13 ^= ReadLE32(m + 52);
+        x14 ^= ReadLE32(m + 56);
+        x15 ^= ReadLE32(m + 60);
+
+        ++j12;
+        if (!j12) ++j13;
+
+        WriteLE32(c + 0, x0);
+        WriteLE32(c + 4, x1);
+        WriteLE32(c + 8, x2);
+        WriteLE32(c + 12, x3);
+        WriteLE32(c + 16, x4);
+        WriteLE32(c + 20, x5);
+        WriteLE32(c + 24, x6);
+        WriteLE32(c + 28, x7);
+        WriteLE32(c + 32, x8);
+        WriteLE32(c + 36, x9);
+        WriteLE32(c + 40, x10);
+        WriteLE32(c + 44, x11);
+        WriteLE32(c + 48, x12);
+        WriteLE32(c + 52, x13);
+        WriteLE32(c + 56, x14);
+        WriteLE32(c + 60, x15);
+
+        if (bytes <= 64) {
+            if (bytes < 64) {
+                for (i = 0;i < bytes;++i) ctarget[i] = c[i];
+            }
+            input[12] = j12;
+            input[13] = j13;
+            return;
+        }
+        bytes -= 64;
+        c += 64;
+        m += 64;
+    }
+}

src/crypto/chacha20.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2017 The Bitcoin Core developers
+// Copyright (c) 2017-2019 The Bitcoin Core developers
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or https://opensource.org/licenses/mit-license.php.
 
@@ -8,7 +8,8 @@
 #include <stdint.h>
 #include <stdlib.h>
 
-/** A PRNG class for ChaCha20. */
+/** A class for ChaCha20 256-bit stream cipher developed by Daniel J. Bernstein
+    https://cr.yp.to/chacha/chacha-20080128.pdf */
 class ChaCha20
 {
 private:
@@ -17,10 +18,17 @@ class ChaCha20
 public:
     ChaCha20();
     ChaCha20(const unsigned char* key, size_t keylen);
-    void SetKey(const unsigned char* key, size_t keylen);
-    void SetIV(uint64_t iv);
-    void Seek(uint64_t pos);
-    void Output(unsigned char* output, size_t bytes);
+    void SetKey(const unsigned char* key, size_t keylen); //!< set key with flexible keylength; 256bit recommended */
+    void SetIV(uint64_t iv); // set the 64bit nonce
+    void Seek(uint64_t pos); // set the 64bit block counter
+
+    /** outputs the keystream of size <bytes> into <c> */
+    void Keystream(unsigned char* c, size_t bytes);
+
+    /** enciphers the message <input> of length <bytes> and write the enciphered representation into <output>
+     *  Used for encryption and decryption (XOR)
+     */
+    void Crypt(const unsigned char* input, unsigned char* output, size_t bytes);
 };
 
 #endif // BITCOIN_CRYPTO_CHACHA20_H

src/crypto/common.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018 The Bitcoin Core developers
+// Copyright (c) 2014-2020 The Bitcoin Core developers
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or https://opensource.org/licenses/mit-license.php.
 
@@ -82,12 +82,12 @@ void static inline WriteBE64(unsigned char* ptr, uint64_t x)
 /** Return the smallest number n such that (x >> n) == 0 (or 64 if the highest bit in x is set. */
 uint64_t static inline CountBits(uint64_t x)
 {
-#if HAVE_DECL___BUILTIN_CLZL
+#if HAVE_BUILTIN_CLZL
     if (sizeof(unsigned long) >= sizeof(uint64_t)) {
         return x ? 8 * sizeof(unsigned long) - __builtin_clzl(x) : 0;
     }
 #endif
-#if HAVE_DECL___BUILTIN_CLZLL
+#if HAVE_BUILTIN_CLZLL
     if (sizeof(unsigned long long) >= sizeof(uint64_t)) {
         return x ? 8 * sizeof(unsigned long long) - __builtin_clzll(x) : 0;
     }

src/crypto/ripemd160.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018 The Bitcoin Core developers
+// Copyright (c) 2014-2019 The Bitcoin Core developers
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or https://opensource.org/licenses/mit-license.php.
 
@@ -256,7 +256,7 @@ CRIPEMD160& CRIPEMD160::Write(const unsigned char* data, size_t len)
         ripemd160::Transform(s, buf);
         bufsize = 0;
     }
-    while (end >= data + 64) {
+    while (end - data >= 64) {
         // Process full chunks directly from the source.
         ripemd160::Transform(s, data);
         bytes += 64;

src/crypto/sha1.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018 The Bitcoin Core developers
+// Copyright (c) 2014-2019 The Bitcoin Core developers
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or https://opensource.org/licenses/mit-license.php.
 
@@ -163,7 +163,7 @@ CSHA1& CSHA1::Write(const unsigned char* data, size_t len)
         sha1::Transform(s, buf);
         bufsize = 0;
     }
-    while (end >= data + 64) {
+    while (end - data >= 64) {
         // Process full chunks directly from the source.
         sha1::Transform(s, data);
         bytes += 64;

src/crypto/sha256.cpp

@@ -9,9 +9,10 @@
 #include <string.h>
 #include <atomic>
 
+#include <compat/cpuid.h>
+
 #if defined(__x86_64__) || defined(__amd64__) || defined(__i386__)
 #if defined(USE_ASM)
-#include <cpuid.h>
 namespace sha256_sse4
 {
 void Transform(uint32_t* s, const unsigned char* chunk, size_t blocks);
@@ -547,18 +548,7 @@ bool SelfTest() {
     return true;
 }
 
-
 #if defined(USE_ASM) && (defined(__x86_64__) || defined(__amd64__) || defined(__i386__))
-// We can't use cpuid.h's __get_cpuid as it does not support subleafs.
-void inline cpuid(uint32_t leaf, uint32_t subleaf, uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d)
-{
-#ifdef __GNUC__
-    __cpuid_count(leaf, subleaf, a, b, c, d);
-#else
-  __asm__ ("cpuid" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(leaf), "2"(subleaf));
-#endif
-}
-
 /** Check whether the OS has enabled AVX registers. */
 bool AVXEnabled()
 {
@@ -573,7 +563,7 @@ bool AVXEnabled()
 std::string SHA256AutoDetect()
 {
     std::string ret = "standard";
-#if defined(USE_ASM) && (defined(__x86_64__) || defined(__amd64__) || defined(__i386__))
+#if defined(USE_ASM) && defined(HAVE_GETCPUID)
     bool have_sse4 = false;
     bool have_xsave = false;
     bool have_avx = false;
@@ -590,15 +580,15 @@ std::string SHA256AutoDetect()
     (void)enabled_avx;
 
     uint32_t eax, ebx, ecx, edx;
-    cpuid(1, 0, eax, ebx, ecx, edx);
+    GetCPUID(1, 0, eax, ebx, ecx, edx);
     have_sse4 = (ecx >> 19) & 1;
     have_xsave = (ecx >> 27) & 1;
     have_avx = (ecx >> 28) & 1;
     if (have_xsave && have_avx) {
         enabled_avx = AVXEnabled();
     }
     if (have_sse4) {
-        cpuid(7, 0, eax, ebx, ecx, edx);
+        GetCPUID(7, 0, eax, ebx, ecx, edx);
         have_avx2 = (ebx >> 5) & 1;
         have_shani = (ebx >> 29) & 1;
     }

src/crypto/sha256.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018 The Bitcoin Core developers
+// Copyright (c) 2014-2019 The Bitcoin Core developers
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or https://opensource.org/licenses/mit-license.php.
 
@@ -24,6 +24,7 @@ class CSHA256
     CSHA256& Write(const unsigned char* data, size_t len);
     void Finalize(unsigned char hash[OUTPUT_SIZE]);
     CSHA256& Reset();
+    uint64_t Size() const { return bytes; }
 };
 
 /** Autodetect the best available SHA256 implementation.