bpo-37630: Use SHA3 and SHAKE XOF from OpenSSL (GH-16049)

OpenSSL 1.1.1 comes with SHA3 and SHAKE builtin.

Signed-off-by: Christian Heimes <[email protected]>

Automerge-Triggered-By: @tiran

Author: tiran

Doc/library/hashlib.rst

@@ -87,6 +87,8 @@ library that Python uses on your platform. On most platforms the
    that the hashing algorithm is not used in a security context, e.g. as a
    non-cryptographic one-way compression function.
 
+   Hashlib now uses SHA3 and SHAKE from OpenSSL 1.1.1 and newer.
+
 For example, to obtain the digest of the byte string ``b'Nobody inspects the
 spammish repetition'``::
 

Lib/hashlib.py

@@ -71,8 +71,6 @@
 __builtin_constructor_cache = {}
 
 __block_openssl_constructor = {
-    'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512',
-    'shake_128', 'shake_256',
     'blake2b', 'blake2s',
 }
 
@@ -125,6 +123,8 @@ def __get_openssl_constructor(name):
         # Prefer our blake2 and sha3 implementation.
         return __get_builtin_constructor(name)
     try:
+        # MD5, SHA1, and SHA2 are in all supported OpenSSL versions
+        # SHA3/shake are available in OpenSSL 1.1.1+
         f = getattr(_hashlib, 'openssl_' + name)
         # Allow the C module to raise ValueError.  The function will be
         # defined but the hash not actually available thanks to OpenSSL.

Lib/test/test_hashlib.py

@@ -27,9 +27,10 @@
 py_hashlib = import_fresh_module('hashlib', blocked=['_hashlib'])
 
 try:
-    from _hashlib import HASH
+    from _hashlib import HASH, HASHXOF
 except ImportError:
     HASH = None
+    HASHXOF = None
 
 try:
     import _blake2
@@ -254,6 +255,9 @@ def test_digest_length_overflow(self):
             h = cons()
             if h.name not in self.shakes:
                 continue
+            if HASH is not None and isinstance(h, HASH):
+                # _hashopenssl's take a size_t
+                continue
             for digest in h.digest, h.hexdigest:
                 self.assertRaises(ValueError, digest, -10)
                 for length in large_sizes:
@@ -860,6 +864,18 @@ def hash_in_chunks(chunk_size):
     def test_get_fips_mode(self):
         self.assertIsInstance(c_hashlib.get_fips_mode(), int)
 
+    @unittest.skipUnless(HASH is not None, 'need _hashlib')
+    def test_internal_types(self):
+        # internal types like _hashlib.HASH are not constructable
+        with self.assertRaisesRegex(
+            TypeError, "cannot create 'HASH' instance"
+        ):
+            HASH()
+        with self.assertRaisesRegex(
+            TypeError, "cannot create 'HASHXOF' instance"
+        ):
+            HASHXOF()
+
 
 class KDFTests(unittest.TestCase):
 

Misc/NEWS.d/next/Library/2020-05-15-19-53-18.bpo-37630.O5kgAw.rst

@@ -0,0 +1,2 @@
+The :mod:`hashlib` module can now use SHA3 hashes and SHAKE XOF from OpenSSL
+when available.