Skip to content

Commit a0c3bb8

Browse files
tomasr8Glyphack
tomasr8
authored and
Glyphack
committed
pythongh-111916: Make hashlib related modules thread-safe without the GIL (python#111981)
Always use an individual lock on hash objects when in free-threaded builds. Fixes python#111916
1 parent 4a23bcc commit a0c3bb8

File tree

10 files changed

+188
-137
lines changed

10 files changed

+188
-137
lines changed

‎Misc/NEWS.d/next/Core and Builtins/2023-11-14-22-12-11.gh-issue-111916.ZGCayL.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Make hashlib related modules thread-safe without the GIL

‎Modules/_blake2/blake2b_impl.c

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
# define Py_BUILD_CORE_MODULE 1
1818
#endif
1919

20+
#include <stdbool.h>
2021
#include "Python.h"
2122
#include "pycore_strhex.h" // _Py_strhex()
2223

@@ -42,7 +43,8 @@ typedef struct {
4243
PyObject_HEAD
4344
blake2b_param param;
4445
blake2b_state state;
45-
PyThread_type_lock lock;
46+
bool use_mutex;
47+
PyMutex mutex;
4648
} BLAKE2bObject;
4749

4850
#include "clinic/blake2b_impl.c.h"
@@ -59,9 +61,11 @@ new_BLAKE2bObject(PyTypeObject *type)
5961
{
6062
BLAKE2bObject *self;
6163
self = (BLAKE2bObject *)type->tp_alloc(type, 0);
62-
if (self != NULL) {
63-
self->lock = NULL;
64+
if (self == NULL) {
65+
return NULL;
6466
}
67+
HASHLIB_INIT_MUTEX(self);
68+
6569
return self;
6670
}
6771

@@ -278,18 +282,19 @@ _blake2_blake2b_update(BLAKE2bObject *self, PyObject *data)
278282

279283
GET_BUFFER_VIEW_OR_ERROUT(data, &buf);
280284

281-
if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE)
282-
self->lock = PyThread_allocate_lock();
283-
284-
if (self->lock != NULL) {
285-
Py_BEGIN_ALLOW_THREADS
286-
PyThread_acquire_lock(self->lock, 1);
287-
blake2b_update(&self->state, buf.buf, buf.len);
288-
PyThread_release_lock(self->lock);
289-
Py_END_ALLOW_THREADS
285+
if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) {
286+
self->use_mutex = true;
287+
}
288+
if (self->use_mutex) {
289+
Py_BEGIN_ALLOW_THREADS
290+
PyMutex_Lock(&self->mutex);
291+
blake2b_update(&self->state, buf.buf, buf.len);
292+
PyMutex_Unlock(&self->mutex);
293+
Py_END_ALLOW_THREADS
290294
} else {
291295
blake2b_update(&self->state, buf.buf, buf.len);
292296
}
297+
293298
PyBuffer_Release(&buf);
294299

295300
Py_RETURN_NONE;
@@ -389,10 +394,6 @@ py_blake2b_dealloc(PyObject *self)
389394
/* Try not to leave state in memory. */
390395
secure_zero_memory(&obj->param, sizeof(obj->param));
391396
secure_zero_memory(&obj->state, sizeof(obj->state));
392-
if (obj->lock) {
393-
PyThread_free_lock(obj->lock);
394-
obj->lock = NULL;
395-
}
396397

397398
PyTypeObject *type = Py_TYPE(self);
398399
PyObject_Free(self);

‎Modules/_blake2/blake2s_impl.c

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
# define Py_BUILD_CORE_MODULE 1
1818
#endif
1919

20+
#include <stdbool.h>
2021
#include "Python.h"
2122
#include "pycore_strhex.h" // _Py_strhex()
2223

@@ -42,7 +43,8 @@ typedef struct {
4243
PyObject_HEAD
4344
blake2s_param param;
4445
blake2s_state state;
45-
PyThread_type_lock lock;
46+
bool use_mutex;
47+
PyMutex mutex;
4648
} BLAKE2sObject;
4749

4850
#include "clinic/blake2s_impl.c.h"
@@ -59,9 +61,11 @@ new_BLAKE2sObject(PyTypeObject *type)
5961
{
6062
BLAKE2sObject *self;
6163
self = (BLAKE2sObject *)type->tp_alloc(type, 0);
62-
if (self != NULL) {
63-
self->lock = NULL;
64+
if (self == NULL) {
65+
return NULL;
6466
}
67+
HASHLIB_INIT_MUTEX(self);
68+
6569
return self;
6670
}
6771

@@ -278,18 +282,19 @@ _blake2_blake2s_update(BLAKE2sObject *self, PyObject *data)
278282

279283
GET_BUFFER_VIEW_OR_ERROUT(data, &buf);
280284

281-
if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE)
282-
self->lock = PyThread_allocate_lock();
283-
284-
if (self->lock != NULL) {
285-
Py_BEGIN_ALLOW_THREADS
286-
PyThread_acquire_lock(self->lock, 1);
287-
blake2s_update(&self->state, buf.buf, buf.len);
288-
PyThread_release_lock(self->lock);
289-
Py_END_ALLOW_THREADS
285+
if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) {
286+
self->use_mutex = true;
287+
}
288+
if (self->use_mutex) {
289+
Py_BEGIN_ALLOW_THREADS
290+
PyMutex_Lock(&self->mutex);
291+
blake2s_update(&self->state, buf.buf, buf.len);
292+
PyMutex_Unlock(&self->mutex);
293+
Py_END_ALLOW_THREADS
290294
} else {
291295
blake2s_update(&self->state, buf.buf, buf.len);
292296
}
297+
293298
PyBuffer_Release(&buf);
294299

295300
Py_RETURN_NONE;
@@ -389,10 +394,6 @@ py_blake2s_dealloc(PyObject *self)
389394
/* Try not to leave state in memory. */
390395
secure_zero_memory(&obj->param, sizeof(obj->param));
391396
secure_zero_memory(&obj->state, sizeof(obj->state));
392-
if (obj->lock) {
393-
PyThread_free_lock(obj->lock);
394-
obj->lock = NULL;
395-
}
396397

397398
PyTypeObject *type = Py_TYPE(self);
398399
PyObject_Free(self);

‎Modules/_hashopenssl.c

Lines changed: 18 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
# define Py_BUILD_CORE_MODULE 1
2323
#endif
2424

25+
#include <stdbool.h>
2526
#include "Python.h"
2627
#include "pycore_hashtable.h"
2728
#include "pycore_pyhash.h" // _Py_HashBytes()
@@ -227,16 +228,16 @@ typedef struct {
227228
PyObject_HEAD
228229
EVP_MD_CTX *ctx; /* OpenSSL message digest context */
229230
// Prevents undefined behavior via multiple threads entering the C API.
230-
// The lock will be NULL before threaded access has been enabled.
231-
PyThread_type_lock lock; /* OpenSSL context lock */
231+
bool use_mutex;
232+
PyMutex mutex; /* OpenSSL context lock */
232233
} EVPobject;
233234

234235
typedef struct {
235236
PyObject_HEAD
236237
HMAC_CTX *ctx; /* OpenSSL hmac context */
237238
// Prevents undefined behavior via multiple threads entering the C API.
238-
// The lock will be NULL before threaded access has been enabled.
239-
PyThread_type_lock lock; /* HMAC context lock */
239+
bool use_mutex;
240+
PyMutex mutex; /* HMAC context lock */
240241
} HMACobject;
241242

242243
#include "clinic/_hashopenssl.c.h"
@@ -414,8 +415,7 @@ newEVPobject(PyTypeObject *type)
414415
if (retval == NULL) {
415416
return NULL;
416417
}
417-
418-
retval->lock = NULL;
418+
HASHLIB_INIT_MUTEX(retval);
419419

420420
retval->ctx = EVP_MD_CTX_new();
421421
if (retval->ctx == NULL) {
@@ -453,8 +453,6 @@ static void
453453
EVP_dealloc(EVPobject *self)
454454
{
455455
PyTypeObject *tp = Py_TYPE(self);
456-
if (self->lock != NULL)
457-
PyThread_free_lock(self->lock);
458456
EVP_MD_CTX_free(self->ctx);
459457
PyObject_Free(self);
460458
Py_DECREF(tp);
@@ -582,16 +580,14 @@ EVP_update(EVPobject *self, PyObject *obj)
582580

583581
GET_BUFFER_VIEW_OR_ERROUT(obj, &view);
584582

585-
if (self->lock == NULL && view.len >= HASHLIB_GIL_MINSIZE) {
586-
self->lock = PyThread_allocate_lock();
587-
/* fail? lock = NULL and we fail over to non-threaded code. */
583+
if (!self->use_mutex && view.len >= HASHLIB_GIL_MINSIZE) {
584+
self->use_mutex = true;
588585
}
589-
590-
if (self->lock != NULL) {
586+
if (self->use_mutex) {
591587
Py_BEGIN_ALLOW_THREADS
592-
PyThread_acquire_lock(self->lock, 1);
588+
PyMutex_Lock(&self->mutex);
593589
result = EVP_hash(self, view.buf, view.len);
594-
PyThread_release_lock(self->lock);
590+
PyMutex_Unlock(&self->mutex);
595591
Py_END_ALLOW_THREADS
596592
} else {
597593
result = EVP_hash(self, view.buf, view.len);
@@ -1540,7 +1536,7 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj,
15401536
}
15411537

15421538
self->ctx = ctx;
1543-
self->lock = NULL;
1539+
HASHLIB_INIT_MUTEX(self);
15441540

15451541
if ((msg_obj != NULL) && (msg_obj != Py_None)) {
15461542
if (!_hmac_update(self, msg_obj))
@@ -1582,16 +1578,14 @@ _hmac_update(HMACobject *self, PyObject *obj)
15821578

15831579
GET_BUFFER_VIEW_OR_ERROR(obj, &view, return 0);
15841580

1585-
if (self->lock == NULL && view.len >= HASHLIB_GIL_MINSIZE) {
1586-
self->lock = PyThread_allocate_lock();
1587-
/* fail? lock = NULL and we fail over to non-threaded code. */
1581+
if (!self->use_mutex && view.len >= HASHLIB_GIL_MINSIZE) {
1582+
self->use_mutex = true;
15881583
}
1589-
1590-
if (self->lock != NULL) {
1584+
if (self->use_mutex) {
15911585
Py_BEGIN_ALLOW_THREADS
1592-
PyThread_acquire_lock(self->lock, 1);
1586+
PyMutex_Lock(&self->mutex);
15931587
r = HMAC_Update(self->ctx, (const unsigned char*)view.buf, view.len);
1594-
PyThread_release_lock(self->lock);
1588+
PyMutex_Unlock(&self->mutex);
15951589
Py_END_ALLOW_THREADS
15961590
} else {
15971591
r = HMAC_Update(self->ctx, (const unsigned char*)view.buf, view.len);
@@ -1633,7 +1627,7 @@ _hashlib_HMAC_copy_impl(HMACobject *self)
16331627
return NULL;
16341628
}
16351629
retval->ctx = ctx;
1636-
retval->lock = NULL;
1630+
HASHLIB_INIT_MUTEX(retval);
16371631

16381632
return (PyObject *)retval;
16391633
}
@@ -1642,9 +1636,6 @@ static void
16421636
_hmac_dealloc(HMACobject *self)
16431637
{
16441638
PyTypeObject *tp = Py_TYPE(self);
1645-
if (self->lock != NULL) {
1646-
PyThread_free_lock(self->lock);
1647-
}
16481639
HMAC_CTX_free(self->ctx);
16491640
PyObject_Free(self);
16501641
Py_DECREF(tp);

‎Modules/clinic/md5module.c.h

Lines changed: 58 additions & 7 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)