gh-111916: Make hashlib related modules thread-safe without the GIL

Misc/NEWS.d/next/Core and Builtins/2023-11-14-22-12-11.gh-issue-111916.ZGCayL.rst

@@ -0,0 +1 @@
+Make hashlib related modules thread-safe without the GIL

Modules/_blake2/blake2b_impl.c

@@ -17,6 +17,7 @@
 # define Py_BUILD_CORE_MODULE 1
 #endif
 
+#include <stdbool.h>
 #include "Python.h"
 #include "pycore_strhex.h" // _Py_strhex()
 
@@ -42,7 +43,8 @@ typedef struct {
  PyObject_HEAD
  blake2b_param param;
  blake2b_state state;
- PyThread_type_lock lock;
+ bool use_mutex;
+ PyMutex mutex;
 } BLAKE2bObject;
 
 #include "clinic/blake2b_impl.c.h"
@@ -59,9 +61,11 @@ new_BLAKE2bObject(PyTypeObject *type)
 {
  BLAKE2bObject *self;
  self = (BLAKE2bObject *)type->tp_alloc(type, 0);
- if (self != NULL) {
-self->lock = NULL;
+ if (self == NULL) {
+return NULL;
  }
+ HASHLIB_INIT_MUTEX(self);
+
  return self;
 }
 
@@ -278,18 +282,19 @@ _blake2_blake2b_update(BLAKE2bObject *self, PyObject *data)
 
  GET_BUFFER_VIEW_OR_ERROUT(data, &buf);
 
- if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE)
- self->lock = PyThread_allocate_lock();
-
- if (self->lock != NULL) {
- Py_BEGIN_ALLOW_THREADS
-PyThread_acquire_lock(self->lock, 1);
- blake2b_update(&self->state, buf.buf, buf.len);
-PyThread_release_lock(self->lock);
- Py_END_ALLOW_THREADS
+ if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) {
+ self->use_mutex = true;
+ }
+ if (self->use_mutex) {
+Py_BEGIN_ALLOW_THREADS
+ PyMutex_Lock(&self->mutex);
+blake2b_update(&self->state, buf.buf, buf.len);
+ PyMutex_Unlock(&self->mutex);
+Py_END_ALLOW_THREADS
  } else {
  blake2b_update(&self->state, buf.buf, buf.len);
  }
+
  PyBuffer_Release(&buf);
 
  Py_RETURN_NONE;
@@ -389,10 +394,6 @@ py_blake2b_dealloc(PyObject *self)
  /* Try not to leave state in memory. */
  secure_zero_memory(&obj->param, sizeof(obj->param));
  secure_zero_memory(&obj->state, sizeof(obj->state));
- if (obj->lock) {
- PyThread_free_lock(obj->lock);
- obj->lock = NULL;
- }
 
  PyTypeObject *type = Py_TYPE(self);
  PyObject_Free(self);

Modules/_blake2/blake2s_impl.c

@@ -17,6 +17,7 @@
 # define Py_BUILD_CORE_MODULE 1
 #endif
 
+#include <stdbool.h>
 #include "Python.h"
 #include "pycore_strhex.h" // _Py_strhex()
 
@@ -42,7 +43,8 @@ typedef struct {
  PyObject_HEAD
  blake2s_param param;
  blake2s_state state;
- PyThread_type_lock lock;
+ bool use_mutex;
+ PyMutex mutex;
 } BLAKE2sObject;
 
 #include "clinic/blake2s_impl.c.h"
@@ -59,9 +61,11 @@ new_BLAKE2sObject(PyTypeObject *type)
 {
  BLAKE2sObject *self;
  self = (BLAKE2sObject *)type->tp_alloc(type, 0);
- if (self != NULL) {
-self->lock = NULL;
+ if (self == NULL) {
+return NULL;
  }
+ HASHLIB_INIT_MUTEX(self);
+
  return self;
 }
 
@@ -278,18 +282,19 @@ _blake2_blake2s_update(BLAKE2sObject *self, PyObject *data)
 
  GET_BUFFER_VIEW_OR_ERROUT(data, &buf);
 
- if (self->lock == NULL && buf.len >= HASHLIB_GIL_MINSIZE)
- self->lock = PyThread_allocate_lock();
-
- if (self->lock != NULL) {
- Py_BEGIN_ALLOW_THREADS
-PyThread_acquire_lock(self->lock, 1);
- blake2s_update(&self->state, buf.buf, buf.len);
-PyThread_release_lock(self->lock);
- Py_END_ALLOW_THREADS
+ if (!self->use_mutex && buf.len >= HASHLIB_GIL_MINSIZE) {
+ self->use_mutex = true;
+ }
+ if (self->use_mutex) {
+Py_BEGIN_ALLOW_THREADS
+ PyMutex_Lock(&self->mutex);
+blake2s_update(&self->state, buf.buf, buf.len);
+ PyMutex_Unlock(&self->mutex);
+Py_END_ALLOW_THREADS
  } else {
  blake2s_update(&self->state, buf.buf, buf.len);
  }
+
  PyBuffer_Release(&buf);
 
  Py_RETURN_NONE;
@@ -389,10 +394,6 @@ py_blake2s_dealloc(PyObject *self)
  /* Try not to leave state in memory. */
  secure_zero_memory(&obj->param, sizeof(obj->param));
  secure_zero_memory(&obj->state, sizeof(obj->state));
- if (obj->lock) {
- PyThread_free_lock(obj->lock);
- obj->lock = NULL;
- }
 
  PyTypeObject *type = Py_TYPE(self);
  PyObject_Free(self);

Modules/_hashopenssl.c

@@ -22,6 +22,7 @@
 # define Py_BUILD_CORE_MODULE 1
 #endif
 
+#include <stdbool.h>
 #include "Python.h"
 #include "pycore_hashtable.h"
 #include "pycore_pyhash.h" // _Py_HashBytes()
@@ -227,16 +228,16 @@ typedef struct {
  PyObject_HEAD
  EVP_MD_CTX *ctx; /* OpenSSL message digest context */
  // Prevents undefined behavior via multiple threads entering the C API.
-// The lock will be NULL before threaded access has been enabled.
-PyThread_type_lock lock; /* OpenSSL context lock */
+bool use_mutex;
+PyMutex mutex; /* OpenSSL context lock */
 } EVPobject;
 
 typedef struct {
  PyObject_HEAD
  HMAC_CTX *ctx; /* OpenSSL hmac context */
  // Prevents undefined behavior via multiple threads entering the C API.
-// The lock will be NULL before threaded access has been enabled.
-PyThread_type_lock lock; /* HMAC context lock */
+bool use_mutex;
+PyMutex mutex; /* HMAC context lock */
 } HMACobject;
 
 #include "clinic/_hashopenssl.c.h"
@@ -414,8 +415,7 @@ newEVPobject(PyTypeObject *type)
  if (retval == NULL) {
  return NULL;
  }
-
- retval->lock = NULL;
+ HASHLIB_INIT_MUTEX(retval);
 
  retval->ctx = EVP_MD_CTX_new();
  if (retval->ctx == NULL) {
@@ -453,8 +453,6 @@ static void
 EVP_dealloc(EVPobject *self)
 {
  PyTypeObject *tp = Py_TYPE(self);
- if (self->lock != NULL)
- PyThread_free_lock(self->lock);
  EVP_MD_CTX_free(self->ctx);
  PyObject_Free(self);
  Py_DECREF(tp);
@@ -582,16 +580,14 @@ EVP_update(EVPobject *self, PyObject *obj)
 
  GET_BUFFER_VIEW_OR_ERROUT(obj, &view);
 
- if (self->lock == NULL && view.len >= HASHLIB_GIL_MINSIZE) {
- self->lock = PyThread_allocate_lock();
- /* fail? lock = NULL and we fail over to non-threaded code. */
+ if (!self->use_mutex && view.len >= HASHLIB_GIL_MINSIZE) {
+ self->use_mutex = true;
  }
-
- if (self->lock != NULL) {
+ if (self->use_mutex) {
  Py_BEGIN_ALLOW_THREADS
-PyThread_acquire_lock(self->lock, 1);
+PyMutex_Lock(&self->mutex);
  result = EVP_hash(self, view.buf, view.len);
-PyThread_release_lock(self->lock);
+PyMutex_Unlock(&self->mutex);
  Py_END_ALLOW_THREADS
  } else {
  result = EVP_hash(self, view.buf, view.len);
@@ -1540,7 +1536,7 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj,
  }
 
  self->ctx = ctx;
- self->lock = NULL;
+HASHLIB_INIT_MUTEX(self);
 
  if ((msg_obj != NULL) && (msg_obj != Py_None)) {
  if (!_hmac_update(self, msg_obj))
@@ -1582,16 +1578,14 @@ _hmac_update(HMACobject *self, PyObject *obj)
 
  GET_BUFFER_VIEW_OR_ERROR(obj, &view, return 0);
 
- if (self->lock == NULL && view.len >= HASHLIB_GIL_MINSIZE) {
- self->lock = PyThread_allocate_lock();
- /* fail? lock = NULL and we fail over to non-threaded code. */
+ if (!self->use_mutex && view.len >= HASHLIB_GIL_MINSIZE) {
+ self->use_mutex = true;
  }
-
- if (self->lock != NULL) {
+ if (self->use_mutex) {
  Py_BEGIN_ALLOW_THREADS
-PyThread_acquire_lock(self->lock, 1);
+PyMutex_Lock(&self->mutex);
  r = HMAC_Update(self->ctx, (const unsigned char*)view.buf, view.len);
-PyThread_release_lock(self->lock);
+PyMutex_Unlock(&self->mutex);
  Py_END_ALLOW_THREADS
  } else {
  r = HMAC_Update(self->ctx, (const unsigned char*)view.buf, view.len);
@@ -1633,7 +1627,7 @@ _hashlib_HMAC_copy_impl(HMACobject *self)
  return NULL;
  }
  retval->ctx = ctx;
- retval->lock = NULL;
+HASHLIB_INIT_MUTEX(retval);
 
  return (PyObject *)retval;
 }
@@ -1642,9 +1636,6 @@ static void
 _hmac_dealloc(HMACobject *self)
 {
  PyTypeObject *tp = Py_TYPE(self);
- if (self->lock != NULL) {
- PyThread_free_lock(self->lock);
- }
  HMAC_CTX_free(self->ctx);
  PyObject_Free(self);
  Py_DECREF(tp);