diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 91970c282391f7..9561c40776f01c 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -611,6 +611,12 @@ Opening network connections to bind the socket locally. The *local_host* and *local_port* are looked up using :meth:`getaddrinfo`. + .. note:: + + On Windows, when using the proactor event loop with ``local_addr=None``, + an :exc:`OSError` with :attr:`!errno.WSAEINVAL` will be raised + when running it. + * *remote_addr*, if given, is a ``(remote_host, remote_port)`` tuple used to connect the socket to a remote address. The *remote_host* and *remote_port* are looked up using :meth:`getaddrinfo`. diff --git a/Makefile.pre.in b/Makefile.pre.in index 0c070131cda200..b7b16ef4cb9d19 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -227,6 +227,7 @@ ENSUREPIP= @ENSUREPIP@ # Internal static libraries LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a LIBEXPAT_A= Modules/expat/libexpat.a +LIBHASHLIB_INTERNAL_A=Modules/_hashlib/libhashlib.a # HACL* build configuration LIBHACL_CFLAGS=@LIBHACL_CFLAGS@ @@ -761,6 +762,17 @@ LIBHACL_HMAC_HEADERS= \ $(LIBHACL_BLAKE2_HEADERS) \ $(LIBHACL_HEADERS) +########################################################################## +# Internal library for cryptographic primitives + +LIBHASHLIB_INTERNAL_OBJS= \ + Modules/_hashlib/hashlib_buffer.o + +LIBHASHLIB_INTERNAL_HEADERS= \ + Modules/_hashlib/hashlib_buffer.h \ + Modules/_hashlib/hashlib_fetch.h \ + Modules/_hashlib/hashlib_mutex.h + ######################################################################### # Rules @@ -1511,6 +1523,17 @@ $(LIBEXPAT_A): $(LIBEXPAT_OBJS) -rm -f $@ $(AR) $(ARFLAGS) $@ $(LIBEXPAT_OBJS) +########################################################################## +# '_hashlib', '_hmac' and HACL*-based modules helpers +LIBHASHLIB_INTERNAL_CFLAGS=@LIBHASHLIB_INTERNAL_CFLAGS@ $(PY_STDMODULE_CFLAGS) $(CCSHARED) + +Modules/_hashlib/hashlib_buffer.o: Modules/_hashlib/hashlib_buffer.c $(LIBHASHLIB_INTERNAL_HEADERS) $(PYTHON_HEADERS) + $(CC) -I$(srcdir)/Modules/_hashlib -c $(LIBHASHLIB_INTERNAL_CFLAGS) -o $@ $(srcdir)/Modules/_hashlib/hashlib_buffer.c + +$(LIBHASHLIB_INTERNAL_A): $(LIBHASHLIB_INTERNAL_OBJS) + -rm -f $@ + $(AR) $(ARFLAGS) $@ $(LIBHASHLIB_INTERNAL_OBJS) + ########################################################################## # HACL* library build # @@ -3353,21 +3376,21 @@ MODULE__CTYPES_TEST_DEPS=$(srcdir)/Modules/_ctypes/_ctypes_test_generated.c.h MODULE__CTYPES_MALLOC_CLOSURE=@MODULE__CTYPES_MALLOC_CLOSURE@ MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@ MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@ -MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h +MODULE__HASHLIB_DEPS=@LIBHASHLIB_INTERNAL@ MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h # HACL*-based cryptographic primitives -MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__MD5_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__MD5_LDEPS=$(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA1_HEADERS) $(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA1_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA1_HEADERS) $(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SHA1_LDEPS=$(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA2_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SHA2_LDEPS=$(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA3_HEADERS) $(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA3_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA3_HEADERS) $(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SHA3_LDEPS=$(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__BLAKE2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_BLAKE2_HEADERS) $(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__BLAKE2_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_BLAKE2_HEADERS) $(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__BLAKE2_LDEPS=$(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__HMAC_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HMAC_HEADERS) $(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__HMAC_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_HMAC_HEADERS) $(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__HMAC_LDEPS=$(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c diff --git a/Misc/NEWS.d/next/Build/2025-07-22-14-47-45.gh-issue-131876.oaYEEP.rst b/Misc/NEWS.d/next/Build/2025-07-22-14-47-45.gh-issue-131876.oaYEEP.rst new file mode 100644 index 00000000000000..304f2c30f664db --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-07-22-14-47-45.gh-issue-131876.oaYEEP.rst @@ -0,0 +1,2 @@ +Remove :file:`!Modules/hashlib.h` and move its content into dedicated files +now located in ``Modules/_hashlib``. Patch by Bénédikt Tran. diff --git a/Modules/_hashlib/hashlib_buffer.c b/Modules/_hashlib/hashlib_buffer.c new file mode 100644 index 00000000000000..032f93ad53ad1b --- /dev/null +++ b/Modules/_hashlib/hashlib_buffer.c @@ -0,0 +1,65 @@ +#include "hashlib_buffer.h" + +int +_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string) +{ + if (data != NULL && string == NULL) { + // called as H(data) or H(data=...) + *res = data; + return 1; + } + else if (data == NULL && string != NULL) { + // called as H(string=...) + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "the 'string' keyword parameter is deprecated since " + "Python 3.15 and slated for removal in Python 3.19; " + "use the 'data' keyword parameter or pass the data " + "to hash as a positional argument instead", 1) < 0) + { + *res = NULL; + return -1; + } + *res = string; + return 1; + } + else if (data == NULL && string == NULL) { + // fast path when no data is given + assert(!PyErr_Occurred()); + *res = NULL; + return 0; + } + else { + // called as H(data=..., string) + *res = NULL; + PyErr_SetString(PyExc_TypeError, + "'data' and 'string' are mutually exclusive " + "and support for 'string' keyword parameter " + "is slated for removal in a future version."); + return -1; + } +} + +int +_Py_hashlib_get_buffer_view(PyObject *obj, Py_buffer *view) +{ + if (PyUnicode_Check(obj)) { + PyErr_SetString(PyExc_TypeError, + "Strings must be encoded before hashing"); + return -1; + } + if (!PyObject_CheckBuffer(obj)) { + PyErr_SetString(PyExc_TypeError, + "object supporting the buffer API required"); + return -1; + } + if (PyObject_GetBuffer(obj, view, PyBUF_SIMPLE) == -1) { + return -1; + } + if (view->ndim > 1) { + PyErr_SetString(PyExc_BufferError, + "Buffer must be single dimension"); + PyBuffer_Release(view); + return -1; + } + return 0; +} diff --git a/Modules/_hashlib/hashlib_buffer.h b/Modules/_hashlib/hashlib_buffer.h new file mode 100644 index 00000000000000..809f19884f41b7 --- /dev/null +++ b/Modules/_hashlib/hashlib_buffer.h @@ -0,0 +1,48 @@ +#ifndef _HASHLIB_HASHLIB_BUFFER_H +#define _HASHLIB_HASHLIB_BUFFER_H + +#include "Python.h" + +/* + * Allow to use the 'data' or 'string' keyword in hashlib.new() + * and other hash functions named constructors. + * + * - If 'data' and 'string' are both non-NULL, set an exception and return -1. + * - If 'data' and 'string' are both NULL, set '*res' to NULL and return 0. + * - Otherwise, set '*res' to 'data' or 'string' and return 1. A deprecation + * warning is set when 'string' is specified. + * + * The symbol is exported for '_hashlib' and HACL*-based extension modules. + */ +PyAPI_FUNC(int) +_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string); + +/* + * Obtain a buffer view from a buffer-like object 'obj'. + * + * On success, store the result in 'view' and return 0. + * On error, set an exception and return -1. + * + * The symbol is exported for '_hashlib' and HACL*-based extension modules. + */ +PyAPI_FUNC(int) +_Py_hashlib_get_buffer_view(PyObject *obj, Py_buffer *view); + +/* + * Call _Py_hashlib_get_buffer_view() and check if it succeeded. + * + * On error, set an exception and execute the ERRACTION statements. + */ +#define GET_BUFFER_VIEW_OR_ERROR(OBJ, VIEW, ERRACTION) \ + do { \ + if (_Py_hashlib_get_buffer_view(OBJ, VIEW) < 0) { \ + assert(PyErr_Occurred()); \ + ERRACTION; \ + } \ + } while (0) + +/* Specialization of GET_BUFFER_VIEW_OR_ERROR() returning NULL on error. */ +#define GET_BUFFER_VIEW_OR_ERROUT(OBJ, VIEW) \ + GET_BUFFER_VIEW_OR_ERROR(OBJ, VIEW, return NULL) + +#endif // !_HASHLIB_HASHLIB_BUFFER_H diff --git a/Modules/_hashlib/hashlib_fetch.h b/Modules/_hashlib/hashlib_fetch.h new file mode 100644 index 00000000000000..09add71e0c798c --- /dev/null +++ b/Modules/_hashlib/hashlib_fetch.h @@ -0,0 +1,19 @@ +/* + * Utilities used when fetching a message digest from a digest-like identifier. + */ + +#ifndef _HASHLIB_HASHLIB_FETCH_H +#define _HASHLIB_HASHLIB_FETCH_H + +#include "Python.h" + +/* + * Internal error messages used for reporting an unsupported hash algorithm. + * The algorithm can be given by its name, a callable or a PEP-247 module. + * The same message is raised by Lib/hashlib.py::__get_builtin_constructor() + * and _hmacmodule.c::find_hash_info(). + */ +#define _Py_HASHLIB_UNSUPPORTED_ALGORITHM "unsupported hash algorithm %S" +#define _Py_HASHLIB_UNSUPPORTED_STR_ALGORITHM "unsupported hash algorithm %s" + +#endif // !_HASHLIB_HASHLIB_FETCH_H diff --git a/Modules/hashlib.h b/Modules/_hashlib/hashlib_mutex.h similarity index 53% rename from Modules/hashlib.h rename to Modules/_hashlib/hashlib_mutex.h index 5de5922c345047..d6924a2ef61e81 100644 --- a/Modules/hashlib.h +++ b/Modules/_hashlib/hashlib_mutex.h @@ -1,45 +1,14 @@ -/* Common code for use by all hashlib related modules. */ +#ifndef _HASHLIB_HASHLIB_MUTEX_H +#define _HASHLIB_HASHLIB_MUTEX_H -#include "pycore_lock.h" // PyMutex +#include "Python.h" +#include "pycore_lock.h" // PyMutex /* - * Internal error messages used for reporting an unsupported hash algorithm. - * The algorithm can be given by its name, a callable or a PEP-247 module. - * The same message is raised by Lib/hashlib.py::__get_builtin_constructor() - * and _hmacmodule.c::find_hash_info(). - */ -#define HASHLIB_UNSUPPORTED_ALGORITHM "unsupported hash algorithm %S" -#define HASHLIB_UNSUPPORTED_STR_ALGORITHM "unsupported hash algorithm %s" - -/* - * Given a PyObject* obj, fill in the Py_buffer* viewp with the result - * of PyObject_GetBuffer. Sets an exception and issues the erraction - * on any errors, e.g. 'return NULL' or 'goto error'. + * Message length above which the GIL is to be released + * when performing hashing operations. */ -#define GET_BUFFER_VIEW_OR_ERROR(obj, viewp, erraction) do { \ - if (PyUnicode_Check((obj))) { \ - PyErr_SetString(PyExc_TypeError, \ - "Strings must be encoded before hashing");\ - erraction; \ - } \ - if (!PyObject_CheckBuffer((obj))) { \ - PyErr_SetString(PyExc_TypeError, \ - "object supporting the buffer API required"); \ - erraction; \ - } \ - if (PyObject_GetBuffer((obj), (viewp), PyBUF_SIMPLE) == -1) { \ - erraction; \ - } \ - if ((viewp)->ndim > 1) { \ - PyErr_SetString(PyExc_BufferError, \ - "Buffer must be single dimension"); \ - PyBuffer_Release((viewp)); \ - erraction; \ - } \ - } while(0) - -#define GET_BUFFER_VIEW_OR_ERROUT(obj, viewp) \ - GET_BUFFER_VIEW_OR_ERROR(obj, viewp, return NULL) +#define HASHLIB_GIL_MINSIZE 2048 /* * Helper code to synchronize access to the hash object when the GIL is @@ -64,12 +33,6 @@ #define HASHLIB_ACQUIRE_LOCK(OBJ) PyMutex_Lock(&(OBJ)->mutex) #define HASHLIB_RELEASE_LOCK(OBJ) PyMutex_Unlock(&(OBJ)->mutex) -/* - * Message length above which the GIL is to be released - * when performing hashing operations. - */ -#define HASHLIB_GIL_MINSIZE 2048 - // Macros for executing code while conditionally holding the GIL. // // These only drop the GIL if the lock acquisition itself is likely to @@ -116,41 +79,4 @@ } \ } while (0) -static inline int -_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string) -{ - if (data != NULL && string == NULL) { - // called as H(data) or H(data=...) - *res = data; - return 1; - } - else if (data == NULL && string != NULL) { - // called as H(string=...) - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "the 'string' keyword parameter is deprecated since " - "Python 3.15 and slated for removal in Python 3.19; " - "use the 'data' keyword parameter or pass the data " - "to hash as a positional argument instead", 1) < 0) - { - *res = NULL; - return -1; - } - *res = string; - return 1; - } - else if (data == NULL && string == NULL) { - // fast path when no data is given - assert(!PyErr_Occurred()); - *res = NULL; - return 0; - } - else { - // called as H(data=..., string) - *res = NULL; - PyErr_SetString(PyExc_TypeError, - "'data' and 'string' are mutually exclusive " - "and support for 'string' keyword parameter " - "is slated for removal in a future version."); - return -1; - } -} +#endif // !_HASHLIB_HASHLIB_MUTEX_H diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index d79e4b360e95c5..26412cb62430c9 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -24,14 +24,17 @@ #include "Python.h" #include "pycore_hashtable.h" -#include "pycore_strhex.h" // _Py_strhex() -#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_RELAXED -#include "hashlib.h" +#include "pycore_strhex.h" // _Py_strhex() +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_RELAXED + +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_fetch.h" +#include "_hashlib/hashlib_mutex.h" /* EVP is the preferred interface to hashing in OpenSSL */ #include #include -#include // FIPS_mode() +#include // FIPS_mode() /* We use the object interface to discover what hashes OpenSSL supports. */ #include #include @@ -532,7 +535,7 @@ raise_unsupported_algorithm_error(_hashlibstate *state, PyObject *digestmod) { raise_unsupported_algorithm_impl( state->unsupported_digestmod_error, - HASHLIB_UNSUPPORTED_ALGORITHM, + _Py_HASHLIB_UNSUPPORTED_ALGORITHM, digestmod ); } @@ -542,7 +545,7 @@ raise_unsupported_str_algorithm_error(_hashlibstate *state, const char *name) { raise_unsupported_algorithm_impl( state->unsupported_digestmod_error, - HASHLIB_UNSUPPORTED_STR_ALGORITHM, + _Py_HASHLIB_UNSUPPORTED_STR_ALGORITHM, name ); } @@ -1251,15 +1254,11 @@ _hashlib_HASH(_hashlibstate *state, const char *digestname, PyObject *data_obj, return (PyObject *)self; } +// In Python 3.19, we can remove the "STRING" argument and would also be able +// to remove the macro (or keep it as an alias for better naming) since calls +// to _hashlib_HASH_new_impl() would fit on 80 characters. #define CALL_HASHLIB_NEW(MODULE, NAME, DATA, STRING, USEDFORSECURITY) \ - do { \ - PyObject *data_obj; \ - if (_Py_hashlib_data_argument(&data_obj, DATA, STRING) < 0) { \ - return NULL; \ - } \ - _hashlibstate *state = get_hashlib_state(MODULE); \ - return _hashlib_HASH(state, NAME, data_obj, USEDFORSECURITY); \ - } while (0) + return _hashlib_HASH_new_impl(MODULE, NAME, DATA, USEDFORSECURITY, STRING) /* The module-level function: new() */ @@ -1285,7 +1284,12 @@ _hashlib_HASH_new_impl(PyObject *module, const char *name, PyObject *data, int usedforsecurity, PyObject *string) /*[clinic end generated code: output=b905aaf9840c1bbd input=c34af6c6e696d44e]*/ { - CALL_HASHLIB_NEW(module, name, data, string, usedforsecurity); + PyObject *data_obj; + if (_Py_hashlib_data_argument(&data_obj, data, string) < 0) { + return NULL; + } + _hashlibstate *state = get_hashlib_state(module); + return _hashlib_HASH(state, name, data_obj, usedforsecurity); } diff --git a/Modules/blake2module.c b/Modules/blake2module.c index 163f238a4268d0..13c969056be354 100644 --- a/Modules/blake2module.c +++ b/Modules/blake2module.c @@ -15,10 +15,12 @@ #endif #include "Python.h" -#include "hashlib.h" -#include "pycore_strhex.h" // _Py_strhex() -#include "pycore_typeobject.h" #include "pycore_moduleobject.h" +#include "pycore_strhex.h" // _Py_strhex() +#include "pycore_typeobject.h" + +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_mutex.h" // QUICK CPU AUTODETECTION // diff --git a/Modules/hmacmodule.c b/Modules/hmacmodule.c index b5405c99f1f8ce..92be49c5a879f3 100644 --- a/Modules/hmacmodule.c +++ b/Modules/hmacmodule.c @@ -20,6 +20,10 @@ #include "pycore_hashtable.h" #include "pycore_strhex.h" // _Py_strhex() +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_fetch.h" +#include "_hashlib/hashlib_mutex.h" + /* * Taken from blake2module.c. In the future, detection of SIMD support * should be delegated to https://github.com/python/cpython/pull/125011. @@ -47,8 +51,6 @@ #include -#include "hashlib.h" - // --- Reusable error messages ------------------------------------------------ static inline void @@ -656,7 +658,7 @@ find_hash_info(hmacmodule_state *state, PyObject *hash_info_ref) } if (rc == 0) { PyErr_Format(state->unknown_hash_error, - HASHLIB_UNSUPPORTED_ALGORITHM, hash_info_ref); + _Py_HASHLIB_UNSUPPORTED_ALGORITHM, hash_info_ref); return NULL; } assert(info != NULL); @@ -1099,42 +1101,60 @@ _hmac_compute_digest_impl(PyObject *module, PyObject *key, PyObject *msg, } /* - * One-shot HMAC-HASH using the given HACL_HID. + * Obtain a view for 'key' and 'msg', storing it in 'keyview' and 'msgview'. + * + * Return 0 on success; otherwise set an exception and return -1. * * The length of the key and message buffers must not exceed UINT32_MAX, * lest an OverflowError is raised. The Python implementation takes care * of dispatching to the OpenSSL implementation in this case. */ -#define Py_HMAC_HACL_ONESHOT(HACL_HID, KEY, MSG) \ - do { \ - Py_buffer keyview, msgview; \ - GET_BUFFER_VIEW_OR_ERROUT((KEY), &keyview); \ - if (!has_uint32_t_buffer_length(&keyview)) { \ - PyBuffer_Release(&keyview); \ - set_invalid_key_length_error(); \ - return NULL; \ - } \ - GET_BUFFER_VIEW_OR_ERROR((MSG), &msgview, \ - PyBuffer_Release(&keyview); \ - return NULL); \ - if (!has_uint32_t_buffer_length(&msgview)) { \ - PyBuffer_Release(&msgview); \ - PyBuffer_Release(&keyview); \ - set_invalid_msg_length_error(); \ - return NULL; \ - } \ - uint8_t out[Py_hmac_## HACL_HID ##_digest_size]; \ - Py_hmac_## HACL_HID ##_compute_func( \ - out, \ - (uint8_t *)keyview.buf, (uint32_t)keyview.len, \ - (uint8_t *)msgview.buf, (uint32_t)msgview.len \ - ); \ - PyBuffer_Release(&msgview); \ - PyBuffer_Release(&keyview); \ - return PyBytes_FromStringAndSize( \ - (const char *)out, \ - Py_hmac_## HACL_HID ##_digest_size \ - ); \ +static int +hmac_get_buffer_views(PyObject *key, Py_buffer *keyview, + PyObject *msg, Py_buffer *msgview) +{ + if (_Py_hashlib_get_buffer_view(key, keyview) < 0) { + return -1; + } + if (!has_uint32_t_buffer_length(keyview)) { + PyBuffer_Release(keyview); + set_invalid_key_length_error(); + return -1; + } + if (_Py_hashlib_get_buffer_view(msg, msgview) < 0) { + PyBuffer_Release(keyview); + return -1; + } + if (!has_uint32_t_buffer_length(msgview)) { + PyBuffer_Release(msgview); + PyBuffer_Release(keyview); + set_invalid_msg_length_error(); + return -1; + } + return 0; +} + +/* + * One-shot HMAC-HASH using the given HACL_HID. + */ +#define HACL_HMAC_COMPUTE_NAMED_DIGEST(HACL_HID, KEY, MSG) \ + do { \ + Py_buffer keyview, msgview; \ + if (hmac_get_buffer_views(key, &keyview, msg, &msgview) < 0) { \ + return NULL; \ + } \ + uint8_t out[Py_hmac_## HACL_HID ##_digest_size]; \ + Py_hmac_## HACL_HID ##_compute_func( \ + out, \ + (uint8_t *)keyview.buf, (uint32_t)keyview.len, \ + (uint8_t *)msgview.buf, (uint32_t)msgview.len \ + ); \ + PyBuffer_Release(&msgview); \ + PyBuffer_Release(&keyview); \ + return PyBytes_FromStringAndSize( \ + (const char *)out, \ + Py_hmac_## HACL_HID ##_digest_size \ + ); \ } while (0) /*[clinic input] @@ -1150,7 +1170,7 @@ static PyObject * _hmac_compute_md5_impl(PyObject *module, PyObject *key, PyObject *msg) /*[clinic end generated code: output=7837a4ceccbbf636 input=77a4b774c7d61218]*/ { - Py_HMAC_HACL_ONESHOT(md5, key, msg); + HACL_HMAC_COMPUTE_NAMED_DIGEST(md5, key, msg); } /*[clinic input] @@ -1166,7 +1186,7 @@ static PyObject * _hmac_compute_sha1_impl(PyObject *module, PyObject *key, PyObject *msg) /*[clinic end generated code: output=79fd7689c83691d8 input=3b64dccc6bdbe4ba]*/ { - Py_HMAC_HACL_ONESHOT(sha1, key, msg); + HACL_HMAC_COMPUTE_NAMED_DIGEST(sha1, key, msg); } /*[clinic input] @@ -1182,7 +1202,7 @@ static PyObject * _hmac_compute_sha2_224_impl(PyObject *module, PyObject *key, PyObject *msg) /*[clinic end generated code: output=7f21f1613e53979e input=a1a75f25f23449af]*/ { - Py_HMAC_HACL_ONESHOT(sha2_224, key, msg); + HACL_HMAC_COMPUTE_NAMED_DIGEST(sha2_224, key, msg); } /*[clinic input] @@ -1198,7 +1218,7 @@ static PyObject * _hmac_compute_sha2_256_impl(PyObject *module, PyObject *key, PyObject *msg) /*[clinic end generated code: output=d4a291f7d9a82459 input=5c9ccf2df048ace3]*/ { - Py_HMAC_HACL_ONESHOT(sha2_256, key, msg); + HACL_HMAC_COMPUTE_NAMED_DIGEST(sha2_256, key, msg); } /*[clinic input] @@ -1214,7 +1234,7 @@ static PyObject * _hmac_compute_sha2_384_impl(PyObject *module, PyObject *key, PyObject *msg) /*[clinic end generated code: output=f211fa26e3700c27 input=2fee2c14766af231]*/ { - Py_HMAC_HACL_ONESHOT(sha2_384, key, msg); + HACL_HMAC_COMPUTE_NAMED_DIGEST(sha2_384, key, msg); } /*[clinic input] @@ -1230,7 +1250,7 @@ static PyObject * _hmac_compute_sha2_512_impl(PyObject *module, PyObject *key, PyObject *msg) /*[clinic end generated code: output=d5c20373762cecca input=3371eaac315c7864]*/ { - Py_HMAC_HACL_ONESHOT(sha2_512, key, msg); + HACL_HMAC_COMPUTE_NAMED_DIGEST(sha2_512, key, msg); } /*[clinic input] @@ -1246,7 +1266,7 @@ static PyObject * _hmac_compute_sha3_224_impl(PyObject *module, PyObject *key, PyObject *msg) /*[clinic end generated code: output=a242ccac9ad9c22b input=d0ab0c7d189c3d87]*/ { - Py_HMAC_HACL_ONESHOT(sha3_224, key, msg); + HACL_HMAC_COMPUTE_NAMED_DIGEST(sha3_224, key, msg); } /*[clinic input] @@ -1262,7 +1282,7 @@ static PyObject * _hmac_compute_sha3_256_impl(PyObject *module, PyObject *key, PyObject *msg) /*[clinic end generated code: output=b539dbb61af2fe0b input=f05d7b6364b35d02]*/ { - Py_HMAC_HACL_ONESHOT(sha3_256, key, msg); + HACL_HMAC_COMPUTE_NAMED_DIGEST(sha3_256, key, msg); } /*[clinic input] @@ -1278,7 +1298,7 @@ static PyObject * _hmac_compute_sha3_384_impl(PyObject *module, PyObject *key, PyObject *msg) /*[clinic end generated code: output=5eb372fb5c4ffd3a input=d842d393e7aa05ae]*/ { - Py_HMAC_HACL_ONESHOT(sha3_384, key, msg); + HACL_HMAC_COMPUTE_NAMED_DIGEST(sha3_384, key, msg); } /*[clinic input] @@ -1294,7 +1314,7 @@ static PyObject * _hmac_compute_sha3_512_impl(PyObject *module, PyObject *key, PyObject *msg) /*[clinic end generated code: output=154bcbf8c2eacac1 input=166fe5baaeaabfde]*/ { - Py_HMAC_HACL_ONESHOT(sha3_512, key, msg); + HACL_HMAC_COMPUTE_NAMED_DIGEST(sha3_512, key, msg); } /*[clinic input] @@ -1310,7 +1330,7 @@ static PyObject * _hmac_compute_blake2s_32_impl(PyObject *module, PyObject *key, PyObject *msg) /*[clinic end generated code: output=cfc730791bc62361 input=d22c36e7fe31a985]*/ { - Py_HMAC_HACL_ONESHOT(blake2s_32, key, msg); + HACL_HMAC_COMPUTE_NAMED_DIGEST(blake2s_32, key, msg); } /*[clinic input] @@ -1326,9 +1346,11 @@ static PyObject * _hmac_compute_blake2b_32_impl(PyObject *module, PyObject *key, PyObject *msg) /*[clinic end generated code: output=765c5c4fb9124636 input=4a35ee058d172f4b]*/ { - Py_HMAC_HACL_ONESHOT(blake2b_32, key, msg); + HACL_HMAC_COMPUTE_NAMED_DIGEST(blake2b_32, key, msg); } +#undef HACL_HMAC_COMPUTE_NAMED_DIGEST + // --- HMAC module methods ---------------------------------------------------- static PyMethodDef hmacmodule_methods[] = { diff --git a/Modules/md5module.c b/Modules/md5module.c index 8b6dd4a8195dfb..d5dc4f60a575d4 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -22,9 +22,10 @@ #endif #include "Python.h" -#include "pycore_strhex.h" // _Py_strhex() +#include "pycore_strhex.h" // _Py_strhex() -#include "hashlib.h" +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_mutex.h" #include "_hacl/Hacl_Hash_MD5.h" diff --git a/Modules/sha1module.c b/Modules/sha1module.c index faa9dcccc5755b..86e5691e8463e4 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -20,9 +20,11 @@ #endif #include "Python.h" -#include "hashlib.h" -#include "pycore_strhex.h" // _Py_strhex() -#include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "pycore_strhex.h" // _Py_strhex() +#include "pycore_typeobject.h" // _PyType_GetModuleState() + +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_mutex.h" #include "_hacl/Hacl_Hash_SHA1.h" diff --git a/Modules/sha2module.c b/Modules/sha2module.c index 36300ba899fd44..dbf6dde1b8c121 100644 --- a/Modules/sha2module.c +++ b/Modules/sha2module.c @@ -21,11 +21,12 @@ #endif #include "Python.h" -#include "pycore_moduleobject.h" // _PyModule_GetState() -#include "pycore_typeobject.h" // _PyType_GetModuleState() -#include "pycore_strhex.h" // _Py_strhex() +#include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_strhex.h" // _Py_strhex() +#include "pycore_typeobject.h" // _PyType_GetModuleState() -#include "hashlib.h" +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_mutex.h" #include "_hacl/Hacl_Hash_SHA2.h" diff --git a/Modules/sha3module.c b/Modules/sha3module.c index 5764556bb680f3..c67bfadbe4664a 100644 --- a/Modules/sha3module.c +++ b/Modules/sha3module.c @@ -21,9 +21,11 @@ #endif #include "Python.h" -#include "pycore_strhex.h" // _Py_strhex() -#include "pycore_typeobject.h" // _PyType_GetModuleState() -#include "hashlib.h" +#include "pycore_strhex.h" // _Py_strhex() +#include "pycore_typeobject.h" // _PyType_GetModuleState() + +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_mutex.h" #include "_hacl/Hacl_Hash_SHA3.h" diff --git a/PCbuild/_hashlib.vcxproj b/PCbuild/_hashlib.vcxproj index 2cd205224bc089..cfb43cee935b86 100644 --- a/PCbuild/_hashlib.vcxproj +++ b/PCbuild/_hashlib.vcxproj @@ -100,6 +100,11 @@ + + + + + diff --git a/PCbuild/_hashlib.vcxproj.filters b/PCbuild/_hashlib.vcxproj.filters index 7a0700c007f644..d465d92a956eda 100644 --- a/PCbuild/_hashlib.vcxproj.filters +++ b/PCbuild/_hashlib.vcxproj.filters @@ -18,4 +18,4 @@ Resource Files - \ No newline at end of file + diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 517103acea8d8e..c59b380d814ed9 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -447,6 +447,10 @@ HACL_CAN_COMPILE_VEC128;%(PreprocessorDefinitions) /arch:AVX %(AdditionalOptions) + + + + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index e9eedfd1312fae..1410cbbef6c849 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -255,6 +255,15 @@ Include + + Modules\_hashlib + + + Modules\_hashlib + + + Modules\_hashlib + Modules @@ -971,6 +980,9 @@ Modules + + Modules\_hashlib + Modules diff --git a/configure b/configure index 74df430d10d6bd..0e7aefed5ee62d 100755 --- a/configure +++ b/configure @@ -725,6 +725,8 @@ LIBHACL_BLAKE2_SIMD128_OBJS LIBHACL_SIMD128_FLAGS LIBHACL_LDFLAGS LIBHACL_CFLAGS +LIBHASHLIB_INTERNAL +LIBHASHLIB_INTERNAL_CFLAGS MODULE_UNICODEDATA_FALSE MODULE_UNICODEDATA_TRUE MODULE__MULTIBYTECODEC_FALSE @@ -29949,6 +29951,7 @@ SRCDIRS="\ Modules/_decimal \ Modules/_decimal/libmpdec \ Modules/_hacl \ + Modules/_hashlib \ Modules/_io \ Modules/_multiprocessing \ Modules/_sqlite \ @@ -32525,6 +32528,15 @@ then : fi +############################################################################### +# Cryptographic primitives +LIBHASHLIB_INTERNAL_CFLAGS="-I\$(srcdir)/Modules/_hashlib" +LIBHASHLIB_INTERNAL_LDFLAGS="-lm \$(LIBHASHLIB_INTERNAL_A)" +LIBHASHLIB_INTERNAL="\$(LIBHASHLIB_INTERNAL_HEADERS) \$(LIBHASHLIB_INTERNAL_A)" + + + + ############################################################################### # HACL* compilation and linking configuration (contact: @picnixz) # @@ -32773,8 +32785,8 @@ fi if test "x$py_cv_module__md5" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__MD5_CFLAGS=$LIBHACL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__MD5_LDFLAGS=\$($LIBHACL_MD5_LDFLAGS)$as_nl" + as_fn_append MODULE_BLOCK "MODULE__MD5_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__MD5_LDFLAGS=\$($LIBHACL_MD5_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__md5" = yes; then @@ -32818,8 +32830,8 @@ fi if test "x$py_cv_module__sha1" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA1_CFLAGS=$LIBHACL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__SHA1_LDFLAGS=\$($LIBHACL_SHA1_LDFLAGS)$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA1_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA1_LDFLAGS=\$($LIBHACL_SHA1_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__sha1" = yes; then @@ -32863,8 +32875,8 @@ fi if test "x$py_cv_module__sha2" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA2_CFLAGS=$LIBHACL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__SHA2_LDFLAGS=\$($LIBHACL_SHA2_LDFLAGS)$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA2_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA2_LDFLAGS=\$($LIBHACL_SHA2_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__sha2" = yes; then @@ -32908,8 +32920,8 @@ fi if test "x$py_cv_module__sha3" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA3_CFLAGS=$LIBHACL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__SHA3_LDFLAGS=\$($LIBHACL_SHA3_LDFLAGS)$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA3_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA3_LDFLAGS=\$($LIBHACL_SHA3_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__sha3" = yes; then @@ -32953,8 +32965,8 @@ fi if test "x$py_cv_module__blake2" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__BLAKE2_CFLAGS=$LIBHACL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__BLAKE2_LDFLAGS=\$($LIBHACL_BLAKE2_LDFLAGS)$as_nl" + as_fn_append MODULE_BLOCK "MODULE__BLAKE2_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__BLAKE2_LDFLAGS=\$($LIBHACL_BLAKE2_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__blake2" = yes; then @@ -32999,8 +33011,8 @@ fi if test "x$py_cv_module__hmac" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__HMAC_CFLAGS=$LIBHACL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__HMAC_LDFLAGS=\$($LIBHACL_HMAC_LDFLAGS)$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HMAC_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HMAC_LDFLAGS=\$($LIBHACL_HMAC_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__hmac" = yes; then @@ -33681,8 +33693,8 @@ fi if test "x$py_cv_module__hashlib" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__HASHLIB_CFLAGS=$OPENSSL_INCLUDES$as_nl" - as_fn_append MODULE_BLOCK "MODULE__HASHLIB_LDFLAGS=$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HASHLIB_CFLAGS=$OPENSSL_INCLUDES $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HASHLIB_LDFLAGS=$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__hashlib" = yes; then diff --git a/configure.ac b/configure.ac index 29e36f217402f1..1e590e1d0fd727 100644 --- a/configure.ac +++ b/configure.ac @@ -7187,6 +7187,7 @@ SRCDIRS="\ Modules/_decimal \ Modules/_decimal/libmpdec \ Modules/_hacl \ + Modules/_hashlib \ Modules/_io \ Modules/_multiprocessing \ Modules/_sqlite \ @@ -7957,6 +7958,15 @@ PY_STDLIB_MOD_SIMPLE([_codecs_tw]) PY_STDLIB_MOD_SIMPLE([_multibytecodec]) PY_STDLIB_MOD_SIMPLE([unicodedata]) +############################################################################### +# Cryptographic primitives +LIBHASHLIB_INTERNAL_CFLAGS="-I\$(srcdir)/Modules/_hashlib" +LIBHASHLIB_INTERNAL_LDFLAGS="-lm \$(LIBHASHLIB_INTERNAL_A)" +LIBHASHLIB_INTERNAL="\$(LIBHASHLIB_INTERNAL_HEADERS) \$(LIBHASHLIB_INTERNAL_A)" + +AC_SUBST([LIBHASHLIB_INTERNAL_CFLAGS]) +AC_SUBST([LIBHASHLIB_INTERNAL]) + ############################################################################### # HACL* compilation and linking configuration (contact: @picnixz) # @@ -8093,7 +8103,9 @@ dnl The EXTNAME is the name of the extension module being built. AC_DEFUN([PY_HACL_CREATE_MODULE], [ AS_VAR_PUSHDEF([v], [[LIBHACL_][$1][_LDFLAGS]]) AS_VAR_SET([v], [[LIBHACL_][$1][_LIB_${LIBHACL_LDEPS_LIBTYPE}]]) - PY_STDLIB_MOD([$2], [$3], [], [$LIBHACL_CFLAGS], [\$($v)]) + PY_STDLIB_MOD([$2], [$3], [], + [$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS], + [\$($v) $LIBHASHLIB_INTERNAL_LDFLAGS]) AS_VAR_POPDEF([v]) ]) @@ -8174,7 +8186,8 @@ dnl OpenSSL bindings PY_STDLIB_MOD([_ssl], [], [test "$ac_cv_working_openssl_ssl" = yes], [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $OPENSSL_LIBS]) PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes], - [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS]) + [$OPENSSL_INCLUDES $LIBHASHLIB_INTERNAL_CFLAGS], + [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS $LIBHASHLIB_INTERNAL_LDFLAGS]) dnl test modules PY_STDLIB_MOD([_testcapi],