From 94cde64342fa8c7cd55a6d2b60f1e57654220bb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 10 Jun 2025 13:42:25 +0200 Subject: [PATCH 01/13] faster CI --- .github/workflows/build.yml | 283 +----------------------------------- .github/workflows/lint.yml | 28 ---- 2 files changed, 5 insertions(+), 306 deletions(-) delete mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 54ebc914b46821..6312530669623e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,12 +43,6 @@ jobs: # uses: ./.github/workflows/reusable-context.yml - check-docs: - name: Docs - needs: build-context - if: fromJSON(needs.build-context.outputs.run-docs) - uses: ./.github/workflows/reusable-docs.yml - check-autoconf-regen: name: 'Check if Autoconf files are up to date' # Don't use ubuntu-latest but a specific version to make the job @@ -160,15 +154,12 @@ jobs: needs: build-context if: fromJSON(needs.build-context.outputs.run-windows-tests) strategy: - fail-fast: false + fail-fast: true matrix: arch: - x64 - - Win32 - - arm64 free-threading: - false - - true exclude: # Skip Win32 on free-threaded builds - { arch: Win32, free-threading: true } @@ -177,22 +168,6 @@ jobs: arch: ${{ matrix.arch }} free-threading: ${{ matrix.free-threading }} - build-windows-msi: - name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category - Windows MSI${{ '' }} - needs: build-context - if: fromJSON(needs.build-context.outputs.run-windows-msi) - strategy: - fail-fast: false - matrix: - arch: - - x86 - - x64 - - arm64 - uses: ./.github/workflows/reusable-windows-msi.yml - with: - arch: ${{ matrix.arch }} - build-macos: name: >- macOS @@ -200,20 +175,17 @@ jobs: needs: build-context if: needs.build-context.outputs.run-tests == 'true' strategy: - fail-fast: false + fail-fast: true matrix: # Cirrus and macos-14 are M1, macos-13 is default GHA Intel. # macOS 13 only runs tests against the GIL-enabled CPython. # Cirrus used for upstream, macos-14 for forks. os: - ghcr.io/cirruslabs/macos-runner:sonoma - - macos-14 - - macos-13 is-fork: # only used for the exclusion trick - ${{ github.repository_owner != 'python' }} free-threading: - false - - true exclude: - os: ghcr.io/cirruslabs/macos-runner:sonoma is-fork: true @@ -235,17 +207,14 @@ jobs: needs: build-context if: needs.build-context.outputs.run-tests == 'true' strategy: - fail-fast: false + fail-fast: true matrix: bolt: - false - - true free-threading: - false - - true os: - ubuntu-24.04 - - ubuntu-24.04-arm exclude: # Do not test BOLT with free-threading, to conserve resources - bolt: true @@ -267,10 +236,10 @@ jobs: needs: build-context if: needs.build-context.outputs.run-tests == 'true' strategy: - fail-fast: false + fail-fast: true matrix: os: [ubuntu-24.04] - openssl_ver: [3.0.16, 3.1.8, 3.2.4, 3.3.3, 3.4.1] + openssl_ver: [3.0.16] # See Tools/ssl/make_ssl_data.py for notes on adding a new version env: OPENSSL_VER: ${{ matrix.openssl_ver }} @@ -330,203 +299,6 @@ jobs: with: config_hash: ${{ needs.build-context.outputs.config-hash }} - test-hypothesis: - name: "Hypothesis tests on Ubuntu" - runs-on: ubuntu-24.04 - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - env: - OPENSSL_VER: 3.0.16 - PYTHONSTRICTEXTENSIONBUILD: 1 - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v4 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - - name: Setup directory envs for out-of-tree builds - run: | - echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" - echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" - - name: Create directories for read-only out-of-tree builds - run: mkdir -p "$CPYTHON_RO_SRCDIR" "$CPYTHON_BUILDDIR" - - name: Bind mount sources read-only - run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - - name: Configure CPython out-of-tree - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: | - ../cpython-ro-srcdir/configure \ - --config-cache \ - --with-pydebug \ - --enable-slower-safety \ - --with-openssl="$OPENSSL_DIR" - - name: Build CPython out-of-tree - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make -j4 - - name: Display build info - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make pythoninfo - - name: Remount sources writable for tests - # some tests write to srcdir, lack of pyc files slows down testing - run: sudo mount "$CPYTHON_RO_SRCDIR" -oremount,rw - - name: Setup directory envs for out-of-tree builds - run: | - echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" - - name: "Create hypothesis venv" - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: | - VENV_LOC=$(realpath -m .)/hypovenv - VENV_PYTHON=$VENV_LOC/bin/python - echo "HYPOVENV=${VENV_LOC}" >> "$GITHUB_ENV" - echo "VENV_PYTHON=${VENV_PYTHON}" >> "$GITHUB_ENV" - ./python -m venv "$VENV_LOC" && "$VENV_PYTHON" -m pip install -r "${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt" - - name: 'Restore Hypothesis database' - id: cache-hypothesis-database - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/ - key: hypothesis-database-${{ github.head_ref || github.run_id }} - restore-keys: | - hypothesis-database- - - name: "Run tests" - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: | - # Most of the excluded tests are slow test suites with no property tests - # - # (GH-104097) test_sysconfig is skipped because it has tests that are - # failing when executed from inside a virtual environment. - "${VENV_PYTHON}" -m test \ - -W \ - --slowest \ - -j4 \ - --timeout 900 \ - -x test_asyncio \ - -x test_multiprocessing_fork \ - -x test_multiprocessing_forkserver \ - -x test_multiprocessing_spawn \ - -x test_concurrent_futures \ - -x test_socket \ - -x test_subprocess \ - -x test_signal \ - -x test_sysconfig - - uses: actions/upload-artifact@v4 - if: always() - with: - name: hypothesis-example-db - path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/examples/ - - build-asan: - name: 'Address sanitizer' - runs-on: ${{ matrix.os }} - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - os: [ubuntu-24.04] - env: - OPENSSL_VER: 3.0.16 - PYTHONSTRICTEXTENSIONBUILD: 1 - ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Set up GCC-10 for ASAN - uses: egor-tensin/setup-gcc@v1 - with: - version: 10 - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v4 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - - name: Configure CPython - run: ./configure --config-cache --with-address-sanitizer --without-pymalloc - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: Tests - run: xvfb-run make ci - - build-tsan: - name: >- - Thread sanitizer - ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - free-threading: - - false - - true - uses: ./.github/workflows/reusable-tsan.yml - with: - config_hash: ${{ needs.build-context.outputs.config-hash }} - free-threading: ${{ matrix.free-threading }} - cross-build-linux: name: Cross build Linux runs-on: ubuntu-latest @@ -567,45 +339,6 @@ jobs: run: | "$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed - # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ - cifuzz: - name: CIFuzz - runs-on: ubuntu-latest - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-ci-fuzz == 'true' - permissions: - security-events: write - strategy: - fail-fast: false - matrix: - sanitizer: [address, undefined, memory] - steps: - - name: Build fuzzers (${{ matrix.sanitizer }}) - id: build - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master - with: - oss-fuzz-project-name: cpython3 - sanitizer: ${{ matrix.sanitizer }} - - name: Run fuzzers (${{ matrix.sanitizer }}) - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master - with: - fuzz-seconds: 600 - oss-fuzz-project-name: cpython3 - output-sarif: true - sanitizer: ${{ matrix.sanitizer }} - - name: Upload crash - if: failure() && steps.build.outcome == 'success' - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.sanitizer }}-artifacts - path: ./out/artifacts - - name: Upload SARIF - if: always() && steps.build.outcome == 'success' - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: cifuzz-sarif/results.sarif - checkout_path: cifuzz-sarif all-required-green: # This job does nothing and is only used for the branch protection name: All required checks pass @@ -613,20 +346,14 @@ jobs: timeout-minutes: 5 needs: - build-context # Transitive dependency, needed to access `run-tests` value - - check-docs - check-autoconf-regen - check-generated-files - build-windows - - build-windows-msi - build-macos - build-ubuntu - build-ubuntu-ssltests - build-wasi - - test-hypothesis - - build-asan - - build-tsan - cross-build-linux - - cifuzz if: always() steps: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index d74ce8fcc256dc..00000000000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Lint - -on: [push, pull_request, workflow_dispatch] - -permissions: - contents: read - -env: - FORCE_COLOR: 1 - RUFF_OUTPUT_FORMAT: github - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - lint: - runs-on: ubuntu-latest - timeout-minutes: 10 - - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - uses: pre-commit/action@v3.0.1 From 3dd0a97801d707a7312910ad37f0bcb2cc139a04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 10 Jun 2025 12:56:07 +0200 Subject: [PATCH 02/13] extract `_hashlib` helpers into a separate directory --- Makefile.pre.in | 41 ++++++--- Modules/_hashlib/hashlib_buffer.c | 40 +++++++++ Modules/_hashlib/hashlib_buffer.h | 60 +++++++++++++ Modules/_hashlib/hashlib_fetch.c | 1 + Modules/_hashlib/hashlib_fetch.h | 130 +++++++++++++++++++++++++++ Modules/_hashlib/hashlib_mutex.h | 62 +++++++++++++ Modules/_hashopenssl.c | 13 ++- Modules/blake2module.c | 11 ++- Modules/hashlib.h | 116 ------------------------ Modules/hmacmodule.c | 5 +- Modules/md5module.c | 11 ++- Modules/sha1module.c | 11 ++- Modules/sha2module.c | 13 ++- Modules/sha3module.c | 10 +-- PCbuild/_hashlib.vcxproj | 6 ++ PCbuild/_hashlib.vcxproj.filters | 2 +- PCbuild/pythoncore.vcxproj | 4 + PCbuild/pythoncore.vcxproj.filters | 15 ++++ Tools/c-analyzer/cpython/ignored.tsv | 1 + configure | 35 +++++--- configure.ac | 14 ++- 21 files changed, 417 insertions(+), 184 deletions(-) create mode 100644 Modules/_hashlib/hashlib_buffer.c create mode 100644 Modules/_hashlib/hashlib_buffer.h create mode 100644 Modules/_hashlib/hashlib_fetch.c create mode 100644 Modules/_hashlib/hashlib_fetch.h create mode 100644 Modules/_hashlib/hashlib_mutex.h delete mode 100644 Modules/hashlib.h diff --git a/Makefile.pre.in b/Makefile.pre.in index b5703fbe6ae974..ba681d92e46465 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1481,6 +1481,12 @@ $(LIBEXPAT_A): $(LIBEXPAT_OBJS) -rm -f $@ $(AR) $(ARFLAGS) $@ $(LIBEXPAT_OBJS) +########################################################################## +# '_hashlib', '_hmac' and HACL*-based modules helpers + +Modules/_hashlib/hashlib_buffer.o: $(srcdir)/Modules/_hashlib/hashlib_buffer.c $(srcdir)/Modules/_hashlib/hashlib_buffer.h $(PYTHON_HEADERS) + $(CC) -I$(srcdir)/Modules/_hashlib -c $(PY_STDMODULE_CFLAGS) $(CCSHARED) -o $@ $(srcdir)/Modules/_hashlib/hashlib_buffer.c + ########################################################################## # HACL* library build # @@ -3323,22 +3329,31 @@ MODULE__CTYPES_TEST_DEPS=$(srcdir)/Modules/_ctypes/_ctypes_test_generated.c.h MODULE__CTYPES_MALLOC_CLOSURE=@MODULE__CTYPES_MALLOC_CLOSURE@ MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@ MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@ -MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h +MODULE__HASHLIB_DEPS= \ + $(srcdir)/Modules/_hashlib/hashlib_buffer.h \ + $(srcdir)/Modules/_hashlib/hashlib_fetch.h \ + $(srcdir)/Modules/_hashlib/hashlib_mutex.h + +MODULE__HASHLIB_LDEPS= \ + Modules/_hashlib/hashlib_buffer.o \ + Modules/_hashlib/hashlib_fetch.o + # HACL*-based cryptographic primitives -MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__MD5_LDEPS=$(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA1_HEADERS) $(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA1_LDEPS=$(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA2_LDEPS=$(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA3_HEADERS) $(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA3_LDEPS=$(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__BLAKE2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_BLAKE2_HEADERS) $(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__BLAKE2_LDEPS=$(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__HMAC_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HMAC_HEADERS) $(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__HMAC_LDEPS=$(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__MD5_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__MD5_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA1_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA1_HEADERS) $(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA1_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA2_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA2_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA3_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA3_HEADERS) $(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA3_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__BLAKE2_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_BLAKE2_HEADERS) $(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__BLAKE2_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) + +MODULE__HMAC_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_HMAC_HEADERS) $(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__HMAC_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h diff --git a/Modules/_hashlib/hashlib_buffer.c b/Modules/_hashlib/hashlib_buffer.c new file mode 100644 index 00000000000000..34811c6266fd64 --- /dev/null +++ b/Modules/_hashlib/hashlib_buffer.c @@ -0,0 +1,40 @@ +#include "hashlib_buffer.h" + +int +_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string) +{ + if (data != NULL && string == NULL) { + // called as H(data) or H(data=...) + *res = data; + return 1; + } + else if (data == NULL && string != NULL) { + // called as H(string=...) + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "the 'string' keyword parameter is deprecated since " + "Python 3.15 and slated for removal in Python 3.19; " + "use the 'data' keyword parameter or pass the data " + "to hash as a positional argument instead", 1) < 0) + { + *res = NULL; + return -1; + } + *res = string; + return 1; + } + else if (data == NULL && string == NULL) { + // fast path when no data is given + assert(!PyErr_Occurred()); + *res = NULL; + return 0; + } + else { + // called as H(data=..., string) + *res = NULL; + PyErr_SetString(PyExc_TypeError, + "'data' and 'string' are mutually exclusive " + "and support for 'string' keyword parameter " + "is slated for removal in a future version."); + return -1; + } +} diff --git a/Modules/_hashlib/hashlib_buffer.h b/Modules/_hashlib/hashlib_buffer.h new file mode 100644 index 00000000000000..13b0aa1ab3cb61 --- /dev/null +++ b/Modules/_hashlib/hashlib_buffer.h @@ -0,0 +1,60 @@ +#ifndef _HASHLIB_HASHLIB_BUFFER_H +#define _HASHLIB_HASHLIB_BUFFER_H + +#include "Python.h" + +/* + * Given an buffer-like OBJ, fill in the buffer VIEW with the result + * of PyObject_GetBuffer. + * + * On error, set an exception and execute the ERRACTION statements, + * e.g. 'return NULL' or 'goto error'. + * + * Parameters + * + * OBJ An object supporting the buffer API. + * VIEW A Py_buffer pointer to fill. + * ERRACTION The statements to execute on error. + */ +#define GET_BUFFER_VIEW_OR_ERROR(OBJ, VIEW, ERRACTION) \ + do { \ + if (PyUnicode_Check((OBJ))) { \ + PyErr_SetString(PyExc_TypeError, \ + "strings must be encoded before hashing"); \ + ERRACTION; \ + } \ + if (!PyObject_CheckBuffer((OBJ))) { \ + PyErr_SetString(PyExc_TypeError, \ + "object supporting the buffer API required"); \ + ERRACTION; \ + } \ + if (PyObject_GetBuffer((OBJ), (VIEW), PyBUF_SIMPLE) == -1) { \ + ERRACTION; \ + } \ + if ((VIEW)->ndim > 1) { \ + PyErr_SetString(PyExc_BufferError, \ + "buffer must be one-dimensional"); \ + PyBuffer_Release((VIEW)); \ + ERRACTION; \ + } \ + } while(0) + +/* Specialization of GET_BUFFER_VIEW_OR_ERROR() returning NULL on error. */ +#define GET_BUFFER_VIEW_OR_ERROUT(OBJ, VIEW) \ + GET_BUFFER_VIEW_OR_ERROR(OBJ, VIEW, return NULL) + +/* + * Allow to use the 'data' or 'string' keyword in hashlib.new() + * and other hash functions named constructors. + * + * - If 'data' and 'string' are both non-NULL, set an exception and return -1. + * - If 'data' and 'string' are both NULL, set '*res' to NULL and return 0. + * - Otherwise, set '*res' to 'data' or 'string' and return 1. A deprecation + * warning is set when 'string' is specified. + * + * The symbol is exported for '_hashlib' and HACL*-based extension modules. + */ +PyAPI_FUNC(int) +_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string); + +#endif // !_HASHLIB_HASHLIB_BUFFER_H diff --git a/Modules/_hashlib/hashlib_fetch.c b/Modules/_hashlib/hashlib_fetch.c new file mode 100644 index 00000000000000..6b772030ad87d0 --- /dev/null +++ b/Modules/_hashlib/hashlib_fetch.c @@ -0,0 +1 @@ +#include "hashlib_fetch.h" diff --git a/Modules/_hashlib/hashlib_fetch.h b/Modules/_hashlib/hashlib_fetch.h new file mode 100644 index 00000000000000..ee647ea6220f4d --- /dev/null +++ b/Modules/_hashlib/hashlib_fetch.h @@ -0,0 +1,130 @@ +/* + * Interface for fetching a message digest from a digest-like identifier. + * + * The following table summaries the possible algorthms: + * + * +----------+--------------+--------------+---------------------------------+ + * | Family | Algorithm | Python Name | Notes | + * +==========+==============+==============+=================================+ + * | MD @ | + * | +--------------+--------------+---------------------------------+ + * | | MD5 | "md5" | | + * +----------+--------------+--------------+---------------------------------+ + * | SHA1 @ | + * | +--------------+--------------+---------------------------------+ + * | | SHA1-160 | "sha1" | | + * +----------+--------------+--------------+---------------------------------+ + * | SHA2 @ | + * | +--------------+--------------+---------------------------------+ + * | | SHA2-224 | "sha224" | | + * | | SHA2-256 | "sha256" | | + * | | SHA2-384 | "sha384" | | + * | | SHA2-512 | "sha512" | | + * +----------+--------------+--------------+---------------------------------+ + * | SHA2t @ Truncated SHA2-512 | + * | +--------------+--------------+---------------------------------+ + * | | SHA2-512/224 | "sha512_224" | | + * | | SHA2-512/256 | "sha512_256" | | + * +----------+--------------+--------------+---------------------------------+ + * | SHA3 @ | + * | +--------------+--------------+---------------------------------+ + * | | SHA3-224 | "sha3_224" | | + * | | SHA3-256 | "sha3_256" | | + * | | SHA3-384 | "sha3_384" | | + * | | SHA3-512 | "sha3_512" | | + * +----------+--------------+--------------+---------------------------------+ + * | SHA3-XOF @ Extensible Output Functions | + * | +--------------+--------------+---------------------------------+ + * | | SHAKE-128 | "shake_128" | | + * | | SHAKE-256 | "shake_256" | | + * +----------+--------------+--------------+---------------------------------+ + * | BLAKE2 @ | + * | +--------------+--------------+---------------------------------+ + * | | BLAKE2b | "blake2b" | | + * | | BLAKE2s | "blake2s" | | + * +----------+--------------+--------------+---------------------------------+ + */ + +#ifndef _HASHLIB_HASHLIB_FETCH_H +#define _HASHLIB_HASHLIB_FETCH_H + +#include "Python.h" + +#define Py_HASHLIB_MD_NS(ATTR) Py_hashlib_message_digest_ ## ATTR +#define Py_HASHLIB_MD_FAMILY(FAMILY_ID) Py_HASHLIB_MD_NS(family_ ## FAMILY_ID) +#define Py_HASHLIB_MD_MEMBER(MEMBER_ID) Py_HASHLIB_MD_NS(member_ ## MEMBER_ID) + +#define Py_HASHLIB_MD_NAMES Py_HASHLIB_MD_NS(NAMES) +#define Py_HASHLIB_MD_COUNT Py_ARRAY_LENGTH(Py_HASHLIB_MD_NAMES) +#define Py_HASHLIB_MD_NAME(MEMBER_ID) \ + ( \ + assert(Py_HASHLIB_MD_NAME(MEMBER_ID) < Py_HASHLIB_MD_COUNT), \ + Py_HASHLIB_MD_NAMES[Py_HASHLIB_MD_MEMBER(MEMBER_ID)] \ + ) + +typedef enum { + Py_HASHLIB_MD_FAMILY(MD) = 0, + Py_HASHLIB_MD_FAMILY(SHA1), + Py_HASHLIB_MD_FAMILY(SHA2), + Py_HASHLIB_MD_FAMILY(SHA2t), + Py_HASHLIB_MD_FAMILY(SHA3), + Py_HASHLIB_MD_FAMILY(SHA3_XOF), + Py_HASHLIB_MD_FAMILY(BLAKE2), +} Py_HASHLIB_MD_NS(family); + +typedef enum { + /* MD-family */ + Py_HASHLIB_MD_MEMBER(md5) = 0, + /* SHA-1 family */ + Py_HASHLIB_MD_MEMBER(sha1), + /* SHA-2 family */ + Py_HASHLIB_MD_MEMBER(sha224), + Py_HASHLIB_MD_MEMBER(sha256), + Py_HASHLIB_MD_MEMBER(sha384), + Py_HASHLIB_MD_MEMBER(sha512), + /* Truncated SHA-2 family */ + Py_HASHLIB_MD_MEMBER(sha512_224), + Py_HASHLIB_MD_MEMBER(sha512_256), + /* SHA-3 family */ + Py_HASHLIB_MD_MEMBER(sha3_224), + Py_HASHLIB_MD_MEMBER(sha3_256), + Py_HASHLIB_MD_MEMBER(sha3_384), + Py_HASHLIB_MD_MEMBER(sha3_512), + /* SHA-3 XOF SHAKE family */ + Py_HASHLIB_MD_MEMBER(shake_128), + Py_HASHLIB_MD_MEMBER(shake_256), + /* BLAKE-2 family */ + Py_HASHLIB_MD_MEMBER(blake2b), + Py_HASHLIB_MD_MEMBER(blake2s), +} Py_HASHLIB_MD_NS(member); + +static const char *Py_HASHLIB_MD_NAMES[] = { +#define DECL_MESSAGE_DIGEST_NAME(ID) [Py_HASHLIB_MD_MEMBER(ID)] = #ID + /* MD-family */ + DECL_MESSAGE_DIGEST_NAME(md5), + /* SHA-1 family */ + DECL_MESSAGE_DIGEST_NAME(sha1), + /* SHA-2 family */ + DECL_MESSAGE_DIGEST_NAME(sha224), + DECL_MESSAGE_DIGEST_NAME(sha256), + DECL_MESSAGE_DIGEST_NAME(sha384), + DECL_MESSAGE_DIGEST_NAME(sha512), + /* Truncated SHA-2 family */ + DECL_MESSAGE_DIGEST_NAME(sha512_224), + DECL_MESSAGE_DIGEST_NAME(sha512_256), + /* SHA-3 family */ + DECL_MESSAGE_DIGEST_NAME(sha3_224), + DECL_MESSAGE_DIGEST_NAME(sha3_256), + DECL_MESSAGE_DIGEST_NAME(sha3_384), + DECL_MESSAGE_DIGEST_NAME(sha3_512), + /* SHA-3 XOF SHAKE family */ + DECL_MESSAGE_DIGEST_NAME(shake_128), + DECL_MESSAGE_DIGEST_NAME(shake_256), + /* BLAKE-2 family */ + DECL_MESSAGE_DIGEST_NAME(blake2b), + DECL_MESSAGE_DIGEST_NAME(blake2s), +#undef DECL_MESSAGE_DIGEST_NAME + NULL /* sentinel */ +}; + +#endif // !_HASHLIB_HASHLIB_FETCH_H diff --git a/Modules/_hashlib/hashlib_mutex.h b/Modules/_hashlib/hashlib_mutex.h new file mode 100644 index 00000000000000..cba7524f0cc668 --- /dev/null +++ b/Modules/_hashlib/hashlib_mutex.h @@ -0,0 +1,62 @@ +#ifndef _HASHLIB_HASHLIB_MUTEX_H +#define _HASHLIB_HASHLIB_MUTEX_H + +#include "Python.h" +#include "pycore_lock.h" // PyMutex + +/* + * Maximum number of bytes for a message for which the GIL is held + * when performing incremental hashing. + */ +#define HASHLIB_GIL_MINSIZE 2048 + +/* + * Helper code to synchronize access to the hash object when the GIL is + * released around a CPU consuming hashlib operation. All code paths that + * access a mutable part of obj must be enclosed in an ENTER_HASHLIB / + * LEAVE_HASHLIB block or explicitly acquire and release the lock inside + * a PY_BEGIN / END_ALLOW_THREADS block if they wish to release the GIL for + * an operation. + * + * These only drop the GIL if the lock acquisition itself is likely to + * block. Thus the non-blocking acquire gating the GIL release for a + * blocking lock acquisition. The intent of these macros is to surround + * the assumed always "fast" operations that you aren't releasing the + * GIL around. Otherwise use code similar to what you see in hash + * function update() methods. + */ + +/* Prevent undefined behaviors via multiple threads entering the C API. */ +#define HASHLIB_LOCK_HEAD \ + bool use_mutex; \ + PyMutex mutex; + +#ifdef Py_GIL_DISABLED +#define HASHLIB_INIT_MUTEX(OBJ) \ + do { \ + (OBJ)->mutex = (PyMutex){0}; \ + (OBJ)->use_mutex = true; \ + } while (0) +#else +#define HASHLIB_INIT_MUTEX(OBJ) \ + do { \ + (OBJ)->mutex = (PyMutex){0}; \ + (OBJ)->use_mutex = false; \ + } while (0) +#endif + +#define ENTER_HASHLIB(OBJ) \ + do { \ + if ((OBJ)->use_mutex) { \ + PyMutex_Lock(&(OBJ)->mutex); \ + } \ + } while (0) + +#define LEAVE_HASHLIB(OBJ) \ + do { \ + if ((OBJ)->use_mutex) { \ + PyMutex_Unlock(&(OBJ)->mutex); \ + } \ + } while (0) + +#endif // !_HASHLIB_HASHLIB_MUTEX_H diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index ce9603d5db841f..8d3db312709502 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -26,7 +26,8 @@ #include "pycore_hashtable.h" #include "pycore_strhex.h" // _Py_strhex() #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_RELAXED -#include "hashlib.h" +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_mutex.h" /* EVP is the preferred interface to hashing in OpenSSL */ #include @@ -279,20 +280,16 @@ get_hashlib_state(PyObject *module) typedef struct { PyObject_HEAD + HASHLIB_LOCK_HEAD EVP_MD_CTX *ctx; /* OpenSSL message digest context */ - // Prevents undefined behavior via multiple threads entering the C API. - bool use_mutex; - PyMutex mutex; /* OpenSSL context lock */ } HASHobject; #define HASHobject_CAST(op) ((HASHobject *)(op)) typedef struct { PyObject_HEAD + HASHLIB_LOCK_HEAD HMAC_CTX *ctx; /* OpenSSL hmac context */ - // Prevents undefined behavior via multiple threads entering the C API. - bool use_mutex; - PyMutex mutex; /* HMAC context lock */ } HMACobject; #define HMACobject_CAST(op) ((HMACobject *)(op)) @@ -1126,7 +1123,7 @@ _hashlib_HASH(PyObject *module, const char *digestname, PyObject *data_obj, if (view.buf && view.len) { if (view.len >= HASHLIB_GIL_MINSIZE) { - /* We do not initialize self->lock here as this is the constructor + /* Do not initialize self->mutex here as this is the constructor * where it is not yet possible to have concurrent access. */ Py_BEGIN_ALLOW_THREADS result = _hashlib_HASH_hash(self, view.buf, view.len); diff --git a/Modules/blake2module.c b/Modules/blake2module.c index 2ce8c0cd3d7b6f..84b499dbacb2c7 100644 --- a/Modules/blake2module.c +++ b/Modules/blake2module.c @@ -2,6 +2,7 @@ * Written in 2013 by Dmitry Chestnykh * Modified for CPython by Christian Heimes * Updated to use HACL* by Jonathan Protzenko + * Refactored by Bénédikt Tran <10796600+picnixz@users.noreply.github.com> * * To the extent possible under law, the author have dedicated all * copyright and related and neighboring rights to this software to @@ -14,10 +15,11 @@ #endif #include "Python.h" -#include "hashlib.h" +#include "pycore_moduleobject.h" #include "pycore_strhex.h" // _Py_strhex() #include "pycore_typeobject.h" -#include "pycore_moduleobject.h" +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_mutex.h" // QUICK CPU AUTODETECTION // @@ -353,6 +355,7 @@ type_to_impl(PyTypeObject *type) typedef struct { PyObject_HEAD + HASHLIB_LOCK_HEAD union { Hacl_Hash_Blake2s_state_t *blake2s_state; Hacl_Hash_Blake2b_state_t *blake2b_state; @@ -364,8 +367,6 @@ typedef struct { #endif }; blake2_impl impl; - bool use_mutex; - PyMutex mutex; } Blake2Object; #define _Blake2Object_CAST(op) ((Blake2Object *)(op)) @@ -647,6 +648,8 @@ py_blake2_new(PyTypeObject *type, PyObject *data, int digest_size, Py_buffer buf; GET_BUFFER_VIEW_OR_ERROR(data, &buf, goto error); if (buf.len >= HASHLIB_GIL_MINSIZE) { + /* Do not initialize self->mutex here as this is the constructor + * where it is not yet possible to have concurrent access. */ Py_BEGIN_ALLOW_THREADS update(self, buf.buf, buf.len); Py_END_ALLOW_THREADS diff --git a/Modules/hashlib.h b/Modules/hashlib.h deleted file mode 100644 index e82ec92be25c57..00000000000000 --- a/Modules/hashlib.h +++ /dev/null @@ -1,116 +0,0 @@ -/* Common code for use by all hashlib related modules. */ - -#include "pycore_lock.h" // PyMutex - -/* - * Given a PyObject* obj, fill in the Py_buffer* viewp with the result - * of PyObject_GetBuffer. Sets an exception and issues the erraction - * on any errors, e.g. 'return NULL' or 'goto error'. - */ -#define GET_BUFFER_VIEW_OR_ERROR(obj, viewp, erraction) do { \ - if (PyUnicode_Check((obj))) { \ - PyErr_SetString(PyExc_TypeError, \ - "Strings must be encoded before hashing");\ - erraction; \ - } \ - if (!PyObject_CheckBuffer((obj))) { \ - PyErr_SetString(PyExc_TypeError, \ - "object supporting the buffer API required"); \ - erraction; \ - } \ - if (PyObject_GetBuffer((obj), (viewp), PyBUF_SIMPLE) == -1) { \ - erraction; \ - } \ - if ((viewp)->ndim > 1) { \ - PyErr_SetString(PyExc_BufferError, \ - "Buffer must be single dimension"); \ - PyBuffer_Release((viewp)); \ - erraction; \ - } \ - } while(0) - -#define GET_BUFFER_VIEW_OR_ERROUT(obj, viewp) \ - GET_BUFFER_VIEW_OR_ERROR(obj, viewp, return NULL) - -/* - * Helper code to synchronize access to the hash object when the GIL is - * released around a CPU consuming hashlib operation. All code paths that - * access a mutable part of obj must be enclosed in an ENTER_HASHLIB / - * LEAVE_HASHLIB block or explicitly acquire and release the lock inside - * a PY_BEGIN / END_ALLOW_THREADS block if they wish to release the GIL for - * an operation. - * - * These only drop the GIL if the lock acquisition itself is likely to - * block. Thus the non-blocking acquire gating the GIL release for a - * blocking lock acquisition. The intent of these macros is to surround - * the assumed always "fast" operations that you aren't releasing the - * GIL around. Otherwise use code similar to what you see in hash - * function update() methods. - */ - -#include "pythread.h" -#define ENTER_HASHLIB(obj) \ - if ((obj)->use_mutex) { \ - PyMutex_Lock(&(obj)->mutex); \ - } -#define LEAVE_HASHLIB(obj) \ - if ((obj)->use_mutex) { \ - PyMutex_Unlock(&(obj)->mutex); \ - } - -#ifdef Py_GIL_DISABLED -#define HASHLIB_INIT_MUTEX(obj) \ - do { \ - (obj)->mutex = (PyMutex){0}; \ - (obj)->use_mutex = true; \ - } while (0) -#else -#define HASHLIB_INIT_MUTEX(obj) \ - do { \ - (obj)->mutex = (PyMutex){0}; \ - (obj)->use_mutex = false; \ - } while (0) -#endif - -/* TODO(gpshead): We should make this a module or class attribute - * to allow the user to optimize based on the platform they're using. */ -#define HASHLIB_GIL_MINSIZE 2048 - -static inline int -_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string) -{ - if (data != NULL && string == NULL) { - // called as H(data) or H(data=...) - *res = data; - return 1; - } - else if (data == NULL && string != NULL) { - // called as H(string=...) - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "the 'string' keyword parameter is deprecated since " - "Python 3.15 and slated for removal in Python 3.19; " - "use the 'data' keyword parameter or pass the data " - "to hash as a positional argument instead", 1) < 0) - { - *res = NULL; - return -1; - } - *res = string; - return 1; - } - else if (data == NULL && string == NULL) { - // fast path when no data is given - assert(!PyErr_Occurred()); - *res = NULL; - return 0; - } - else { - // called as H(data=..., string) - *res = NULL; - PyErr_SetString(PyExc_TypeError, - "'data' and 'string' are mutually exclusive " - "and support for 'string' keyword parameter " - "is slated for removal in a future version."); - return -1; - } -} diff --git a/Modules/hmacmodule.c b/Modules/hmacmodule.c index b404d5732ec857..6c6593f9522bee 100644 --- a/Modules/hmacmodule.c +++ b/Modules/hmacmodule.c @@ -44,9 +44,10 @@ #include "_hacl/Hacl_Streaming_HMAC.h" // Hacl_Agile_Hash_* identifiers #include "_hacl/Hacl_Streaming_Types.h" // Hacl_Streaming_Types_error_code -#include +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_mutex.h" -#include "hashlib.h" +#include // --- Reusable error messages ------------------------------------------------ diff --git a/Modules/md5module.c b/Modules/md5module.c index 08dbcd2cbce844..7fe956b9f5fc3b 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -8,6 +8,7 @@ Andrew Kuchling (amk@amk.ca) Greg Stein (gstein@lyra.org) Trevor Perrin (trevp@trevp.net) + Bénédikt Tran (10796600+picnixz@users.noreply.github.com) Copyright (C) 2005-2007 Gregory P. Smith (greg@krypto.org) Licensed to PSF under a Contributor Agreement. @@ -21,7 +22,8 @@ #endif #include "Python.h" -#include "hashlib.h" +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_mutex.h" /*[clinic input] module _md5 @@ -36,12 +38,9 @@ class MD5Type "MD5object *" "&PyType_Type" #include "_hacl/Hacl_Hash_MD5.h" - typedef struct { PyObject_HEAD - // Prevents undefined behavior via multiple threads entering the C API. - bool use_mutex; - PyMutex mutex; + HASHLIB_LOCK_HEAD Hacl_Hash_MD5_state_t *hash_state; } MD5object; @@ -320,7 +319,7 @@ _md5_md5_impl(PyObject *module, PyObject *data, int usedforsecurity, if (string) { if (buf.len >= HASHLIB_GIL_MINSIZE) { - /* We do not initialize self->lock here as this is the constructor + /* Do not initialize self->mutex here as this is the constructor * where it is not yet possible to have concurrent access. */ Py_BEGIN_ALLOW_THREADS update(new->hash_state, buf.buf, buf.len); diff --git a/Modules/sha1module.c b/Modules/sha1module.c index a746bf74f8d4c1..6a0fbcd99c28cc 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -8,6 +8,7 @@ Andrew Kuchling (amk@amk.ca) Greg Stein (gstein@lyra.org) Trevor Perrin (trevp@trevp.net) + Bénédikt Tran (10796600+picnixz@users.noreply.github.com) Copyright (C) 2005-2007 Gregory P. Smith (greg@krypto.org) Licensed to PSF under a Contributor Agreement. @@ -20,9 +21,10 @@ #endif #include "Python.h" -#include "hashlib.h" #include "pycore_strhex.h" // _Py_strhex() #include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_mutex.h" /*[clinic input] module _sha1 @@ -39,10 +41,7 @@ class SHA1Type "SHA1object *" "&PyType_Type" typedef struct { PyObject_HEAD - // Prevents undefined behavior via multiple threads entering the C API. - bool use_mutex; - PyMutex mutex; - PyThread_type_lock lock; + HASHLIB_LOCK_HEAD Hacl_Hash_SHA1_state_t *hash_state; } SHA1object; @@ -315,7 +314,7 @@ _sha1_sha1_impl(PyObject *module, PyObject *data, int usedforsecurity, } if (string) { if (buf.len >= HASHLIB_GIL_MINSIZE) { - /* We do not initialize self->lock here as this is the constructor + /* Do not initialize self->mutex here as this is the constructor * where it is not yet possible to have concurrent access. */ Py_BEGIN_ALLOW_THREADS update(new->hash_state, buf.buf, buf.len); diff --git a/Modules/sha2module.c b/Modules/sha2module.c index 72931910c5d720..e3e5bde88e6992 100644 --- a/Modules/sha2module.c +++ b/Modules/sha2module.c @@ -9,6 +9,7 @@ Greg Stein (gstein@lyra.org) Trevor Perrin (trevp@trevp.net) Jonathan Protzenko (jonathan@protzenko.fr) + Bénédikt Tran (10796600+picnixz@users.noreply.github.com) Copyright (C) 2005-2007 Gregory P. Smith (greg@krypto.org) Licensed to PSF under a Contributor Agreement. @@ -23,10 +24,10 @@ #include "Python.h" #include "pycore_bitutils.h" // _Py_bswap32() #include "pycore_moduleobject.h" // _PyModule_GetState() -#include "pycore_typeobject.h" // _PyType_GetModuleState() #include "pycore_strhex.h" // _Py_strhex() - -#include "hashlib.h" +#include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_mutex.h" /*[clinic input] module _sha2 @@ -51,10 +52,8 @@ class SHA512Type "SHA512object *" "&PyType_Type" typedef struct { PyObject_HEAD + HASHLIB_LOCK_HEAD int digestsize; - // Prevents undefined behavior via multiple threads entering the C API. - bool use_mutex; - PyMutex mutex; Hacl_Hash_SHA2_state_t_256 *state; } SHA256object; @@ -639,7 +638,7 @@ _sha2_sha256_impl(PyObject *module, PyObject *data, int usedforsecurity, } if (string) { if (buf.len >= HASHLIB_GIL_MINSIZE) { - /* We do not initialize self->lock here as this is the constructor + /* Do not initialize self->mutex here as this is the constructor * where it is not yet possible to have concurrent access. */ Py_BEGIN_ALLOW_THREADS update_256(new->state, buf.buf, buf.len); diff --git a/Modules/sha3module.c b/Modules/sha3module.c index cfbf0cbcc042c5..202de3eb690f73 100644 --- a/Modules/sha3module.c +++ b/Modules/sha3module.c @@ -9,6 +9,7 @@ * Greg Stein (gstein@lyra.org) * Trevor Perrin (trevp@trevp.net) * Gregory P. Smith (greg@krypto.org) + * Bénédikt Tran (10796600+picnixz@users.noreply.github.com) * * Copyright (C) 2012-2022 Christian Heimes (christian@python.org) * Licensed to PSF under a Contributor Agreement. @@ -22,7 +23,8 @@ #include "Python.h" #include "pycore_strhex.h" // _Py_strhex() #include "pycore_typeobject.h" // _PyType_GetModuleState() -#include "hashlib.h" +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_mutex.h" #define SHA3_MAX_DIGESTSIZE 64 /* 64 Bytes (512 Bits) for 224 to 512 */ @@ -60,9 +62,7 @@ class _sha3.shake_256 "SHA3object *" "&SHAKE256type" typedef struct { PyObject_HEAD - // Prevents undefined behavior via multiple threads entering the C API. - bool use_mutex; - PyMutex mutex; + HASHLIB_LOCK_HEAD Hacl_Hash_SHA3_state_t *hash_state; } SHA3object; @@ -164,7 +164,7 @@ py_sha3_new_impl(PyTypeObject *type, PyObject *data_obj, int usedforsecurity, if (data) { GET_BUFFER_VIEW_OR_ERROR(data, &buf, goto error); if (buf.len >= HASHLIB_GIL_MINSIZE) { - /* We do not initialize self->lock here as this is the constructor + /* Do not initialize self->mutex here as this is the constructor * where it is not yet possible to have concurrent access. */ Py_BEGIN_ALLOW_THREADS sha3_update(self->hash_state, buf.buf, buf.len); diff --git a/PCbuild/_hashlib.vcxproj b/PCbuild/_hashlib.vcxproj index 2cd205224bc089..9e6dcee40793d3 100644 --- a/PCbuild/_hashlib.vcxproj +++ b/PCbuild/_hashlib.vcxproj @@ -100,6 +100,12 @@ + + + + + + diff --git a/PCbuild/_hashlib.vcxproj.filters b/PCbuild/_hashlib.vcxproj.filters index 7a0700c007f644..d465d92a956eda 100644 --- a/PCbuild/_hashlib.vcxproj.filters +++ b/PCbuild/_hashlib.vcxproj.filters @@ -18,4 +18,4 @@ Resource Files - \ No newline at end of file + diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 32a8f2dbad3d5e..5695923019d218 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -443,6 +443,10 @@ HACL_CAN_COMPILE_VEC128;%(PreprocessorDefinitions) /arch:AVX %(AdditionalOptions) + + + + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 0e6d42cc959ba5..ad3fc157ff6308 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -255,6 +255,15 @@ Include + + Modules + + + Modules + + + Modules + Modules @@ -971,6 +980,12 @@ Modules + + Modules + + + Modules + Modules diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 15b18f5286b399..f3154f5a70ffda 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -236,6 +236,7 @@ Modules/_decimal/_decimal.c - signal_map_template - Modules/_decimal/_decimal.c - ssize_constants - Modules/_decimal/_decimal.c - INVALID_SIGNALDICT_ERROR_MSG - Modules/_elementtree.c - ExpatMemoryHandler - +Modules/_hashlib/hashlib_fetch.h - Py_hashlib_message_digest_NAMES - Modules/_hashopenssl.c - py_hashes - Modules/_hacl/Hacl_Hash_SHA1.c - _h0 - Modules/_hacl/Hacl_Hash_MD5.c - _h0 - diff --git a/configure b/configure index 029bf527da4e3d..ae87370302defd 100755 --- a/configure +++ b/configure @@ -29970,6 +29970,7 @@ SRCDIRS="\ Modules/_decimal \ Modules/_decimal/libmpdec \ Modules/_hacl \ + Modules/_hashlib \ Modules/_io \ Modules/_multiprocessing \ Modules/_sqlite \ @@ -32548,6 +32549,12 @@ then : fi +############################################################################### +# Cryptographic primitives +LIBHASHLIB_INCL='-I$(srcdir)/Modules/_hashlib' +LIBHASHLIB_LIBS='Modules/_hashlib/hashlib_buffer.o Modules/_hashlib/hashlib_fetch.o' + + ############################################################################### # HACL* compilation and linking configuration (contact: @picnixz) # @@ -32796,8 +32803,8 @@ fi if test "x$py_cv_module__md5" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__MD5_CFLAGS=$LIBHACL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__MD5_LDFLAGS=\$($LIBHACL_MD5_LDFLAGS)$as_nl" + as_fn_append MODULE_BLOCK "MODULE__MD5_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INCL$as_nl" + as_fn_append MODULE_BLOCK "MODULE__MD5_LDFLAGS=\$($LIBHACL_MD5_LDFLAGS) $LIBHASHLIB_LIBS$as_nl" fi if test "$py_cv_module__md5" = yes; then @@ -32841,8 +32848,8 @@ fi if test "x$py_cv_module__sha1" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA1_CFLAGS=$LIBHACL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__SHA1_LDFLAGS=\$($LIBHACL_SHA1_LDFLAGS)$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA1_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INCL$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA1_LDFLAGS=\$($LIBHACL_SHA1_LDFLAGS) $LIBHASHLIB_LIBS$as_nl" fi if test "$py_cv_module__sha1" = yes; then @@ -32886,8 +32893,8 @@ fi if test "x$py_cv_module__sha2" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA2_CFLAGS=$LIBHACL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__SHA2_LDFLAGS=\$($LIBHACL_SHA2_LDFLAGS)$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA2_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INCL$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA2_LDFLAGS=\$($LIBHACL_SHA2_LDFLAGS) $LIBHASHLIB_LIBS$as_nl" fi if test "$py_cv_module__sha2" = yes; then @@ -32931,8 +32938,8 @@ fi if test "x$py_cv_module__sha3" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA3_CFLAGS=$LIBHACL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__SHA3_LDFLAGS=\$($LIBHACL_SHA3_LDFLAGS)$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA3_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INCL$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA3_LDFLAGS=\$($LIBHACL_SHA3_LDFLAGS) $LIBHASHLIB_LIBS$as_nl" fi if test "$py_cv_module__sha3" = yes; then @@ -32976,8 +32983,8 @@ fi if test "x$py_cv_module__blake2" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__BLAKE2_CFLAGS=$LIBHACL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__BLAKE2_LDFLAGS=\$($LIBHACL_BLAKE2_LDFLAGS)$as_nl" + as_fn_append MODULE_BLOCK "MODULE__BLAKE2_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INCL$as_nl" + as_fn_append MODULE_BLOCK "MODULE__BLAKE2_LDFLAGS=\$($LIBHACL_BLAKE2_LDFLAGS) $LIBHASHLIB_LIBS$as_nl" fi if test "$py_cv_module__blake2" = yes; then @@ -33022,8 +33029,8 @@ fi if test "x$py_cv_module__hmac" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__HMAC_CFLAGS=$LIBHACL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__HMAC_LDFLAGS=\$($LIBHACL_HMAC_LDFLAGS)$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HMAC_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INCL$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HMAC_LDFLAGS=\$($LIBHACL_HMAC_LDFLAGS) $LIBHASHLIB_LIBS$as_nl" fi if test "$py_cv_module__hmac" = yes; then @@ -33692,8 +33699,8 @@ fi if test "x$py_cv_module__hashlib" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__HASHLIB_CFLAGS=$OPENSSL_INCLUDES$as_nl" - as_fn_append MODULE_BLOCK "MODULE__HASHLIB_LDFLAGS=$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HASHLIB_CFLAGS=$OPENSSL_INCLUDES $LIBHASHLIB_INCL$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HASHLIB_LDFLAGS=$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS $LIBHASHLIB_LIBS$as_nl" fi if test "$py_cv_module__hashlib" = yes; then diff --git a/configure.ac b/configure.ac index 371b2e8ed73525..f180e4ca58e805 100644 --- a/configure.ac +++ b/configure.ac @@ -7199,6 +7199,7 @@ SRCDIRS="\ Modules/_decimal \ Modules/_decimal/libmpdec \ Modules/_hacl \ + Modules/_hashlib \ Modules/_io \ Modules/_multiprocessing \ Modules/_sqlite \ @@ -7971,6 +7972,12 @@ PY_STDLIB_MOD_SIMPLE([_codecs_tw]) PY_STDLIB_MOD_SIMPLE([_multibytecodec]) PY_STDLIB_MOD_SIMPLE([unicodedata]) +############################################################################### +# Cryptographic primitives +LIBHASHLIB_INCL='-I$(srcdir)/Modules/_hashlib' +LIBHASHLIB_LIBS='Modules/_hashlib/hashlib_buffer.o Modules/_hashlib/hashlib_fetch.o' + + ############################################################################### # HACL* compilation and linking configuration (contact: @picnixz) # @@ -8105,7 +8112,9 @@ dnl The EXTNAME is the name of the extension module being built. AC_DEFUN([PY_HACL_CREATE_MODULE], [ AS_VAR_PUSHDEF([v], [[LIBHACL_][$1][_LDFLAGS]]) AS_VAR_SET([v], [[LIBHACL_][$1][_LIB_${LIBHACL_LDEPS_LIBTYPE}]]) - PY_STDLIB_MOD([$2], [$3], [], [$LIBHACL_CFLAGS], [\$($v)]) + PY_STDLIB_MOD([$2], [$3], [], + [$LIBHACL_CFLAGS $LIBHASHLIB_INCL], + [\$($v) $LIBHASHLIB_LIBS]) AS_VAR_POPDEF([v]) ]) @@ -8176,7 +8185,8 @@ dnl OpenSSL bindings PY_STDLIB_MOD([_ssl], [], [test "$ac_cv_working_openssl_ssl" = yes], [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $OPENSSL_LIBS]) PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes], - [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS]) + [$OPENSSL_INCLUDES $LIBHASHLIB_INCL], + [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS $LIBHASHLIB_LIBS]) dnl test modules PY_STDLIB_MOD([_testcapi], From e0d2e784d49d06164b5ed493cdda3e34d60617e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 14 Jul 2025 13:05:15 +0200 Subject: [PATCH 03/13] post-merge --- Modules/_hashlib/hashlib_mutex.h | 96 +++++++++++++++++++------------- 1 file changed, 58 insertions(+), 38 deletions(-) diff --git a/Modules/_hashlib/hashlib_mutex.h b/Modules/_hashlib/hashlib_mutex.h index cba7524f0cc668..d6924a2ef61e81 100644 --- a/Modules/_hashlib/hashlib_mutex.h +++ b/Modules/_hashlib/hashlib_mutex.h @@ -5,58 +5,78 @@ #include "pycore_lock.h" // PyMutex /* - * Maximum number of bytes for a message for which the GIL is held - * when performing incremental hashing. + * Message length above which the GIL is to be released + * when performing hashing operations. */ #define HASHLIB_GIL_MINSIZE 2048 /* * Helper code to synchronize access to the hash object when the GIL is - * released around a CPU consuming hashlib operation. All code paths that - * access a mutable part of obj must be enclosed in an ENTER_HASHLIB / - * LEAVE_HASHLIB block or explicitly acquire and release the lock inside - * a PY_BEGIN / END_ALLOW_THREADS block if they wish to release the GIL for - * an operation. + * released around a CPU consuming hashlib operation. * - * These only drop the GIL if the lock acquisition itself is likely to - * block. Thus the non-blocking acquire gating the GIL release for a - * blocking lock acquisition. The intent of these macros is to surround - * the assumed always "fast" operations that you aren't releasing the - * GIL around. Otherwise use code similar to what you see in hash - * function update() methods. + * Code accessing a mutable part of the hash object must be enclosed in + * an HASHLIB_{ACQUIRE,RELEASE}_LOCK block or explicitly acquire and release + * the mutex inside a Py_BEGIN_ALLOW_THREADS -- Py_END_ALLOW_THREADS block if + * they wish to release the GIL for an operation. */ -/* Prevent undefined behaviors via multiple threads entering the C API. */ -#define HASHLIB_LOCK_HEAD \ - bool use_mutex; \ +#define HASHLIB_OBJECT_HEAD \ + PyObject_HEAD \ + /* Guard against race conditions during incremental update(). */ \ PyMutex mutex; -#ifdef Py_GIL_DISABLED -#define HASHLIB_INIT_MUTEX(OBJ) \ - do { \ - (OBJ)->mutex = (PyMutex){0}; \ - (OBJ)->use_mutex = true; \ +#define HASHLIB_INIT_MUTEX(OBJ) \ + do { \ + (OBJ)->mutex = (PyMutex){0}; \ } while (0) -#else -#define HASHLIB_INIT_MUTEX(OBJ) \ - do { \ - (OBJ)->mutex = (PyMutex){0}; \ - (OBJ)->use_mutex = false; \ - } while (0) -#endif -#define ENTER_HASHLIB(OBJ) \ - do { \ - if ((OBJ)->use_mutex) { \ - PyMutex_Lock(&(OBJ)->mutex); \ - } \ +#define HASHLIB_ACQUIRE_LOCK(OBJ) PyMutex_Lock(&(OBJ)->mutex) +#define HASHLIB_RELEASE_LOCK(OBJ) PyMutex_Unlock(&(OBJ)->mutex) + +// Macros for executing code while conditionally holding the GIL. +// +// These only drop the GIL if the lock acquisition itself is likely to +// block. Thus the non-blocking acquire gating the GIL release for a +// blocking lock acquisition. The intent of these macros is to surround +// the assumed always "fast" operations that you aren't releasing the +// GIL around. + +/* + * Execute a suite of C statements 'STATEMENTS'. + * + * The GIL is held if 'SIZE' is below the HASHLIB_GIL_MINSIZE threshold. + */ +#define HASHLIB_EXTERNAL_INSTRUCTIONS_UNLOCKED(SIZE, STATEMENTS) \ + do { \ + if ((SIZE) > HASHLIB_GIL_MINSIZE) { \ + Py_BEGIN_ALLOW_THREADS \ + STATEMENTS; \ + Py_END_ALLOW_THREADS \ + } \ + else { \ + STATEMENTS; \ + } \ } while (0) -#define LEAVE_HASHLIB(OBJ) \ - do { \ - if ((OBJ)->use_mutex) { \ - PyMutex_Unlock(&(OBJ)->mutex); \ - } \ +/* + * Lock 'OBJ' and execute a suite of C statements 'STATEMENTS'. + * + * The GIL is held if 'SIZE' is below the HASHLIB_GIL_MINSIZE threshold. + */ +#define HASHLIB_EXTERNAL_INSTRUCTIONS_LOCKED(OBJ, SIZE, STATEMENTS) \ + do { \ + if ((SIZE) > HASHLIB_GIL_MINSIZE) { \ + Py_BEGIN_ALLOW_THREADS \ + HASHLIB_ACQUIRE_LOCK(OBJ); \ + STATEMENTS; \ + HASHLIB_RELEASE_LOCK(OBJ); \ + Py_END_ALLOW_THREADS \ + } \ + else { \ + HASHLIB_ACQUIRE_LOCK(OBJ); \ + STATEMENTS; \ + HASHLIB_RELEASE_LOCK(OBJ); \ + } \ } while (0) #endif // !_HASHLIB_HASHLIB_MUTEX_H From a8c492d04986bd44f897dbc7858df1ec91544432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 14 Jul 2025 13:08:08 +0200 Subject: [PATCH 04/13] post-merge --- PCbuild/pythoncore.vcxproj.filters | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index ad3fc157ff6308..9cc395cac31b15 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -256,13 +256,13 @@ Include - Modules + Modules\_hashlib - Modules + Modules\_hashlib - Modules + Modules\_hashlib Modules @@ -981,10 +981,10 @@ Modules - Modules + Modules\_hashlib - Modules + Modules\_hashlib Modules From 71a1b8f7896fd4e22c6839d52efdd333a9e5e20f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 14 Jul 2025 13:19:11 +0200 Subject: [PATCH 05/13] simplify CI --- .github/workflows/build.yml | 375 ++++++++++++++++++++++++++++++++++-- 1 file changed, 362 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6312530669623e..e09ee877c821c0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,6 +43,12 @@ jobs: # uses: ./.github/workflows/reusable-context.yml + check-docs: + name: Docs + needs: build-context + if: fromJSON(needs.build-context.outputs.run-docs) + uses: ./.github/workflows/reusable-docs.yml + check-autoconf-regen: name: 'Check if Autoconf files are up to date' # Don't use ubuntu-latest but a specific version to make the job @@ -154,12 +160,15 @@ jobs: needs: build-context if: fromJSON(needs.build-context.outputs.run-windows-tests) strategy: - fail-fast: true + fail-fast: false matrix: arch: - x64 + - Win32 + - arm64 free-threading: - false + - true exclude: # Skip Win32 on free-threaded builds - { arch: Win32, free-threading: true } @@ -168,6 +177,22 @@ jobs: arch: ${{ matrix.arch }} free-threading: ${{ matrix.free-threading }} + build-windows-msi: + name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category + Windows MSI${{ '' }} + needs: build-context + if: fromJSON(needs.build-context.outputs.run-windows-msi) + strategy: + fail-fast: false + matrix: + arch: + - x86 + - x64 + - arm64 + uses: ./.github/workflows/reusable-windows-msi.yml + with: + arch: ${{ matrix.arch }} + build-macos: name: >- macOS @@ -175,17 +200,20 @@ jobs: needs: build-context if: needs.build-context.outputs.run-tests == 'true' strategy: - fail-fast: true + fail-fast: false matrix: # Cirrus and macos-14 are M1, macos-13 is default GHA Intel. # macOS 13 only runs tests against the GIL-enabled CPython. # Cirrus used for upstream, macos-14 for forks. os: - ghcr.io/cirruslabs/macos-runner:sonoma + - macos-14 + - macos-13 is-fork: # only used for the exclusion trick - ${{ github.repository_owner != 'python' }} free-threading: - false + - true exclude: - os: ghcr.io/cirruslabs/macos-runner:sonoma is-fork: true @@ -207,14 +235,17 @@ jobs: needs: build-context if: needs.build-context.outputs.run-tests == 'true' strategy: - fail-fast: true + fail-fast: false matrix: bolt: - false + - true free-threading: - false + - true os: - ubuntu-24.04 + - ubuntu-24.04-arm exclude: # Do not test BOLT with free-threading, to conserve resources - bolt: true @@ -229,17 +260,17 @@ jobs: free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} - build-ubuntu-ssltests: + build-ubuntu-ssltests-openssl: name: 'Ubuntu SSL tests with OpenSSL' runs-on: ${{ matrix.os }} timeout-minutes: 60 needs: build-context if: needs.build-context.outputs.run-tests == 'true' strategy: - fail-fast: true + fail-fast: false matrix: os: [ubuntu-24.04] - openssl_ver: [3.0.16] + openssl_ver: [3.0.16, 3.1.8, 3.2.4, 3.3.3, 3.4.1] # See Tools/ssl/make_ssl_data.py for notes on adding a new version env: OPENSSL_VER: ${{ matrix.openssl_ver }} @@ -291,6 +322,81 @@ jobs: - name: SSL tests run: ./python Lib/test/ssltests.py + build-ubuntu-ssltests-awslc: + name: 'Ubuntu SSL tests with AWS-LC' + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04] + awslc_ver: [1.55.0] + env: + AWSLC_VER: ${{ matrix.awslc_ver}} + MULTISSL_DIR: ${{ github.workspace }}/multissl + OPENSSL_DIR: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }} + LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }}/lib + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: config.cache + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure SSL lib env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}/lib" >> "$GITHUB_ENV" + - name: 'Restore AWS-LC build' + id: cache-aws-lc + uses: actions/cache@v4 + with: + path: ./multissl/aws-lc/${{ matrix.awslc_ver }} + key: ${{ matrix.os }}-multissl-aws-lc-${{ matrix.awslc_ver }} + - name: Install AWS-LC + if: steps.cache-aws-lc.outputs.cache-hit != 'true' + run: | + python3 Tools/ssl/multissltests.py \ + --steps=library \ + --base-directory "$MULTISSL_DIR" \ + --awslc ${{ matrix.awslc_ver }} \ + --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: false + - name: Configure CPython + run: | + ./configure CFLAGS="-fdiagnostics-format=json" \ + --config-cache \ + --enable-slower-safety \ + --with-pydebug \ + --with-openssl="$OPENSSL_DIR" \ + --with-builtin-hashlib-hashes=blake2 \ + --with-ssl-default-suites=openssl + - name: Build CPython + run: make -j + - name: Display build info + run: make pythoninfo + - name: Verify python is linked to AWS-LC + run: ./python -c 'import ssl; print(ssl.OPENSSL_VERSION)' | grep AWS-LC + - name: SSL tests + run: ./python Lib/test/ssltests.py + build-wasi: name: 'WASI' needs: build-context @@ -299,6 +405,211 @@ jobs: with: config_hash: ${{ needs.build-context.outputs.config-hash }} + test-hypothesis: + name: "Hypothesis tests on Ubuntu" + runs-on: ubuntu-24.04 + timeout-minutes: 60 + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + env: + OPENSSL_VER: 3.0.16 + PYTHONSTRICTEXTENSIONBUILD: 1 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure OpenSSL env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" + - name: 'Restore OpenSSL build' + id: cache-openssl + uses: actions/cache@v4 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: false + - name: Setup directory envs for out-of-tree builds + run: | + echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" + echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" + - name: Create directories for read-only out-of-tree builds + run: mkdir -p "$CPYTHON_RO_SRCDIR" "$CPYTHON_BUILDDIR" + - name: Bind mount sources read-only + run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: ${{ env.CPYTHON_BUILDDIR }}/config.cache + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} + - name: Configure CPython out-of-tree + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: | + ../cpython-ro-srcdir/configure \ + --config-cache \ + --with-pydebug \ + --enable-slower-safety \ + --with-openssl="$OPENSSL_DIR" + - name: Build CPython out-of-tree + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: make -j4 + - name: Display build info + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: make pythoninfo + - name: Remount sources writable for tests + # some tests write to srcdir, lack of pyc files slows down testing + run: sudo mount "$CPYTHON_RO_SRCDIR" -oremount,rw + - name: Setup directory envs for out-of-tree builds + run: | + echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" + - name: "Create hypothesis venv" + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: | + VENV_LOC=$(realpath -m .)/hypovenv + VENV_PYTHON=$VENV_LOC/bin/python + echo "HYPOVENV=${VENV_LOC}" >> "$GITHUB_ENV" + echo "VENV_PYTHON=${VENV_PYTHON}" >> "$GITHUB_ENV" + ./python -m venv "$VENV_LOC" && "$VENV_PYTHON" -m pip install -r "${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt" + - name: 'Restore Hypothesis database' + id: cache-hypothesis-database + uses: actions/cache@v4 + with: + path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/ + key: hypothesis-database-${{ github.head_ref || github.run_id }} + restore-keys: | + hypothesis-database- + - name: "Run tests" + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: | + # Most of the excluded tests are slow test suites with no property tests + # + # (GH-104097) test_sysconfig is skipped because it has tests that are + # failing when executed from inside a virtual environment. + "${VENV_PYTHON}" -m test \ + -W \ + --slowest \ + -j4 \ + --timeout 900 \ + -x test_asyncio \ + -x test_multiprocessing_fork \ + -x test_multiprocessing_forkserver \ + -x test_multiprocessing_spawn \ + -x test_concurrent_futures \ + -x test_socket \ + -x test_subprocess \ + -x test_signal \ + -x test_sysconfig + - uses: actions/upload-artifact@v4 + if: always() + with: + name: hypothesis-example-db + path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/examples/ + + build-asan: + name: 'Address sanitizer' + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04] + env: + OPENSSL_VER: 3.0.16 + PYTHONSTRICTEXTENSIONBUILD: 1 + ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: config.cache + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Set up GCC-10 for ASAN + uses: egor-tensin/setup-gcc@v1 + with: + version: 10 + - name: Configure OpenSSL env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" + - name: 'Restore OpenSSL build' + id: cache-openssl + uses: actions/cache@v4 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" + - name: Configure CPython + run: ./configure --config-cache --with-address-sanitizer --without-pymalloc + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: xvfb-run make ci + + build-tsan: + name: >- + Thread sanitizer + ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + strategy: + fail-fast: false + matrix: + free-threading: + - false + - true + uses: ./.github/workflows/reusable-tsan.yml + with: + config_hash: ${{ needs.build-context.outputs.config-hash }} + free-threading: ${{ matrix.free-threading }} + + build-ubsan: + name: Undefined behavior sanitizer + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + uses: ./.github/workflows/reusable-ubsan.yml + with: + config_hash: ${{ needs.build-context.outputs.config-hash }} + cross-build-linux: name: Cross build Linux runs-on: ubuntu-latest @@ -339,6 +650,45 @@ jobs: run: | "$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed + # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ + cifuzz: + name: CIFuzz + runs-on: ubuntu-latest + timeout-minutes: 60 + needs: build-context + if: needs.build-context.outputs.run-ci-fuzz == 'true' + permissions: + security-events: write + strategy: + fail-fast: false + matrix: + sanitizer: [address, undefined, memory] + steps: + - name: Build fuzzers (${{ matrix.sanitizer }}) + id: build + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master + with: + oss-fuzz-project-name: cpython3 + sanitizer: ${{ matrix.sanitizer }} + - name: Run fuzzers (${{ matrix.sanitizer }}) + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master + with: + fuzz-seconds: 600 + oss-fuzz-project-name: cpython3 + output-sarif: true + sanitizer: ${{ matrix.sanitizer }} + - name: Upload crash + if: failure() && steps.build.outcome == 'success' + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.sanitizer }}-artifacts + path: ./out/artifacts + - name: Upload SARIF + if: always() && steps.build.outcome == 'success' + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: cifuzz-sarif/results.sarif + checkout_path: cifuzz-sarif all-required-green: # This job does nothing and is only used for the branch protection name: All required checks pass @@ -348,12 +698,9 @@ jobs: - build-context # Transitive dependency, needed to access `run-tests` value - check-autoconf-regen - check-generated-files - - build-windows - - build-macos - - build-ubuntu - - build-ubuntu-ssltests + - build-ubuntu-ssltests-awslc + - build-ubuntu-ssltests-openssl - build-wasi - - cross-build-linux if: always() steps: @@ -362,7 +709,8 @@ jobs: with: allowed-failures: >- build-windows-msi, - build-ubuntu-ssltests, + build-ubuntu-ssltests-awslc, + build-ubuntu-ssltests-openssl, test-hypothesis, cifuzz, allowed-skips: >- @@ -380,7 +728,8 @@ jobs: check-generated-files, build-macos, build-ubuntu, - build-ubuntu-ssltests, + build-ubuntu-ssltests-awslc, + build-ubuntu-ssltests-openssl, build-wasi, test-hypothesis, build-asan, From a60b5cae287ae1a77b2fffe47157c8a465a51969 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 14 Jul 2025 13:21:54 +0200 Subject: [PATCH 06/13] simplify CI --- .github/workflows/build.yml | 542 +----------------------------------- 1 file changed, 1 insertion(+), 541 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e09ee877c821c0..f52f8828505b04 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,12 +43,6 @@ jobs: # uses: ./.github/workflows/reusable-context.yml - check-docs: - name: Docs - needs: build-context - if: fromJSON(needs.build-context.outputs.run-docs) - uses: ./.github/workflows/reusable-docs.yml - check-autoconf-regen: name: 'Check if Autoconf files are up to date' # Don't use ubuntu-latest but a specific version to make the job @@ -177,226 +171,6 @@ jobs: arch: ${{ matrix.arch }} free-threading: ${{ matrix.free-threading }} - build-windows-msi: - name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category - Windows MSI${{ '' }} - needs: build-context - if: fromJSON(needs.build-context.outputs.run-windows-msi) - strategy: - fail-fast: false - matrix: - arch: - - x86 - - x64 - - arm64 - uses: ./.github/workflows/reusable-windows-msi.yml - with: - arch: ${{ matrix.arch }} - - build-macos: - name: >- - macOS - ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - # Cirrus and macos-14 are M1, macos-13 is default GHA Intel. - # macOS 13 only runs tests against the GIL-enabled CPython. - # Cirrus used for upstream, macos-14 for forks. - os: - - ghcr.io/cirruslabs/macos-runner:sonoma - - macos-14 - - macos-13 - is-fork: # only used for the exclusion trick - - ${{ github.repository_owner != 'python' }} - free-threading: - - false - - true - exclude: - - os: ghcr.io/cirruslabs/macos-runner:sonoma - is-fork: true - - os: macos-14 - is-fork: false - - os: macos-13 - free-threading: true - uses: ./.github/workflows/reusable-macos.yml - with: - config_hash: ${{ needs.build-context.outputs.config-hash }} - free-threading: ${{ matrix.free-threading }} - os: ${{ matrix.os }} - - build-ubuntu: - name: >- - Ubuntu - ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} - ${{ fromJSON(matrix.bolt) && '(bolt)' || '' }} - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - bolt: - - false - - true - free-threading: - - false - - true - os: - - ubuntu-24.04 - - ubuntu-24.04-arm - exclude: - # Do not test BOLT with free-threading, to conserve resources - - bolt: true - free-threading: true - # BOLT currently crashes during instrumentation on aarch64 - - os: ubuntu-24.04-arm - bolt: true - uses: ./.github/workflows/reusable-ubuntu.yml - with: - config_hash: ${{ needs.build-context.outputs.config-hash }} - bolt-optimizations: ${{ matrix.bolt }} - free-threading: ${{ matrix.free-threading }} - os: ${{ matrix.os }} - - build-ubuntu-ssltests-openssl: - name: 'Ubuntu SSL tests with OpenSSL' - runs-on: ${{ matrix.os }} - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - os: [ubuntu-24.04] - openssl_ver: [3.0.16, 3.1.8, 3.2.4, 3.3.3, 3.4.1] - # See Tools/ssl/make_ssl_data.py for notes on adding a new version - env: - OPENSSL_VER: ${{ matrix.openssl_ver }} - MULTISSL_DIR: ${{ github.workspace }}/multissl - OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }} - LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v4 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - - name: Configure CPython - run: ./configure CFLAGS="-fdiagnostics-format=json" --config-cache --enable-slower-safety --with-pydebug --with-openssl="$OPENSSL_DIR" - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: SSL tests - run: ./python Lib/test/ssltests.py - - build-ubuntu-ssltests-awslc: - name: 'Ubuntu SSL tests with AWS-LC' - runs-on: ${{ matrix.os }} - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - os: [ubuntu-24.04] - awslc_ver: [1.55.0] - env: - AWSLC_VER: ${{ matrix.awslc_ver}} - MULTISSL_DIR: ${{ github.workspace }}/multissl - OPENSSL_DIR: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }} - LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }}/lib - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure SSL lib env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore AWS-LC build' - id: cache-aws-lc - uses: actions/cache@v4 - with: - path: ./multissl/aws-lc/${{ matrix.awslc_ver }} - key: ${{ matrix.os }}-multissl-aws-lc-${{ matrix.awslc_ver }} - - name: Install AWS-LC - if: steps.cache-aws-lc.outputs.cache-hit != 'true' - run: | - python3 Tools/ssl/multissltests.py \ - --steps=library \ - --base-directory "$MULTISSL_DIR" \ - --awslc ${{ matrix.awslc_ver }} \ - --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - - name: Configure CPython - run: | - ./configure CFLAGS="-fdiagnostics-format=json" \ - --config-cache \ - --enable-slower-safety \ - --with-pydebug \ - --with-openssl="$OPENSSL_DIR" \ - --with-builtin-hashlib-hashes=blake2 \ - --with-ssl-default-suites=openssl - - name: Build CPython - run: make -j - - name: Display build info - run: make pythoninfo - - name: Verify python is linked to AWS-LC - run: ./python -c 'import ssl; print(ssl.OPENSSL_VERSION)' | grep AWS-LC - - name: SSL tests - run: ./python Lib/test/ssltests.py - build-wasi: name: 'WASI' needs: build-context @@ -405,291 +179,6 @@ jobs: with: config_hash: ${{ needs.build-context.outputs.config-hash }} - test-hypothesis: - name: "Hypothesis tests on Ubuntu" - runs-on: ubuntu-24.04 - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - env: - OPENSSL_VER: 3.0.16 - PYTHONSTRICTEXTENSIONBUILD: 1 - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v4 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - - name: Setup directory envs for out-of-tree builds - run: | - echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" - echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" - - name: Create directories for read-only out-of-tree builds - run: mkdir -p "$CPYTHON_RO_SRCDIR" "$CPYTHON_BUILDDIR" - - name: Bind mount sources read-only - run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - - name: Configure CPython out-of-tree - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: | - ../cpython-ro-srcdir/configure \ - --config-cache \ - --with-pydebug \ - --enable-slower-safety \ - --with-openssl="$OPENSSL_DIR" - - name: Build CPython out-of-tree - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make -j4 - - name: Display build info - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make pythoninfo - - name: Remount sources writable for tests - # some tests write to srcdir, lack of pyc files slows down testing - run: sudo mount "$CPYTHON_RO_SRCDIR" -oremount,rw - - name: Setup directory envs for out-of-tree builds - run: | - echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" - - name: "Create hypothesis venv" - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: | - VENV_LOC=$(realpath -m .)/hypovenv - VENV_PYTHON=$VENV_LOC/bin/python - echo "HYPOVENV=${VENV_LOC}" >> "$GITHUB_ENV" - echo "VENV_PYTHON=${VENV_PYTHON}" >> "$GITHUB_ENV" - ./python -m venv "$VENV_LOC" && "$VENV_PYTHON" -m pip install -r "${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt" - - name: 'Restore Hypothesis database' - id: cache-hypothesis-database - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/ - key: hypothesis-database-${{ github.head_ref || github.run_id }} - restore-keys: | - hypothesis-database- - - name: "Run tests" - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: | - # Most of the excluded tests are slow test suites with no property tests - # - # (GH-104097) test_sysconfig is skipped because it has tests that are - # failing when executed from inside a virtual environment. - "${VENV_PYTHON}" -m test \ - -W \ - --slowest \ - -j4 \ - --timeout 900 \ - -x test_asyncio \ - -x test_multiprocessing_fork \ - -x test_multiprocessing_forkserver \ - -x test_multiprocessing_spawn \ - -x test_concurrent_futures \ - -x test_socket \ - -x test_subprocess \ - -x test_signal \ - -x test_sysconfig - - uses: actions/upload-artifact@v4 - if: always() - with: - name: hypothesis-example-db - path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/examples/ - - build-asan: - name: 'Address sanitizer' - runs-on: ${{ matrix.os }} - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - os: [ubuntu-24.04] - env: - OPENSSL_VER: 3.0.16 - PYTHONSTRICTEXTENSIONBUILD: 1 - ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Set up GCC-10 for ASAN - uses: egor-tensin/setup-gcc@v1 - with: - version: 10 - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v4 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - - name: Configure CPython - run: ./configure --config-cache --with-address-sanitizer --without-pymalloc - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: Tests - run: xvfb-run make ci - - build-tsan: - name: >- - Thread sanitizer - ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - free-threading: - - false - - true - uses: ./.github/workflows/reusable-tsan.yml - with: - config_hash: ${{ needs.build-context.outputs.config-hash }} - free-threading: ${{ matrix.free-threading }} - - build-ubsan: - name: Undefined behavior sanitizer - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - uses: ./.github/workflows/reusable-ubsan.yml - with: - config_hash: ${{ needs.build-context.outputs.config-hash }} - - cross-build-linux: - name: Cross build Linux - runs-on: ubuntu-latest - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Set build dir - run: - # an absolute path outside of the working directoy - echo "BUILD_DIR=$(realpath ${{ github.workspace }}/../build)" >> "$GITHUB_ENV" - - name: Install dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure host build - run: ./configure --prefix="$BUILD_DIR/host-python" - - name: Install host Python - run: make -j8 install - - name: Run test subset with host build - run: | - "$BUILD_DIR/host-python/bin/python3" -m test test_sysconfig test_site test_embed - - name: Configure cross build - run: ./configure --prefix="$BUILD_DIR/cross-python" --with-build-python="$BUILD_DIR/host-python/bin/python3" - - name: Install cross Python - run: make -j8 install - - name: Run test subset with host build - run: | - "$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed - - # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ - cifuzz: - name: CIFuzz - runs-on: ubuntu-latest - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-ci-fuzz == 'true' - permissions: - security-events: write - strategy: - fail-fast: false - matrix: - sanitizer: [address, undefined, memory] - steps: - - name: Build fuzzers (${{ matrix.sanitizer }}) - id: build - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master - with: - oss-fuzz-project-name: cpython3 - sanitizer: ${{ matrix.sanitizer }} - - name: Run fuzzers (${{ matrix.sanitizer }}) - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master - with: - fuzz-seconds: 600 - oss-fuzz-project-name: cpython3 - output-sarif: true - sanitizer: ${{ matrix.sanitizer }} - - name: Upload crash - if: failure() && steps.build.outcome == 'success' - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.sanitizer }}-artifacts - path: ./out/artifacts - - name: Upload SARIF - if: always() && steps.build.outcome == 'success' - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: cifuzz-sarif/results.sarif - checkout_path: cifuzz-sarif - all-required-green: # This job does nothing and is only used for the branch protection name: All required checks pass runs-on: ubuntu-latest @@ -698,8 +187,7 @@ jobs: - build-context # Transitive dependency, needed to access `run-tests` value - check-autoconf-regen - check-generated-files - - build-ubuntu-ssltests-awslc - - build-ubuntu-ssltests-openssl + - build-windows - build-wasi if: always() @@ -707,34 +195,13 @@ jobs: - name: Check whether the needed jobs succeeded or failed uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe with: - allowed-failures: >- - build-windows-msi, - build-ubuntu-ssltests-awslc, - build-ubuntu-ssltests-openssl, - test-hypothesis, - cifuzz, allowed-skips: >- - ${{ - !fromJSON(needs.build-context.outputs.run-docs) - && ' - check-docs, - ' - || '' - }} ${{ needs.build-context.outputs.run-tests != 'true' && ' check-autoconf-regen, check-generated-files, - build-macos, - build-ubuntu, - build-ubuntu-ssltests-awslc, - build-ubuntu-ssltests-openssl, build-wasi, - test-hypothesis, - build-asan, - build-tsan, - cross-build-linux, ' || '' }} @@ -745,11 +212,4 @@ jobs: ' || '' }} - ${{ - !fromJSON(needs.build-context.outputs.run-ci-fuzz) - && ' - cifuzz, - ' - || '' - }} jobs: ${{ toJSON(needs) }} From 0845dcf878b8815b399fa2abb08343e32b926cc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 14 Jul 2025 13:25:00 +0200 Subject: [PATCH 07/13] less runs --- .github/workflows/build.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f52f8828505b04..891b83403f85f1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -158,11 +158,8 @@ jobs: matrix: arch: - x64 - - Win32 - - arm64 free-threading: - false - - true exclude: # Skip Win32 on free-threaded builds - { arch: Win32, free-threading: true } From 7a45a9be467159efff5aba67574305be478881cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 22 Jul 2025 11:50:05 +0200 Subject: [PATCH 08/13] post-merge --- Modules/_hashlib/hashlib_fetch.h | 73 ++++++++++++++++++-------------- Modules/_hashopenssl.c | 6 ++- Modules/hmacmodule.c | 3 +- 3 files changed, 47 insertions(+), 35 deletions(-) diff --git a/Modules/_hashlib/hashlib_fetch.h b/Modules/_hashlib/hashlib_fetch.h index ee647ea6220f4d..833016d1267fdc 100644 --- a/Modules/_hashlib/hashlib_fetch.h +++ b/Modules/_hashlib/hashlib_fetch.h @@ -50,56 +50,65 @@ #include "Python.h" -#define Py_HASHLIB_MD_NS(ATTR) Py_hashlib_message_digest_ ## ATTR -#define Py_HASHLIB_MD_FAMILY(FAMILY_ID) Py_HASHLIB_MD_NS(family_ ## FAMILY_ID) -#define Py_HASHLIB_MD_MEMBER(MEMBER_ID) Py_HASHLIB_MD_NS(member_ ## MEMBER_ID) +/* + * Internal error messages used for reporting an unsupported hash algorithm. + * The algorithm can be given by its name, a callable or a PEP-247 module. + * The same message is raised by Lib/hashlib.py::__get_builtin_constructor() + * and _hmacmodule.c::find_hash_info(). + */ +#define _Py_HASHLIB_UNSUPPORTED_ALGORITHM "unsupported hash algorithm %S" +#define _Py_HASHLIB_UNSUPPORTED_STR_ALGORITHM "unsupported hash algorithm %s" + +#define _Py_HASHLIB_MD_NS(ATTR) _Py_hashlib_message_digest_ ## ATTR +#define _Py_HASHLIB_MD_FAMILY(FAMILY) _Py_HASHLIB_MD_NS(family_ ## FAMILY) +#define _Py_HASHLIB_MD_MEMBER(MEMBER) _Py_HASHLIB_MD_NS(member_ ## MEMBER) -#define Py_HASHLIB_MD_NAMES Py_HASHLIB_MD_NS(NAMES) -#define Py_HASHLIB_MD_COUNT Py_ARRAY_LENGTH(Py_HASHLIB_MD_NAMES) -#define Py_HASHLIB_MD_NAME(MEMBER_ID) \ +#define _Py_HASHLIB_MD_NAMES _Py_HASHLIB_MD_NS(NAMES) +#define _Py_HASHLIB_MD_COUNT Py_ARRAY_LENGTH(Py_HASHLIB_MD_NAMES) +#define _Py_HASHLIB_MD_NAME(MEMBER_ID) \ ( \ assert(Py_HASHLIB_MD_NAME(MEMBER_ID) < Py_HASHLIB_MD_COUNT), \ Py_HASHLIB_MD_NAMES[Py_HASHLIB_MD_MEMBER(MEMBER_ID)] \ ) typedef enum { - Py_HASHLIB_MD_FAMILY(MD) = 0, - Py_HASHLIB_MD_FAMILY(SHA1), - Py_HASHLIB_MD_FAMILY(SHA2), - Py_HASHLIB_MD_FAMILY(SHA2t), - Py_HASHLIB_MD_FAMILY(SHA3), - Py_HASHLIB_MD_FAMILY(SHA3_XOF), - Py_HASHLIB_MD_FAMILY(BLAKE2), -} Py_HASHLIB_MD_NS(family); + _Py_HASHLIB_MD_FAMILY(MD) = 0, + _Py_HASHLIB_MD_FAMILY(SHA1), + _Py_HASHLIB_MD_FAMILY(SHA2), + _Py_HASHLIB_MD_FAMILY(SHA2t), + _Py_HASHLIB_MD_FAMILY(SHA3), + _Py_HASHLIB_MD_FAMILY(SHA3_XOF), + _Py_HASHLIB_MD_FAMILY(BLAKE2), +} _Py_HASHLIB_MD_NS(family); typedef enum { /* MD-family */ - Py_HASHLIB_MD_MEMBER(md5) = 0, + _Py_HASHLIB_MD_MEMBER(md5) = 0, /* SHA-1 family */ - Py_HASHLIB_MD_MEMBER(sha1), + _Py_HASHLIB_MD_MEMBER(sha1), /* SHA-2 family */ - Py_HASHLIB_MD_MEMBER(sha224), - Py_HASHLIB_MD_MEMBER(sha256), - Py_HASHLIB_MD_MEMBER(sha384), - Py_HASHLIB_MD_MEMBER(sha512), + _Py_HASHLIB_MD_MEMBER(sha224), + _Py_HASHLIB_MD_MEMBER(sha256), + _Py_HASHLIB_MD_MEMBER(sha384), + _Py_HASHLIB_MD_MEMBER(sha512), /* Truncated SHA-2 family */ - Py_HASHLIB_MD_MEMBER(sha512_224), - Py_HASHLIB_MD_MEMBER(sha512_256), + _Py_HASHLIB_MD_MEMBER(sha512_224), + _Py_HASHLIB_MD_MEMBER(sha512_256), /* SHA-3 family */ - Py_HASHLIB_MD_MEMBER(sha3_224), - Py_HASHLIB_MD_MEMBER(sha3_256), - Py_HASHLIB_MD_MEMBER(sha3_384), - Py_HASHLIB_MD_MEMBER(sha3_512), + _Py_HASHLIB_MD_MEMBER(sha3_224), + _Py_HASHLIB_MD_MEMBER(sha3_256), + _Py_HASHLIB_MD_MEMBER(sha3_384), + _Py_HASHLIB_MD_MEMBER(sha3_512), /* SHA-3 XOF SHAKE family */ - Py_HASHLIB_MD_MEMBER(shake_128), - Py_HASHLIB_MD_MEMBER(shake_256), + _Py_HASHLIB_MD_MEMBER(shake_128), + _Py_HASHLIB_MD_MEMBER(shake_256), /* BLAKE-2 family */ - Py_HASHLIB_MD_MEMBER(blake2b), - Py_HASHLIB_MD_MEMBER(blake2s), -} Py_HASHLIB_MD_NS(member); + _Py_HASHLIB_MD_MEMBER(blake2b), + _Py_HASHLIB_MD_MEMBER(blake2s), +} _Py_HASHLIB_MD_NS(member); static const char *Py_HASHLIB_MD_NAMES[] = { -#define DECL_MESSAGE_DIGEST_NAME(ID) [Py_HASHLIB_MD_MEMBER(ID)] = #ID +#define DECL_MESSAGE_DIGEST_NAME(ID) [_Py_HASHLIB_MD_MEMBER(ID)] = #ID /* MD-family */ DECL_MESSAGE_DIGEST_NAME(md5), /* SHA-1 family */ diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 0d98063dfc2108..c42513a53d4561 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -26,7 +26,9 @@ #include "pycore_hashtable.h" #include "pycore_strhex.h" // _Py_strhex() #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_RELAXED + #include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_fetch.h" #include "_hashlib/hashlib_mutex.h" /* EVP is the preferred interface to hashing in OpenSSL */ @@ -533,7 +535,7 @@ raise_unsupported_algorithm_error(_hashlibstate *state, PyObject *digestmod) { raise_unsupported_algorithm_impl( state->unsupported_digestmod_error, - HASHLIB_UNSUPPORTED_ALGORITHM, + _Py_HASHLIB_UNSUPPORTED_ALGORITHM, digestmod ); } @@ -543,7 +545,7 @@ raise_unsupported_str_algorithm_error(_hashlibstate *state, const char *name) { raise_unsupported_algorithm_impl( state->unsupported_digestmod_error, - HASHLIB_UNSUPPORTED_STR_ALGORITHM, + _Py_HASHLIB_UNSUPPORTED_STR_ALGORITHM, name ); } diff --git a/Modules/hmacmodule.c b/Modules/hmacmodule.c index a058d33d0ca812..4f132f38eb6f2f 100644 --- a/Modules/hmacmodule.c +++ b/Modules/hmacmodule.c @@ -46,6 +46,7 @@ #include "_hacl/Hacl_Streaming_Types.h" // Hacl_Streaming_Types_error_code #include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_fetch.h" #include "_hashlib/hashlib_mutex.h" #include @@ -657,7 +658,7 @@ find_hash_info(hmacmodule_state *state, PyObject *hash_info_ref) } if (rc == 0) { PyErr_Format(state->unknown_hash_error, - HASHLIB_UNSUPPORTED_ALGORITHM, hash_info_ref); + _Py_HASHLIB_UNSUPPORTED_ALGORITHM, hash_info_ref); return NULL; } assert(info != NULL); From cbaaa40a64f33e627cd350d8cc3ec77c1732ae9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 22 Jul 2025 11:52:07 +0200 Subject: [PATCH 09/13] post-merge --- Modules/_hashlib/hashlib_fetch.h | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Modules/_hashlib/hashlib_fetch.h b/Modules/_hashlib/hashlib_fetch.h index 833016d1267fdc..2cfabc6b25aac1 100644 --- a/Modules/_hashlib/hashlib_fetch.h +++ b/Modules/_hashlib/hashlib_fetch.h @@ -59,16 +59,16 @@ #define _Py_HASHLIB_UNSUPPORTED_ALGORITHM "unsupported hash algorithm %S" #define _Py_HASHLIB_UNSUPPORTED_STR_ALGORITHM "unsupported hash algorithm %s" -#define _Py_HASHLIB_MD_NS(ATTR) _Py_hashlib_message_digest_ ## ATTR -#define _Py_HASHLIB_MD_FAMILY(FAMILY) _Py_HASHLIB_MD_NS(family_ ## FAMILY) -#define _Py_HASHLIB_MD_MEMBER(MEMBER) _Py_HASHLIB_MD_NS(member_ ## MEMBER) +#define _Py_HASHLIB_MD_NAMESPACE(NAME) _Py_hashlib_message_digest_ ## NAME +#define _Py_HASHLIB_MD_FAMILY(ID) _Py_HASHLIB_MD_NAMESPACE(family_ ## ID) +#define _Py_HASHLIB_MD_MEMBER(ID) _Py_HASHLIB_MD_NAMESPACE(member_ ## ID) -#define _Py_HASHLIB_MD_NAMES _Py_HASHLIB_MD_NS(NAMES) -#define _Py_HASHLIB_MD_COUNT Py_ARRAY_LENGTH(Py_HASHLIB_MD_NAMES) -#define _Py_HASHLIB_MD_NAME(MEMBER_ID) \ +#define _Py_HASHLIB_MD_NAMES _Py_HASHLIB_MD_NAMESPACE(NAMES) +#define _Py_HASHLIB_MD_COUNT Py_ARRAY_LENGTH(_Py_HASHLIB_MD_NAMES) +#define _Py_HASHLIB_MD_NAME(MEMBER_ID) \ ( \ - assert(Py_HASHLIB_MD_NAME(MEMBER_ID) < Py_HASHLIB_MD_COUNT), \ - Py_HASHLIB_MD_NAMES[Py_HASHLIB_MD_MEMBER(MEMBER_ID)] \ + assert(_Py_HASHLIB_MD_NAME(MEMBER_ID) < _Py_HASHLIB_MD_COUNT), \ + _Py_HASHLIB_MD_NAMES[_Py_HASHLIB_MD_MEMBER(MEMBER_ID)] \ ) typedef enum { @@ -79,7 +79,7 @@ typedef enum { _Py_HASHLIB_MD_FAMILY(SHA3), _Py_HASHLIB_MD_FAMILY(SHA3_XOF), _Py_HASHLIB_MD_FAMILY(BLAKE2), -} _Py_HASHLIB_MD_NS(family); +} _Py_HASHLIB_MD_NAMESPACE(family); typedef enum { /* MD-family */ @@ -105,7 +105,7 @@ typedef enum { /* BLAKE-2 family */ _Py_HASHLIB_MD_MEMBER(blake2b), _Py_HASHLIB_MD_MEMBER(blake2s), -} _Py_HASHLIB_MD_NS(member); +} _Py_HASHLIB_MD_NAMESPACE(member); static const char *Py_HASHLIB_MD_NAMES[] = { #define DECL_MESSAGE_DIGEST_NAME(ID) [_Py_HASHLIB_MD_MEMBER(ID)] = #ID From c653c2908d664fc2124c43582eddbb3a2fab1726 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 22 Jul 2025 13:02:33 +0200 Subject: [PATCH 10/13] fix --- Makefile.pre.in | 35 +++++++++++++++++++++--------- Modules/_hashlib/hashlib_fetch.h | 11 +++++----- Tools/wasm/wasi/__main__.py | 2 +- configure | 37 ++++++++++++++++++-------------- configure.ac | 15 +++++++------ 5 files changed, 62 insertions(+), 38 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 654d6becc984e7..28501ff1676e73 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -227,6 +227,7 @@ ENSUREPIP= @ENSUREPIP@ # Internal static libraries LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a LIBEXPAT_A= Modules/expat/libexpat.a +LIBHASHLIB_INTERNAL_A=Modules/_hashlib/libhashlib.a # HACL* build configuration LIBHACL_CFLAGS=@LIBHACL_CFLAGS@ @@ -761,6 +762,18 @@ LIBHACL_HMAC_HEADERS= \ $(LIBHACL_BLAKE2_HEADERS) \ $(LIBHACL_HEADERS) +########################################################################## +# Internal library for cryptographic primitives + +LIBHASHLIB_INTERNAL_OBJS= \ + Modules/_hashlib/hashlib_buffer.o \ + Modules/_hashlib/hashlib_fetch.o + +LIBHASHLIB_INTERNAL_HEADERS= \ + Modules/_hashlib/hashlib_buffer.h \ + Modules/_hashlib/hashlib_fetch.h \ + Modules/_hashlib/hashlib_mutex.h + ######################################################################### # Rules @@ -1483,9 +1496,17 @@ $(LIBEXPAT_A): $(LIBEXPAT_OBJS) ########################################################################## # '_hashlib', '_hmac' and HACL*-based modules helpers +LIBHASHLIB_INTERNAL_CFLAGS=@LIBHASHLIB_INTERNAL_CFLAGS@ $(PY_STDMODULE_CFLAGS) $(CCSHARED) -Modules/_hashlib/hashlib_buffer.o: $(srcdir)/Modules/_hashlib/hashlib_buffer.c $(srcdir)/Modules/_hashlib/hashlib_buffer.h $(PYTHON_HEADERS) - $(CC) -I$(srcdir)/Modules/_hashlib -c $(PY_STDMODULE_CFLAGS) $(CCSHARED) -o $@ $(srcdir)/Modules/_hashlib/hashlib_buffer.c +Modules/_hashlib/hashlib_buffer.o: Modules/_hashlib/hashlib_buffer.c $(LIBHASHLIB_INTERNAL_HEADERS) $(PYTHON_HEADERS) + $(CC) -I$(srcdir)/Modules/_hashlib -c $(LIBHASHLIB_INTERNAL_CFLAGS) -o $@ $(srcdir)/Modules/_hashlib/hashlib_buffer.c + +Modules/_hashlib/hashlib_fetch.o: Modules/_hashlib/hashlib_fetch.c $(LIBHASHLIB_INTERNAL_HEADERS) $(PYTHON_HEADERS) + $(CC) -I$(srcdir)/Modules/_hashlib -c $(LIBHASHLIB_INTERNAL_CFLAGS) -o $@ $(srcdir)/Modules/_hashlib/hashlib_fetch.c + +$(LIBHASHLIB_INTERNAL_A): $(LIBHASHLIB_INTERNAL_OBJS) + -rm -f $@ + $(AR) $(ARFLAGS) $@ $(LIBHASHLIB_INTERNAL_OBJS) ########################################################################## # HACL* library build @@ -3331,14 +3352,8 @@ MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@ MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@ MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h -MODULE__HASHLIB_DEPS= \ - $(srcdir)/Modules/_hashlib/hashlib_buffer.h \ - $(srcdir)/Modules/_hashlib/hashlib_fetch.h \ - $(srcdir)/Modules/_hashlib/hashlib_mutex.h - -MODULE__HASHLIB_LDEPS= \ - Modules/_hashlib/hashlib_buffer.o \ - Modules/_hashlib/hashlib_fetch.o +MODULE__HASHLIB_DEPS=@LIBHASHLIB_INTERNAL@ +MODULE__HASHLIB_LDEPS= # HACL*-based cryptographic primitives MODULE__MD5_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) diff --git a/Modules/_hashlib/hashlib_fetch.h b/Modules/_hashlib/hashlib_fetch.h index 2cfabc6b25aac1..a583b737923e21 100644 --- a/Modules/_hashlib/hashlib_fetch.h +++ b/Modules/_hashlib/hashlib_fetch.h @@ -65,10 +65,11 @@ #define _Py_HASHLIB_MD_NAMES _Py_HASHLIB_MD_NAMESPACE(NAMES) #define _Py_HASHLIB_MD_COUNT Py_ARRAY_LENGTH(_Py_HASHLIB_MD_NAMES) -#define _Py_HASHLIB_MD_NAME(MEMBER_ID) \ - ( \ - assert(_Py_HASHLIB_MD_NAME(MEMBER_ID) < _Py_HASHLIB_MD_COUNT), \ - _Py_HASHLIB_MD_NAMES[_Py_HASHLIB_MD_MEMBER(MEMBER_ID)] \ +#define _Py_HASHLIB_MD_NAME(MEMBER_ID) \ + ( \ + assert(_Py_HASHLIB_MD_MEMBER(MEMBER_ID) >= 0), \ + assert(_Py_HASHLIB_MD_MEMBER(MEMBER_ID) < _Py_HASHLIB_MD_COUNT), \ + _Py_HASHLIB_MD_NAMES[_Py_HASHLIB_MD_MEMBER(MEMBER_ID)] \ ) typedef enum { @@ -107,7 +108,7 @@ typedef enum { _Py_HASHLIB_MD_MEMBER(blake2s), } _Py_HASHLIB_MD_NAMESPACE(member); -static const char *Py_HASHLIB_MD_NAMES[] = { +static const char *_Py_HASHLIB_MD_NAMES[] = { #define DECL_MESSAGE_DIGEST_NAME(ID) [_Py_HASHLIB_MD_MEMBER(ID)] = #ID /* MD-family */ DECL_MESSAGE_DIGEST_NAME(md5), diff --git a/Tools/wasm/wasi/__main__.py b/Tools/wasm/wasi/__main__.py index 54ccc95157d57d..fed874013e87cd 100644 --- a/Tools/wasm/wasi/__main__.py +++ b/Tools/wasm/wasi/__main__.py @@ -178,7 +178,7 @@ def find_wasi_sdk(): def wasi_sdk_env(context): """Calculate environment variables for building with wasi-sdk.""" - wasi_sdk_path = context.wasi_sdk_path + wasi_sdk_path = context.wasi_sdk_path.absolute() sysroot = wasi_sdk_path / "share" / "wasi-sysroot" env = {"CC": "clang", "CPP": "clang-cpp", "CXX": "clang++", "AR": "llvm-ar", "RANLIB": "ranlib"} diff --git a/configure b/configure index f76aaead2b57fc..cb5f9e4ec5c0fd 100755 --- a/configure +++ b/configure @@ -725,6 +725,8 @@ LIBHACL_BLAKE2_SIMD128_OBJS LIBHACL_SIMD128_FLAGS LIBHACL_LDFLAGS LIBHACL_CFLAGS +LIBHASHLIB_INTERNAL +LIBHASHLIB_INTERNAL_CFLAGS MODULE_UNICODEDATA_FALSE MODULE_UNICODEDATA_TRUE MODULE__MULTIBYTECODEC_FALSE @@ -32528,8 +32530,11 @@ fi ############################################################################### # Cryptographic primitives -LIBHASHLIB_INCL='-I$(srcdir)/Modules/_hashlib' -LIBHASHLIB_LIBS='Modules/_hashlib/hashlib_buffer.o Modules/_hashlib/hashlib_fetch.o' +LIBHASHLIB_INTERNAL_CFLAGS="-I\$(srcdir)/Modules/_hashlib" +LIBHASHLIB_INTERNAL_LDFLAGS="-lm \$(LIBHASHLIB_INTERNAL_A)" +LIBHASHLIB_INTERNAL="\$(LIBHASHLIB_INTERNAL_HEADERS) \$(LIBHASHLIB_INTERNAL_A)" + + ############################################################################### @@ -32780,8 +32785,8 @@ fi if test "x$py_cv_module__md5" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__MD5_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INCL$as_nl" - as_fn_append MODULE_BLOCK "MODULE__MD5_LDFLAGS=\$($LIBHACL_MD5_LDFLAGS) $LIBHASHLIB_LIBS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__MD5_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__MD5_LDFLAGS=\$($LIBHACL_MD5_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__md5" = yes; then @@ -32825,8 +32830,8 @@ fi if test "x$py_cv_module__sha1" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA1_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INCL$as_nl" - as_fn_append MODULE_BLOCK "MODULE__SHA1_LDFLAGS=\$($LIBHACL_SHA1_LDFLAGS) $LIBHASHLIB_LIBS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA1_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA1_LDFLAGS=\$($LIBHACL_SHA1_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__sha1" = yes; then @@ -32870,8 +32875,8 @@ fi if test "x$py_cv_module__sha2" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA2_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INCL$as_nl" - as_fn_append MODULE_BLOCK "MODULE__SHA2_LDFLAGS=\$($LIBHACL_SHA2_LDFLAGS) $LIBHASHLIB_LIBS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA2_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA2_LDFLAGS=\$($LIBHACL_SHA2_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__sha2" = yes; then @@ -32915,8 +32920,8 @@ fi if test "x$py_cv_module__sha3" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA3_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INCL$as_nl" - as_fn_append MODULE_BLOCK "MODULE__SHA3_LDFLAGS=\$($LIBHACL_SHA3_LDFLAGS) $LIBHASHLIB_LIBS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA3_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA3_LDFLAGS=\$($LIBHACL_SHA3_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__sha3" = yes; then @@ -32960,8 +32965,8 @@ fi if test "x$py_cv_module__blake2" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__BLAKE2_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INCL$as_nl" - as_fn_append MODULE_BLOCK "MODULE__BLAKE2_LDFLAGS=\$($LIBHACL_BLAKE2_LDFLAGS) $LIBHASHLIB_LIBS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__BLAKE2_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__BLAKE2_LDFLAGS=\$($LIBHACL_BLAKE2_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__blake2" = yes; then @@ -33006,8 +33011,8 @@ fi if test "x$py_cv_module__hmac" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__HMAC_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INCL$as_nl" - as_fn_append MODULE_BLOCK "MODULE__HMAC_LDFLAGS=\$($LIBHACL_HMAC_LDFLAGS) $LIBHASHLIB_LIBS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HMAC_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HMAC_LDFLAGS=\$($LIBHACL_HMAC_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__hmac" = yes; then @@ -33688,8 +33693,8 @@ fi if test "x$py_cv_module__hashlib" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__HASHLIB_CFLAGS=$OPENSSL_INCLUDES $LIBHASHLIB_INCL$as_nl" - as_fn_append MODULE_BLOCK "MODULE__HASHLIB_LDFLAGS=$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS $LIBHASHLIB_LIBS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HASHLIB_CFLAGS=$OPENSSL_INCLUDES $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HASHLIB_LDFLAGS=$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__hashlib" = yes; then diff --git a/configure.ac b/configure.ac index a767d4f62bea7e..3ca67bcf188f3b 100644 --- a/configure.ac +++ b/configure.ac @@ -7961,9 +7961,12 @@ PY_STDLIB_MOD_SIMPLE([unicodedata]) ############################################################################### # Cryptographic primitives -LIBHASHLIB_INCL='-I$(srcdir)/Modules/_hashlib' -LIBHASHLIB_LIBS='Modules/_hashlib/hashlib_buffer.o Modules/_hashlib/hashlib_fetch.o' +LIBHASHLIB_INTERNAL_CFLAGS="-I\$(srcdir)/Modules/_hashlib" +LIBHASHLIB_INTERNAL_LDFLAGS="-lm \$(LIBHASHLIB_INTERNAL_A)" +LIBHASHLIB_INTERNAL="\$(LIBHASHLIB_INTERNAL_HEADERS) \$(LIBHASHLIB_INTERNAL_A)" +AC_SUBST([LIBHASHLIB_INTERNAL_CFLAGS]) +AC_SUBST([LIBHASHLIB_INTERNAL]) ############################################################################### # HACL* compilation and linking configuration (contact: @picnixz) @@ -8102,8 +8105,8 @@ AC_DEFUN([PY_HACL_CREATE_MODULE], [ AS_VAR_PUSHDEF([v], [[LIBHACL_][$1][_LDFLAGS]]) AS_VAR_SET([v], [[LIBHACL_][$1][_LIB_${LIBHACL_LDEPS_LIBTYPE}]]) PY_STDLIB_MOD([$2], [$3], [], - [$LIBHACL_CFLAGS $LIBHASHLIB_INCL], - [\$($v) $LIBHASHLIB_LIBS]) + [$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS], + [\$($v) $LIBHASHLIB_INTERNAL_LDFLAGS]) AS_VAR_POPDEF([v]) ]) @@ -8184,8 +8187,8 @@ dnl OpenSSL bindings PY_STDLIB_MOD([_ssl], [], [test "$ac_cv_working_openssl_ssl" = yes], [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $OPENSSL_LIBS]) PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes], - [$OPENSSL_INCLUDES $LIBHASHLIB_INCL], - [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS $LIBHASHLIB_LIBS]) + [$OPENSSL_INCLUDES $LIBHASHLIB_INTERNAL_CFLAGS], + [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS $LIBHASHLIB_INTERNAL_LDFLAGS]) dnl test modules PY_STDLIB_MOD([_testcapi], From 826af90e079289ce32fd004d9158e8cd98faf627 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 22 Jul 2025 13:10:55 +0200 Subject: [PATCH 11/13] fixup --- Tools/c-analyzer/cpython/ignored.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 4716b8bf11fcbf..5cd494dfd66722 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -238,7 +238,7 @@ Modules/_decimal/_decimal.c - signal_map_template - Modules/_decimal/_decimal.c - ssize_constants - Modules/_decimal/_decimal.c - INVALID_SIGNALDICT_ERROR_MSG - Modules/_elementtree.c - ExpatMemoryHandler - -Modules/_hashlib/hashlib_fetch.h - Py_hashlib_message_digest_NAMES - +Modules/_hashlib/hashlib_fetch.h - _Py_hashlib_message_digest_NAMES - Modules/_hashopenssl.c - py_hashes - Modules/_hacl/Hacl_Hash_SHA1.c - _h0 - Modules/_hacl/Hacl_Hash_MD5.c - _h0 - From 2efb676feb94ac81f67417059a8f7d31b1296737 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 22 Jul 2025 13:26:40 +0200 Subject: [PATCH 12/13] fix --- Makefile.pre.in | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 28501ff1676e73..7d149a52a646fe 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -3350,25 +3350,22 @@ MODULE__CTYPES_TEST_DEPS=$(srcdir)/Modules/_ctypes/_ctypes_test_generated.c.h MODULE__CTYPES_MALLOC_CLOSURE=@MODULE__CTYPES_MALLOC_CLOSURE@ MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@ MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@ -MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h - MODULE__HASHLIB_DEPS=@LIBHASHLIB_INTERNAL@ -MODULE__HASHLIB_LDEPS= +MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h # HACL*-based cryptographic primitives MODULE__MD5_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__MD5_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__MD5_LDEPS=$(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SHA1_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA1_HEADERS) $(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA1_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA1_LDEPS=$(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SHA2_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA2_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA2_LDEPS=$(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SHA3_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA3_HEADERS) $(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA3_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA3_LDEPS=$(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__BLAKE2_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_BLAKE2_HEADERS) $(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__BLAKE2_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) - +MODULE__BLAKE2_LDEPS=$(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__HMAC_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_HMAC_HEADERS) $(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__HMAC_LDEPS=$(MODULE__HASHLIB_LDEPS) $(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__HMAC_LDEPS=$(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h From 972a185a9df76c868ad648de851d82a63c0a5f45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Tue, 22 Jul 2025 13:39:35 +0200 Subject: [PATCH 13/13] fixup --- PCbuild/pythoncore.vcxproj | 1 + Tools/wasm/wasi/__main__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 7fbb8f9dc7609c..a83ede77fb7f39 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -449,6 +449,7 @@ + diff --git a/Tools/wasm/wasi/__main__.py b/Tools/wasm/wasi/__main__.py index fed874013e87cd..54ccc95157d57d 100644 --- a/Tools/wasm/wasi/__main__.py +++ b/Tools/wasm/wasi/__main__.py @@ -178,7 +178,7 @@ def find_wasi_sdk(): def wasi_sdk_env(context): """Calculate environment variables for building with wasi-sdk.""" - wasi_sdk_path = context.wasi_sdk_path.absolute() + wasi_sdk_path = context.wasi_sdk_path sysroot = wasi_sdk_path / "share" / "wasi-sysroot" env = {"CC": "clang", "CPP": "clang-cpp", "CXX": "clang++", "AR": "llvm-ar", "RANLIB": "ranlib"}