diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f07f5e8040acf0..891b83403f85f1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,12 +43,6 @@ jobs: # uses: ./.github/workflows/reusable-context.yml - check-docs: - name: Docs - needs: build-context - if: fromJSON(needs.build-context.outputs.run-docs) - uses: ./.github/workflows/reusable-docs.yml - check-autoconf-regen: name: 'Check if Autoconf files are up to date' # Don't use ubuntu-latest but a specific version to make the job @@ -164,11 +158,8 @@ jobs: matrix: arch: - x64 - - Win32 - - arm64 free-threading: - false - - true exclude: # Skip Win32 on free-threaded builds - { arch: Win32, free-threading: true } @@ -177,226 +168,6 @@ jobs: arch: ${{ matrix.arch }} free-threading: ${{ matrix.free-threading }} - build-windows-msi: - name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category - Windows MSI${{ '' }} - needs: build-context - if: fromJSON(needs.build-context.outputs.run-windows-msi) - strategy: - fail-fast: false - matrix: - arch: - - x86 - - x64 - - arm64 - uses: ./.github/workflows/reusable-windows-msi.yml - with: - arch: ${{ matrix.arch }} - - build-macos: - name: >- - macOS - ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - # Cirrus and macos-14 are M1, macos-13 is default GHA Intel. - # macOS 13 only runs tests against the GIL-enabled CPython. - # Cirrus used for upstream, macos-14 for forks. - os: - - ghcr.io/cirruslabs/macos-runner:sonoma - - macos-14 - - macos-13 - is-fork: # only used for the exclusion trick - - ${{ github.repository_owner != 'python' }} - free-threading: - - false - - true - exclude: - - os: ghcr.io/cirruslabs/macos-runner:sonoma - is-fork: true - - os: macos-14 - is-fork: false - - os: macos-13 - free-threading: true - uses: ./.github/workflows/reusable-macos.yml - with: - config_hash: ${{ needs.build-context.outputs.config-hash }} - free-threading: ${{ matrix.free-threading }} - os: ${{ matrix.os }} - - build-ubuntu: - name: >- - Ubuntu - ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} - ${{ fromJSON(matrix.bolt) && '(bolt)' || '' }} - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - bolt: - - false - - true - free-threading: - - false - - true - os: - - ubuntu-24.04 - - ubuntu-24.04-arm - exclude: - # Do not test BOLT with free-threading, to conserve resources - - bolt: true - free-threading: true - # BOLT currently crashes during instrumentation on aarch64 - - os: ubuntu-24.04-arm - bolt: true - uses: ./.github/workflows/reusable-ubuntu.yml - with: - config_hash: ${{ needs.build-context.outputs.config-hash }} - bolt-optimizations: ${{ matrix.bolt }} - free-threading: ${{ matrix.free-threading }} - os: ${{ matrix.os }} - - build-ubuntu-ssltests-openssl: - name: 'Ubuntu SSL tests with OpenSSL' - runs-on: ${{ matrix.os }} - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - os: [ubuntu-24.04] - openssl_ver: [3.0.16, 3.1.8, 3.2.4, 3.3.3, 3.4.1] - # See Tools/ssl/make_ssl_data.py for notes on adding a new version - env: - OPENSSL_VER: ${{ matrix.openssl_ver }} - MULTISSL_DIR: ${{ github.workspace }}/multissl - OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }} - LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v4 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - - name: Configure CPython - run: ./configure CFLAGS="-fdiagnostics-format=json" --config-cache --enable-slower-safety --with-pydebug --with-openssl="$OPENSSL_DIR" - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: SSL tests - run: ./python Lib/test/ssltests.py - - build-ubuntu-ssltests-awslc: - name: 'Ubuntu SSL tests with AWS-LC' - runs-on: ${{ matrix.os }} - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - os: [ubuntu-24.04] - awslc_ver: [1.55.0] - env: - AWSLC_VER: ${{ matrix.awslc_ver}} - MULTISSL_DIR: ${{ github.workspace }}/multissl - OPENSSL_DIR: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }} - LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }}/lib - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure SSL lib env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore AWS-LC build' - id: cache-aws-lc - uses: actions/cache@v4 - with: - path: ./multissl/aws-lc/${{ matrix.awslc_ver }} - key: ${{ matrix.os }}-multissl-aws-lc-${{ matrix.awslc_ver }} - - name: Install AWS-LC - if: steps.cache-aws-lc.outputs.cache-hit != 'true' - run: | - python3 Tools/ssl/multissltests.py \ - --steps=library \ - --base-directory "$MULTISSL_DIR" \ - --awslc ${{ matrix.awslc_ver }} \ - --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - - name: Configure CPython - run: | - ./configure CFLAGS="-fdiagnostics-format=json" \ - --config-cache \ - --enable-slower-safety \ - --with-pydebug \ - --with-openssl="$OPENSSL_DIR" \ - --with-builtin-hashlib-hashes=blake2 \ - --with-ssl-default-suites=openssl - - name: Build CPython - run: make -j - - name: Display build info - run: make pythoninfo - - name: Verify python is linked to AWS-LC - run: ./python -c 'import ssl; print(ssl.OPENSSL_VERSION)' | grep AWS-LC - - name: SSL tests - run: ./python Lib/test/ssltests.py - build-wasi: name: 'WASI' needs: build-context @@ -405,346 +176,29 @@ jobs: with: config_hash: ${{ needs.build-context.outputs.config-hash }} - test-hypothesis: - name: "Hypothesis tests on Ubuntu" - runs-on: ubuntu-24.04 - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - env: - OPENSSL_VER: 3.0.16 - PYTHONSTRICTEXTENSIONBUILD: 1 - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v4 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - - name: Setup directory envs for out-of-tree builds - run: | - echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" - echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" - - name: Create directories for read-only out-of-tree builds - run: mkdir -p "$CPYTHON_RO_SRCDIR" "$CPYTHON_BUILDDIR" - - name: Bind mount sources read-only - run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - - name: Configure CPython out-of-tree - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: | - ../cpython-ro-srcdir/configure \ - --config-cache \ - --with-pydebug \ - --enable-slower-safety \ - --with-openssl="$OPENSSL_DIR" - - name: Build CPython out-of-tree - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make -j4 - - name: Display build info - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make pythoninfo - - name: Remount sources writable for tests - # some tests write to srcdir, lack of pyc files slows down testing - run: sudo mount "$CPYTHON_RO_SRCDIR" -oremount,rw - - name: Setup directory envs for out-of-tree builds - run: | - echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" - - name: "Create hypothesis venv" - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: | - VENV_LOC=$(realpath -m .)/hypovenv - VENV_PYTHON=$VENV_LOC/bin/python - echo "HYPOVENV=${VENV_LOC}" >> "$GITHUB_ENV" - echo "VENV_PYTHON=${VENV_PYTHON}" >> "$GITHUB_ENV" - ./python -m venv "$VENV_LOC" && "$VENV_PYTHON" -m pip install -r "${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt" - - name: 'Restore Hypothesis database' - id: cache-hypothesis-database - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/ - key: hypothesis-database-${{ github.head_ref || github.run_id }} - restore-keys: | - hypothesis-database- - - name: "Run tests" - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: | - # Most of the excluded tests are slow test suites with no property tests - # - # (GH-104097) test_sysconfig is skipped because it has tests that are - # failing when executed from inside a virtual environment. - "${VENV_PYTHON}" -m test \ - -W \ - --slowest \ - -j4 \ - --timeout 900 \ - -x test_asyncio \ - -x test_multiprocessing_fork \ - -x test_multiprocessing_forkserver \ - -x test_multiprocessing_spawn \ - -x test_concurrent_futures \ - -x test_socket \ - -x test_subprocess \ - -x test_signal \ - -x test_sysconfig - - uses: actions/upload-artifact@v4 - if: always() - with: - name: hypothesis-example-db - path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/examples/ - - build-asan: - name: 'Address sanitizer' - runs-on: ${{ matrix.os }} - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - os: [ubuntu-24.04] - env: - OPENSSL_VER: 3.0.16 - PYTHONSTRICTEXTENSIONBUILD: 1 - ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Set up GCC-10 for ASAN - uses: egor-tensin/setup-gcc@v1 - with: - version: 10 - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v4 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - - name: Configure CPython - run: ./configure --config-cache --with-address-sanitizer --without-pymalloc - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: Tests - run: xvfb-run make ci - - build-san: - name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category - Sanitizers${{ '' }} - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - strategy: - fail-fast: false - matrix: - check-name: - - Thread - free-threading: - - false - - true - sanitizer: - - TSan - include: - - check-name: Undefined behavior - sanitizer: UBSan - free-threading: false - uses: ./.github/workflows/reusable-san.yml - with: - sanitizer: ${{ matrix.sanitizer }} - config_hash: ${{ needs.build-context.outputs.config-hash }} - free-threading: ${{ matrix.free-threading }} - - cross-build-linux: - name: Cross build Linux - runs-on: ubuntu-latest - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-tests == 'true' - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Set build dir - run: - # an absolute path outside of the working directoy - echo "BUILD_DIR=$(realpath ${{ github.workspace }}/../build)" >> "$GITHUB_ENV" - - name: Install dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure host build - run: ./configure --prefix="$BUILD_DIR/host-python" - - name: Install host Python - run: make -j8 install - - name: Run test subset with host build - run: | - "$BUILD_DIR/host-python/bin/python3" -m test test_sysconfig test_site test_embed - - name: Configure cross build - run: ./configure --prefix="$BUILD_DIR/cross-python" --with-build-python="$BUILD_DIR/host-python/bin/python3" - - name: Install cross Python - run: make -j8 install - - name: Run test subset with host build - run: | - "$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed - - # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ - cifuzz: - name: CIFuzz - runs-on: ubuntu-latest - timeout-minutes: 60 - needs: build-context - if: needs.build-context.outputs.run-ci-fuzz == 'true' - permissions: - security-events: write - strategy: - fail-fast: false - matrix: - sanitizer: [address, undefined, memory] - steps: - - name: Build fuzzers (${{ matrix.sanitizer }}) - id: build - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master - with: - oss-fuzz-project-name: cpython3 - sanitizer: ${{ matrix.sanitizer }} - - name: Run fuzzers (${{ matrix.sanitizer }}) - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master - with: - fuzz-seconds: 600 - oss-fuzz-project-name: cpython3 - output-sarif: true - sanitizer: ${{ matrix.sanitizer }} - - name: Upload crash - if: failure() && steps.build.outcome == 'success' - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.sanitizer }}-artifacts - path: ./out/artifacts - - name: Upload SARIF - if: always() && steps.build.outcome == 'success' - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: cifuzz-sarif/results.sarif - checkout_path: cifuzz-sarif - all-required-green: # This job does nothing and is only used for the branch protection name: All required checks pass runs-on: ubuntu-latest timeout-minutes: 5 needs: - build-context # Transitive dependency, needed to access `run-tests` value - - check-docs - check-autoconf-regen - check-generated-files - build-windows - - build-windows-msi - - build-macos - - build-ubuntu - - build-ubuntu-ssltests-awslc - - build-ubuntu-ssltests-openssl - build-wasi - - test-hypothesis - - build-asan - - build-san - - cross-build-linux - - cifuzz if: always() steps: - name: Check whether the needed jobs succeeded or failed uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe with: - allowed-failures: >- - build-windows-msi, - build-ubuntu-ssltests-awslc, - build-ubuntu-ssltests-openssl, - test-hypothesis, - cifuzz, allowed-skips: >- - ${{ - !fromJSON(needs.build-context.outputs.run-docs) - && ' - check-docs, - ' - || '' - }} ${{ needs.build-context.outputs.run-tests != 'true' && ' check-autoconf-regen, check-generated-files, - build-macos, - build-ubuntu, - build-ubuntu-ssltests-awslc, - build-ubuntu-ssltests-openssl, build-wasi, - test-hypothesis, - build-asan, - build-san, - cross-build-linux, ' || '' }} @@ -755,11 +209,4 @@ jobs: ' || '' }} - ${{ - !fromJSON(needs.build-context.outputs.run-ci-fuzz) - && ' - cifuzz, - ' - || '' - }} jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index d74ce8fcc256dc..00000000000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Lint - -on: [push, pull_request, workflow_dispatch] - -permissions: - contents: read - -env: - FORCE_COLOR: 1 - RUFF_OUTPUT_FORMAT: github - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - lint: - runs-on: ubuntu-latest - timeout-minutes: 10 - - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - uses: pre-commit/action@v3.0.1 diff --git a/Makefile.pre.in b/Makefile.pre.in index 959ccb891f283c..7d149a52a646fe 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -227,6 +227,7 @@ ENSUREPIP= @ENSUREPIP@ # Internal static libraries LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a LIBEXPAT_A= Modules/expat/libexpat.a +LIBHASHLIB_INTERNAL_A=Modules/_hashlib/libhashlib.a # HACL* build configuration LIBHACL_CFLAGS=@LIBHACL_CFLAGS@ @@ -761,6 +762,18 @@ LIBHACL_HMAC_HEADERS= \ $(LIBHACL_BLAKE2_HEADERS) \ $(LIBHACL_HEADERS) +########################################################################## +# Internal library for cryptographic primitives + +LIBHASHLIB_INTERNAL_OBJS= \ + Modules/_hashlib/hashlib_buffer.o \ + Modules/_hashlib/hashlib_fetch.o + +LIBHASHLIB_INTERNAL_HEADERS= \ + Modules/_hashlib/hashlib_buffer.h \ + Modules/_hashlib/hashlib_fetch.h \ + Modules/_hashlib/hashlib_mutex.h + ######################################################################### # Rules @@ -1481,6 +1494,20 @@ $(LIBEXPAT_A): $(LIBEXPAT_OBJS) -rm -f $@ $(AR) $(ARFLAGS) $@ $(LIBEXPAT_OBJS) +########################################################################## +# '_hashlib', '_hmac' and HACL*-based modules helpers +LIBHASHLIB_INTERNAL_CFLAGS=@LIBHASHLIB_INTERNAL_CFLAGS@ $(PY_STDMODULE_CFLAGS) $(CCSHARED) + +Modules/_hashlib/hashlib_buffer.o: Modules/_hashlib/hashlib_buffer.c $(LIBHASHLIB_INTERNAL_HEADERS) $(PYTHON_HEADERS) + $(CC) -I$(srcdir)/Modules/_hashlib -c $(LIBHASHLIB_INTERNAL_CFLAGS) -o $@ $(srcdir)/Modules/_hashlib/hashlib_buffer.c + +Modules/_hashlib/hashlib_fetch.o: Modules/_hashlib/hashlib_fetch.c $(LIBHASHLIB_INTERNAL_HEADERS) $(PYTHON_HEADERS) + $(CC) -I$(srcdir)/Modules/_hashlib -c $(LIBHASHLIB_INTERNAL_CFLAGS) -o $@ $(srcdir)/Modules/_hashlib/hashlib_fetch.c + +$(LIBHASHLIB_INTERNAL_A): $(LIBHASHLIB_INTERNAL_OBJS) + -rm -f $@ + $(AR) $(ARFLAGS) $@ $(LIBHASHLIB_INTERNAL_OBJS) + ########################################################################## # HACL* library build # @@ -3323,21 +3350,21 @@ MODULE__CTYPES_TEST_DEPS=$(srcdir)/Modules/_ctypes/_ctypes_test_generated.c.h MODULE__CTYPES_MALLOC_CLOSURE=@MODULE__CTYPES_MALLOC_CLOSURE@ MODULE__DECIMAL_DEPS=$(srcdir)/Modules/_decimal/docstrings.h @LIBMPDEC_INTERNAL@ MODULE__ELEMENTTREE_DEPS=$(srcdir)/Modules/pyexpat.c @LIBEXPAT_INTERNAL@ -MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h +MODULE__HASHLIB_DEPS=@LIBHASHLIB_INTERNAL@ MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h # HACL*-based cryptographic primitives -MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__MD5_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_MD5_HEADERS) $(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__MD5_LDEPS=$(LIBHACL_MD5_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA1_HEADERS) $(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA1_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA1_HEADERS) $(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SHA1_LDEPS=$(LIBHACL_SHA1_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA2_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SHA2_LDEPS=$(LIBHACL_SHA2_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__SHA3_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA3_HEADERS) $(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__SHA3_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_SHA3_HEADERS) $(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SHA3_LDEPS=$(LIBHACL_SHA3_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__BLAKE2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_BLAKE2_HEADERS) $(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__BLAKE2_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_BLAKE2_HEADERS) $(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__BLAKE2_LDEPS=$(LIBHACL_BLAKE2_LIB_@LIBHACL_LDEPS_LIBTYPE@) -MODULE__HMAC_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HMAC_HEADERS) $(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) +MODULE__HMAC_DEPS=$(MODULE__HASHLIB_DEPS) $(LIBHACL_HMAC_HEADERS) $(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__HMAC_LDEPS=$(LIBHACL_HMAC_LIB_@LIBHACL_LDEPS_LIBTYPE@) MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c diff --git a/Modules/_hashlib/hashlib_buffer.c b/Modules/_hashlib/hashlib_buffer.c new file mode 100644 index 00000000000000..34811c6266fd64 --- /dev/null +++ b/Modules/_hashlib/hashlib_buffer.c @@ -0,0 +1,40 @@ +#include "hashlib_buffer.h" + +int +_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string) +{ + if (data != NULL && string == NULL) { + // called as H(data) or H(data=...) + *res = data; + return 1; + } + else if (data == NULL && string != NULL) { + // called as H(string=...) + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "the 'string' keyword parameter is deprecated since " + "Python 3.15 and slated for removal in Python 3.19; " + "use the 'data' keyword parameter or pass the data " + "to hash as a positional argument instead", 1) < 0) + { + *res = NULL; + return -1; + } + *res = string; + return 1; + } + else if (data == NULL && string == NULL) { + // fast path when no data is given + assert(!PyErr_Occurred()); + *res = NULL; + return 0; + } + else { + // called as H(data=..., string) + *res = NULL; + PyErr_SetString(PyExc_TypeError, + "'data' and 'string' are mutually exclusive " + "and support for 'string' keyword parameter " + "is slated for removal in a future version."); + return -1; + } +} diff --git a/Modules/_hashlib/hashlib_buffer.h b/Modules/_hashlib/hashlib_buffer.h new file mode 100644 index 00000000000000..13b0aa1ab3cb61 --- /dev/null +++ b/Modules/_hashlib/hashlib_buffer.h @@ -0,0 +1,60 @@ +#ifndef _HASHLIB_HASHLIB_BUFFER_H +#define _HASHLIB_HASHLIB_BUFFER_H + +#include "Python.h" + +/* + * Given an buffer-like OBJ, fill in the buffer VIEW with the result + * of PyObject_GetBuffer. + * + * On error, set an exception and execute the ERRACTION statements, + * e.g. 'return NULL' or 'goto error'. + * + * Parameters + * + * OBJ An object supporting the buffer API. + * VIEW A Py_buffer pointer to fill. + * ERRACTION The statements to execute on error. + */ +#define GET_BUFFER_VIEW_OR_ERROR(OBJ, VIEW, ERRACTION) \ + do { \ + if (PyUnicode_Check((OBJ))) { \ + PyErr_SetString(PyExc_TypeError, \ + "strings must be encoded before hashing"); \ + ERRACTION; \ + } \ + if (!PyObject_CheckBuffer((OBJ))) { \ + PyErr_SetString(PyExc_TypeError, \ + "object supporting the buffer API required"); \ + ERRACTION; \ + } \ + if (PyObject_GetBuffer((OBJ), (VIEW), PyBUF_SIMPLE) == -1) { \ + ERRACTION; \ + } \ + if ((VIEW)->ndim > 1) { \ + PyErr_SetString(PyExc_BufferError, \ + "buffer must be one-dimensional"); \ + PyBuffer_Release((VIEW)); \ + ERRACTION; \ + } \ + } while(0) + +/* Specialization of GET_BUFFER_VIEW_OR_ERROR() returning NULL on error. */ +#define GET_BUFFER_VIEW_OR_ERROUT(OBJ, VIEW) \ + GET_BUFFER_VIEW_OR_ERROR(OBJ, VIEW, return NULL) + +/* + * Allow to use the 'data' or 'string' keyword in hashlib.new() + * and other hash functions named constructors. + * + * - If 'data' and 'string' are both non-NULL, set an exception and return -1. + * - If 'data' and 'string' are both NULL, set '*res' to NULL and return 0. + * - Otherwise, set '*res' to 'data' or 'string' and return 1. A deprecation + * warning is set when 'string' is specified. + * + * The symbol is exported for '_hashlib' and HACL*-based extension modules. + */ +PyAPI_FUNC(int) +_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string); + +#endif // !_HASHLIB_HASHLIB_BUFFER_H diff --git a/Modules/_hashlib/hashlib_fetch.c b/Modules/_hashlib/hashlib_fetch.c new file mode 100644 index 00000000000000..6b772030ad87d0 --- /dev/null +++ b/Modules/_hashlib/hashlib_fetch.c @@ -0,0 +1 @@ +#include "hashlib_fetch.h" diff --git a/Modules/_hashlib/hashlib_fetch.h b/Modules/_hashlib/hashlib_fetch.h new file mode 100644 index 00000000000000..a583b737923e21 --- /dev/null +++ b/Modules/_hashlib/hashlib_fetch.h @@ -0,0 +1,140 @@ +/* + * Interface for fetching a message digest from a digest-like identifier. + * + * The following table summaries the possible algorthms: + * + * +----------+--------------+--------------+---------------------------------+ + * | Family | Algorithm | Python Name | Notes | + * +==========+==============+==============+=================================+ + * | MD @ | + * | +--------------+--------------+---------------------------------+ + * | | MD5 | "md5" | | + * +----------+--------------+--------------+---------------------------------+ + * | SHA1 @ | + * | +--------------+--------------+---------------------------------+ + * | | SHA1-160 | "sha1" | | + * +----------+--------------+--------------+---------------------------------+ + * | SHA2 @ | + * | +--------------+--------------+---------------------------------+ + * | | SHA2-224 | "sha224" | | + * | | SHA2-256 | "sha256" | | + * | | SHA2-384 | "sha384" | | + * | | SHA2-512 | "sha512" | | + * +----------+--------------+--------------+---------------------------------+ + * | SHA2t @ Truncated SHA2-512 | + * | +--------------+--------------+---------------------------------+ + * | | SHA2-512/224 | "sha512_224" | | + * | | SHA2-512/256 | "sha512_256" | | + * +----------+--------------+--------------+---------------------------------+ + * | SHA3 @ | + * | +--------------+--------------+---------------------------------+ + * | | SHA3-224 | "sha3_224" | | + * | | SHA3-256 | "sha3_256" | | + * | | SHA3-384 | "sha3_384" | | + * | | SHA3-512 | "sha3_512" | | + * +----------+--------------+--------------+---------------------------------+ + * | SHA3-XOF @ Extensible Output Functions | + * | +--------------+--------------+---------------------------------+ + * | | SHAKE-128 | "shake_128" | | + * | | SHAKE-256 | "shake_256" | | + * +----------+--------------+--------------+---------------------------------+ + * | BLAKE2 @ | + * | +--------------+--------------+---------------------------------+ + * | | BLAKE2b | "blake2b" | | + * | | BLAKE2s | "blake2s" | | + * +----------+--------------+--------------+---------------------------------+ + */ + +#ifndef _HASHLIB_HASHLIB_FETCH_H +#define _HASHLIB_HASHLIB_FETCH_H + +#include "Python.h" + +/* + * Internal error messages used for reporting an unsupported hash algorithm. + * The algorithm can be given by its name, a callable or a PEP-247 module. + * The same message is raised by Lib/hashlib.py::__get_builtin_constructor() + * and _hmacmodule.c::find_hash_info(). + */ +#define _Py_HASHLIB_UNSUPPORTED_ALGORITHM "unsupported hash algorithm %S" +#define _Py_HASHLIB_UNSUPPORTED_STR_ALGORITHM "unsupported hash algorithm %s" + +#define _Py_HASHLIB_MD_NAMESPACE(NAME) _Py_hashlib_message_digest_ ## NAME +#define _Py_HASHLIB_MD_FAMILY(ID) _Py_HASHLIB_MD_NAMESPACE(family_ ## ID) +#define _Py_HASHLIB_MD_MEMBER(ID) _Py_HASHLIB_MD_NAMESPACE(member_ ## ID) + +#define _Py_HASHLIB_MD_NAMES _Py_HASHLIB_MD_NAMESPACE(NAMES) +#define _Py_HASHLIB_MD_COUNT Py_ARRAY_LENGTH(_Py_HASHLIB_MD_NAMES) +#define _Py_HASHLIB_MD_NAME(MEMBER_ID) \ + ( \ + assert(_Py_HASHLIB_MD_MEMBER(MEMBER_ID) >= 0), \ + assert(_Py_HASHLIB_MD_MEMBER(MEMBER_ID) < _Py_HASHLIB_MD_COUNT), \ + _Py_HASHLIB_MD_NAMES[_Py_HASHLIB_MD_MEMBER(MEMBER_ID)] \ + ) + +typedef enum { + _Py_HASHLIB_MD_FAMILY(MD) = 0, + _Py_HASHLIB_MD_FAMILY(SHA1), + _Py_HASHLIB_MD_FAMILY(SHA2), + _Py_HASHLIB_MD_FAMILY(SHA2t), + _Py_HASHLIB_MD_FAMILY(SHA3), + _Py_HASHLIB_MD_FAMILY(SHA3_XOF), + _Py_HASHLIB_MD_FAMILY(BLAKE2), +} _Py_HASHLIB_MD_NAMESPACE(family); + +typedef enum { + /* MD-family */ + _Py_HASHLIB_MD_MEMBER(md5) = 0, + /* SHA-1 family */ + _Py_HASHLIB_MD_MEMBER(sha1), + /* SHA-2 family */ + _Py_HASHLIB_MD_MEMBER(sha224), + _Py_HASHLIB_MD_MEMBER(sha256), + _Py_HASHLIB_MD_MEMBER(sha384), + _Py_HASHLIB_MD_MEMBER(sha512), + /* Truncated SHA-2 family */ + _Py_HASHLIB_MD_MEMBER(sha512_224), + _Py_HASHLIB_MD_MEMBER(sha512_256), + /* SHA-3 family */ + _Py_HASHLIB_MD_MEMBER(sha3_224), + _Py_HASHLIB_MD_MEMBER(sha3_256), + _Py_HASHLIB_MD_MEMBER(sha3_384), + _Py_HASHLIB_MD_MEMBER(sha3_512), + /* SHA-3 XOF SHAKE family */ + _Py_HASHLIB_MD_MEMBER(shake_128), + _Py_HASHLIB_MD_MEMBER(shake_256), + /* BLAKE-2 family */ + _Py_HASHLIB_MD_MEMBER(blake2b), + _Py_HASHLIB_MD_MEMBER(blake2s), +} _Py_HASHLIB_MD_NAMESPACE(member); + +static const char *_Py_HASHLIB_MD_NAMES[] = { +#define DECL_MESSAGE_DIGEST_NAME(ID) [_Py_HASHLIB_MD_MEMBER(ID)] = #ID + /* MD-family */ + DECL_MESSAGE_DIGEST_NAME(md5), + /* SHA-1 family */ + DECL_MESSAGE_DIGEST_NAME(sha1), + /* SHA-2 family */ + DECL_MESSAGE_DIGEST_NAME(sha224), + DECL_MESSAGE_DIGEST_NAME(sha256), + DECL_MESSAGE_DIGEST_NAME(sha384), + DECL_MESSAGE_DIGEST_NAME(sha512), + /* Truncated SHA-2 family */ + DECL_MESSAGE_DIGEST_NAME(sha512_224), + DECL_MESSAGE_DIGEST_NAME(sha512_256), + /* SHA-3 family */ + DECL_MESSAGE_DIGEST_NAME(sha3_224), + DECL_MESSAGE_DIGEST_NAME(sha3_256), + DECL_MESSAGE_DIGEST_NAME(sha3_384), + DECL_MESSAGE_DIGEST_NAME(sha3_512), + /* SHA-3 XOF SHAKE family */ + DECL_MESSAGE_DIGEST_NAME(shake_128), + DECL_MESSAGE_DIGEST_NAME(shake_256), + /* BLAKE-2 family */ + DECL_MESSAGE_DIGEST_NAME(blake2b), + DECL_MESSAGE_DIGEST_NAME(blake2s), +#undef DECL_MESSAGE_DIGEST_NAME + NULL /* sentinel */ +}; + +#endif // !_HASHLIB_HASHLIB_FETCH_H diff --git a/Modules/hashlib.h b/Modules/_hashlib/hashlib_mutex.h similarity index 53% rename from Modules/hashlib.h rename to Modules/_hashlib/hashlib_mutex.h index 5de5922c345047..d6924a2ef61e81 100644 --- a/Modules/hashlib.h +++ b/Modules/_hashlib/hashlib_mutex.h @@ -1,45 +1,14 @@ -/* Common code for use by all hashlib related modules. */ +#ifndef _HASHLIB_HASHLIB_MUTEX_H +#define _HASHLIB_HASHLIB_MUTEX_H -#include "pycore_lock.h" // PyMutex +#include "Python.h" +#include "pycore_lock.h" // PyMutex /* - * Internal error messages used for reporting an unsupported hash algorithm. - * The algorithm can be given by its name, a callable or a PEP-247 module. - * The same message is raised by Lib/hashlib.py::__get_builtin_constructor() - * and _hmacmodule.c::find_hash_info(). - */ -#define HASHLIB_UNSUPPORTED_ALGORITHM "unsupported hash algorithm %S" -#define HASHLIB_UNSUPPORTED_STR_ALGORITHM "unsupported hash algorithm %s" - -/* - * Given a PyObject* obj, fill in the Py_buffer* viewp with the result - * of PyObject_GetBuffer. Sets an exception and issues the erraction - * on any errors, e.g. 'return NULL' or 'goto error'. + * Message length above which the GIL is to be released + * when performing hashing operations. */ -#define GET_BUFFER_VIEW_OR_ERROR(obj, viewp, erraction) do { \ - if (PyUnicode_Check((obj))) { \ - PyErr_SetString(PyExc_TypeError, \ - "Strings must be encoded before hashing");\ - erraction; \ - } \ - if (!PyObject_CheckBuffer((obj))) { \ - PyErr_SetString(PyExc_TypeError, \ - "object supporting the buffer API required"); \ - erraction; \ - } \ - if (PyObject_GetBuffer((obj), (viewp), PyBUF_SIMPLE) == -1) { \ - erraction; \ - } \ - if ((viewp)->ndim > 1) { \ - PyErr_SetString(PyExc_BufferError, \ - "Buffer must be single dimension"); \ - PyBuffer_Release((viewp)); \ - erraction; \ - } \ - } while(0) - -#define GET_BUFFER_VIEW_OR_ERROUT(obj, viewp) \ - GET_BUFFER_VIEW_OR_ERROR(obj, viewp, return NULL) +#define HASHLIB_GIL_MINSIZE 2048 /* * Helper code to synchronize access to the hash object when the GIL is @@ -64,12 +33,6 @@ #define HASHLIB_ACQUIRE_LOCK(OBJ) PyMutex_Lock(&(OBJ)->mutex) #define HASHLIB_RELEASE_LOCK(OBJ) PyMutex_Unlock(&(OBJ)->mutex) -/* - * Message length above which the GIL is to be released - * when performing hashing operations. - */ -#define HASHLIB_GIL_MINSIZE 2048 - // Macros for executing code while conditionally holding the GIL. // // These only drop the GIL if the lock acquisition itself is likely to @@ -116,41 +79,4 @@ } \ } while (0) -static inline int -_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string) -{ - if (data != NULL && string == NULL) { - // called as H(data) or H(data=...) - *res = data; - return 1; - } - else if (data == NULL && string != NULL) { - // called as H(string=...) - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "the 'string' keyword parameter is deprecated since " - "Python 3.15 and slated for removal in Python 3.19; " - "use the 'data' keyword parameter or pass the data " - "to hash as a positional argument instead", 1) < 0) - { - *res = NULL; - return -1; - } - *res = string; - return 1; - } - else if (data == NULL && string == NULL) { - // fast path when no data is given - assert(!PyErr_Occurred()); - *res = NULL; - return 0; - } - else { - // called as H(data=..., string) - *res = NULL; - PyErr_SetString(PyExc_TypeError, - "'data' and 'string' are mutually exclusive " - "and support for 'string' keyword parameter " - "is slated for removal in a future version."); - return -1; - } -} +#endif // !_HASHLIB_HASHLIB_MUTEX_H diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index d79e4b360e95c5..c42513a53d4561 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -26,7 +26,10 @@ #include "pycore_hashtable.h" #include "pycore_strhex.h" // _Py_strhex() #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_RELAXED -#include "hashlib.h" + +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_fetch.h" +#include "_hashlib/hashlib_mutex.h" /* EVP is the preferred interface to hashing in OpenSSL */ #include @@ -532,7 +535,7 @@ raise_unsupported_algorithm_error(_hashlibstate *state, PyObject *digestmod) { raise_unsupported_algorithm_impl( state->unsupported_digestmod_error, - HASHLIB_UNSUPPORTED_ALGORITHM, + _Py_HASHLIB_UNSUPPORTED_ALGORITHM, digestmod ); } @@ -542,7 +545,7 @@ raise_unsupported_str_algorithm_error(_hashlibstate *state, const char *name) { raise_unsupported_algorithm_impl( state->unsupported_digestmod_error, - HASHLIB_UNSUPPORTED_STR_ALGORITHM, + _Py_HASHLIB_UNSUPPORTED_STR_ALGORITHM, name ); } diff --git a/Modules/blake2module.c b/Modules/blake2module.c index 163f238a4268d0..9828437d84a9b2 100644 --- a/Modules/blake2module.c +++ b/Modules/blake2module.c @@ -15,10 +15,11 @@ #endif #include "Python.h" -#include "hashlib.h" +#include "pycore_moduleobject.h" #include "pycore_strhex.h" // _Py_strhex() #include "pycore_typeobject.h" -#include "pycore_moduleobject.h" +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_mutex.h" // QUICK CPU AUTODETECTION // diff --git a/Modules/hmacmodule.c b/Modules/hmacmodule.c index b5405c99f1f8ce..4f132f38eb6f2f 100644 --- a/Modules/hmacmodule.c +++ b/Modules/hmacmodule.c @@ -45,9 +45,11 @@ #include "_hacl/Hacl_Streaming_HMAC.h" // Hacl_Agile_Hash_* identifiers #include "_hacl/Hacl_Streaming_Types.h" // Hacl_Streaming_Types_error_code -#include +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_fetch.h" +#include "_hashlib/hashlib_mutex.h" -#include "hashlib.h" +#include // --- Reusable error messages ------------------------------------------------ @@ -656,7 +658,7 @@ find_hash_info(hmacmodule_state *state, PyObject *hash_info_ref) } if (rc == 0) { PyErr_Format(state->unknown_hash_error, - HASHLIB_UNSUPPORTED_ALGORITHM, hash_info_ref); + _Py_HASHLIB_UNSUPPORTED_ALGORITHM, hash_info_ref); return NULL; } assert(info != NULL); diff --git a/Modules/md5module.c b/Modules/md5module.c index 8b6dd4a8195dfb..07ba6969dc70b1 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -23,8 +23,8 @@ #include "Python.h" #include "pycore_strhex.h" // _Py_strhex() - -#include "hashlib.h" +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_mutex.h" #include "_hacl/Hacl_Hash_MD5.h" diff --git a/Modules/sha1module.c b/Modules/sha1module.c index faa9dcccc5755b..1e9aaa0121f7dd 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -20,9 +20,10 @@ #endif #include "Python.h" -#include "hashlib.h" #include "pycore_strhex.h" // _Py_strhex() #include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_mutex.h" #include "_hacl/Hacl_Hash_SHA1.h" diff --git a/Modules/sha2module.c b/Modules/sha2module.c index 36300ba899fd44..6489fa18f47731 100644 --- a/Modules/sha2module.c +++ b/Modules/sha2module.c @@ -22,10 +22,10 @@ #include "Python.h" #include "pycore_moduleobject.h" // _PyModule_GetState() -#include "pycore_typeobject.h" // _PyType_GetModuleState() #include "pycore_strhex.h" // _Py_strhex() - -#include "hashlib.h" +#include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_mutex.h" #include "_hacl/Hacl_Hash_SHA2.h" diff --git a/Modules/sha3module.c b/Modules/sha3module.c index 5764556bb680f3..98fea127df9235 100644 --- a/Modules/sha3module.c +++ b/Modules/sha3module.c @@ -23,7 +23,8 @@ #include "Python.h" #include "pycore_strhex.h" // _Py_strhex() #include "pycore_typeobject.h" // _PyType_GetModuleState() -#include "hashlib.h" +#include "_hashlib/hashlib_buffer.h" +#include "_hashlib/hashlib_mutex.h" #include "_hacl/Hacl_Hash_SHA3.h" diff --git a/PCbuild/_hashlib.vcxproj b/PCbuild/_hashlib.vcxproj index 2cd205224bc089..9e6dcee40793d3 100644 --- a/PCbuild/_hashlib.vcxproj +++ b/PCbuild/_hashlib.vcxproj @@ -100,6 +100,12 @@ + + + + + + diff --git a/PCbuild/_hashlib.vcxproj.filters b/PCbuild/_hashlib.vcxproj.filters index 7a0700c007f644..d465d92a956eda 100644 --- a/PCbuild/_hashlib.vcxproj.filters +++ b/PCbuild/_hashlib.vcxproj.filters @@ -18,4 +18,4 @@ Resource Files - \ No newline at end of file + diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index b911c9385634d7..a83ede77fb7f39 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -447,6 +447,11 @@ HACL_CAN_COMPILE_VEC128;%(PreprocessorDefinitions) /arch:AVX %(AdditionalOptions) + + + + + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 0e6d42cc959ba5..9cc395cac31b15 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -255,6 +255,15 @@ Include + + Modules\_hashlib + + + Modules\_hashlib + + + Modules\_hashlib + Modules @@ -971,6 +980,12 @@ Modules + + Modules\_hashlib + + + Modules\_hashlib + Modules diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index dc626e4bea0f59..5cd494dfd66722 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -238,6 +238,7 @@ Modules/_decimal/_decimal.c - signal_map_template - Modules/_decimal/_decimal.c - ssize_constants - Modules/_decimal/_decimal.c - INVALID_SIGNALDICT_ERROR_MSG - Modules/_elementtree.c - ExpatMemoryHandler - +Modules/_hashlib/hashlib_fetch.h - _Py_hashlib_message_digest_NAMES - Modules/_hashopenssl.c - py_hashes - Modules/_hacl/Hacl_Hash_SHA1.c - _h0 - Modules/_hacl/Hacl_Hash_MD5.c - _h0 - diff --git a/configure b/configure index ef47f9b0df73a8..cb5f9e4ec5c0fd 100755 --- a/configure +++ b/configure @@ -725,6 +725,8 @@ LIBHACL_BLAKE2_SIMD128_OBJS LIBHACL_SIMD128_FLAGS LIBHACL_LDFLAGS LIBHACL_CFLAGS +LIBHASHLIB_INTERNAL +LIBHASHLIB_INTERNAL_CFLAGS MODULE_UNICODEDATA_FALSE MODULE_UNICODEDATA_TRUE MODULE__MULTIBYTECODEC_FALSE @@ -29947,6 +29949,7 @@ SRCDIRS="\ Modules/_decimal \ Modules/_decimal/libmpdec \ Modules/_hacl \ + Modules/_hashlib \ Modules/_io \ Modules/_multiprocessing \ Modules/_sqlite \ @@ -32525,6 +32528,15 @@ then : fi +############################################################################### +# Cryptographic primitives +LIBHASHLIB_INTERNAL_CFLAGS="-I\$(srcdir)/Modules/_hashlib" +LIBHASHLIB_INTERNAL_LDFLAGS="-lm \$(LIBHASHLIB_INTERNAL_A)" +LIBHASHLIB_INTERNAL="\$(LIBHASHLIB_INTERNAL_HEADERS) \$(LIBHASHLIB_INTERNAL_A)" + + + + ############################################################################### # HACL* compilation and linking configuration (contact: @picnixz) # @@ -32773,8 +32785,8 @@ fi if test "x$py_cv_module__md5" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__MD5_CFLAGS=$LIBHACL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__MD5_LDFLAGS=\$($LIBHACL_MD5_LDFLAGS)$as_nl" + as_fn_append MODULE_BLOCK "MODULE__MD5_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__MD5_LDFLAGS=\$($LIBHACL_MD5_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__md5" = yes; then @@ -32818,8 +32830,8 @@ fi if test "x$py_cv_module__sha1" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA1_CFLAGS=$LIBHACL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__SHA1_LDFLAGS=\$($LIBHACL_SHA1_LDFLAGS)$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA1_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA1_LDFLAGS=\$($LIBHACL_SHA1_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__sha1" = yes; then @@ -32863,8 +32875,8 @@ fi if test "x$py_cv_module__sha2" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA2_CFLAGS=$LIBHACL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__SHA2_LDFLAGS=\$($LIBHACL_SHA2_LDFLAGS)$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA2_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA2_LDFLAGS=\$($LIBHACL_SHA2_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__sha2" = yes; then @@ -32908,8 +32920,8 @@ fi if test "x$py_cv_module__sha3" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__SHA3_CFLAGS=$LIBHACL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__SHA3_LDFLAGS=\$($LIBHACL_SHA3_LDFLAGS)$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA3_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__SHA3_LDFLAGS=\$($LIBHACL_SHA3_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__sha3" = yes; then @@ -32953,8 +32965,8 @@ fi if test "x$py_cv_module__blake2" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__BLAKE2_CFLAGS=$LIBHACL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__BLAKE2_LDFLAGS=\$($LIBHACL_BLAKE2_LDFLAGS)$as_nl" + as_fn_append MODULE_BLOCK "MODULE__BLAKE2_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__BLAKE2_LDFLAGS=\$($LIBHACL_BLAKE2_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__blake2" = yes; then @@ -32999,8 +33011,8 @@ fi if test "x$py_cv_module__hmac" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__HMAC_CFLAGS=$LIBHACL_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__HMAC_LDFLAGS=\$($LIBHACL_HMAC_LDFLAGS)$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HMAC_CFLAGS=$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HMAC_LDFLAGS=\$($LIBHACL_HMAC_LDFLAGS) $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__hmac" = yes; then @@ -33681,8 +33693,8 @@ fi if test "x$py_cv_module__hashlib" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__HASHLIB_CFLAGS=$OPENSSL_INCLUDES$as_nl" - as_fn_append MODULE_BLOCK "MODULE__HASHLIB_LDFLAGS=$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HASHLIB_CFLAGS=$OPENSSL_INCLUDES $LIBHASHLIB_INTERNAL_CFLAGS$as_nl" + as_fn_append MODULE_BLOCK "MODULE__HASHLIB_LDFLAGS=$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS $LIBHASHLIB_INTERNAL_LDFLAGS$as_nl" fi if test "$py_cv_module__hashlib" = yes; then diff --git a/configure.ac b/configure.ac index 23ed9cd35bc94b..3ca67bcf188f3b 100644 --- a/configure.ac +++ b/configure.ac @@ -7186,6 +7186,7 @@ SRCDIRS="\ Modules/_decimal \ Modules/_decimal/libmpdec \ Modules/_hacl \ + Modules/_hashlib \ Modules/_io \ Modules/_multiprocessing \ Modules/_sqlite \ @@ -7958,6 +7959,15 @@ PY_STDLIB_MOD_SIMPLE([_codecs_tw]) PY_STDLIB_MOD_SIMPLE([_multibytecodec]) PY_STDLIB_MOD_SIMPLE([unicodedata]) +############################################################################### +# Cryptographic primitives +LIBHASHLIB_INTERNAL_CFLAGS="-I\$(srcdir)/Modules/_hashlib" +LIBHASHLIB_INTERNAL_LDFLAGS="-lm \$(LIBHASHLIB_INTERNAL_A)" +LIBHASHLIB_INTERNAL="\$(LIBHASHLIB_INTERNAL_HEADERS) \$(LIBHASHLIB_INTERNAL_A)" + +AC_SUBST([LIBHASHLIB_INTERNAL_CFLAGS]) +AC_SUBST([LIBHASHLIB_INTERNAL]) + ############################################################################### # HACL* compilation and linking configuration (contact: @picnixz) # @@ -8094,7 +8104,9 @@ dnl The EXTNAME is the name of the extension module being built. AC_DEFUN([PY_HACL_CREATE_MODULE], [ AS_VAR_PUSHDEF([v], [[LIBHACL_][$1][_LDFLAGS]]) AS_VAR_SET([v], [[LIBHACL_][$1][_LIB_${LIBHACL_LDEPS_LIBTYPE}]]) - PY_STDLIB_MOD([$2], [$3], [], [$LIBHACL_CFLAGS], [\$($v)]) + PY_STDLIB_MOD([$2], [$3], [], + [$LIBHACL_CFLAGS $LIBHASHLIB_INTERNAL_CFLAGS], + [\$($v) $LIBHASHLIB_INTERNAL_LDFLAGS]) AS_VAR_POPDEF([v]) ]) @@ -8175,7 +8187,8 @@ dnl OpenSSL bindings PY_STDLIB_MOD([_ssl], [], [test "$ac_cv_working_openssl_ssl" = yes], [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $OPENSSL_LIBS]) PY_STDLIB_MOD([_hashlib], [], [test "$ac_cv_working_openssl_hashlib" = yes], - [$OPENSSL_INCLUDES], [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS]) + [$OPENSSL_INCLUDES $LIBHASHLIB_INTERNAL_CFLAGS], + [$OPENSSL_LDFLAGS $OPENSSL_LDFLAGS_RPATH $LIBCRYPTO_LIBS $LIBHASHLIB_INTERNAL_LDFLAGS]) dnl test modules PY_STDLIB_MOD([_testcapi],