diff --git a/.github/actions/action.yml b/.github/actions/action.yml index 20a239026e4f..2c271c0c5726 100644 --- a/.github/actions/action.yml +++ b/.github/actions/action.yml @@ -11,7 +11,6 @@ runs: echo DOWNLOAD_OPENBLAS $DOWNLOAD_OPENBLAS echo USE_DEBUG $USE_DEBUG echo NPY_USE_BLAS_ILP64 $NPY_USE_BLAS_ILP64 - echo NUMPY_EXPERIMENTAL_ARRAY_FUNCTION $NUMPY_EXPERIMENTAL_ARRAY_FUNCTION echo USE_ASV $USE_ASV echo PATH $PATH echo python `which python` diff --git a/.github/meson_actions/action.yml b/.github/meson_actions/action.yml index aff70da169bc..fb3042c0ade2 100644 --- a/.github/meson_actions/action.yml +++ b/.github/meson_actions/action.yml @@ -3,27 +3,35 @@ description: "checkout repo, build, and test numpy" runs: using: composite steps: - - name: Install dependencies - shell: bash - run: pip install -r build_requirements.txt - name: Build shell: 'script -q -e -c "bash --noprofile --norc -eo pipefail {0}"' env: TERM: xterm-256color - run: - spin build -- ${MESON_ARGS[@]} - - name: Check build-internal dependencies - shell: bash - run: - ninja -C build -t missingdeps - - name: Check installed test and stub files + PKG_CONFIG_PATH: ./.openblas + run: | + echo "::group::Installing Build Dependencies" + pip install -r build_requirements.txt + echo "::endgroup::" + echo "::group::Building NumPy" + spin build --clean -- ${MESON_ARGS[@]} + echo "::endgroup::" + + - name: Meson Log shell: bash - run: - python tools/check_installed_files.py $(find ./build-install -path '*/site-packages/numpy') + if: always() + run: | + echo "::group::Meson Log" + cat build/meson-logs/meson-log.txt + echo "::endgroup::" + - name: Test shell: 'script -q -e -c "bash --noprofile --norc -eo pipefail {0}"' env: TERM: xterm-256color run: | - pip install pytest pytest-xdist hypothesis typing_extensions - spin test -j auto + echo "::group::Installing Test Dependencies" + pip install pytest pytest-xdist hypothesis typing_extensions setuptools + echo "::endgroup::" + echo "::group::Test NumPy" + spin test + echo "::endgroup::" diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml deleted file mode 100644 index e570c5540415..000000000000 --- a/.github/workflows/build_test.yml +++ /dev/null @@ -1,447 +0,0 @@ -name: Build_Test - -on: - push: - branches: - # coverage comparison in the "full" step needs to run on main after merges - - main - pull_request: - branches: - - main - - maintenance/** - -defaults: - run: - shell: bash - -env: - DOWNLOAD_OPENBLAS: 1 - PYTHON_VERSION: 3.9 - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -permissions: - contents: read # to fetch code (actions/checkout) - -jobs: - lint: - if: github.repository == 'numpy/numpy' && github.event_name != 'push' - runs-on: ubuntu-latest - continue-on-error: true - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - name: Install linter requirements - run: - python -m pip install -r linter_requirements.txt - - name: Run linter on PR diff - run: - python tools/linter.py --branch origin/${{ github.base_ref }} - - smoke_test: - if: "github.repository == 'numpy/numpy'" - runs-on: ubuntu-latest - env: - MESON_ARGS: "-Dallow-noblas=true -Dcpu-baseline=none -Dcpu-dispatch=none" - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - uses: ./.github/meson_actions - - basic: - needs: [smoke_test] - runs-on: ubuntu-latest - if: github.event_name != 'push' - strategy: - matrix: - python-version: ["3.9", "3.10", "3.11", "pypy3.9-v7.3.12"] - env: - EXPECT_CPU_FEATURES: "SSE SSE2 SSE3 SSSE3 SSE41 POPCNT SSE42 AVX F16C FMA3 AVX2 AVX512F AVX512CD AVX512_KNL AVX512_KNM AVX512_SKX AVX512_CLX AVX512_CNL AVX512_ICL" - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: ${{ matrix.python-version }} - - uses: ./.github/actions - - old_gcc: - needs: [smoke_test] - # provides GCC 7, 8 - runs-on: ubuntu-20.04 - if: github.event_name != 'push' - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - submodules: recursive - fetch-depth: 0 - # comes with python3.6 - - name: Install Python3.9 - run: | - sudo apt update - # for add-apt-repository - sudo apt install software-properties-common -y - sudo add-apt-repository ppa:deadsnakes/ppa -y - sudo apt install python3.9-dev ninja-build -y - sudo ln -s /usr/bin/python3.9 /usr/bin/pythonx - pythonx -m pip install --upgrade pip setuptools wheel - pythonx -m pip install -r test_requirements.txt - - name: Install Compilers - run: sudo apt install g++-7 g++-8 -y - - name: Build gcc-7 - run: | - export CC=/usr/bin/gcc-7 - export CXX=/usr/bin/g++-7 - rm -rf build && pythonx setup.py install --user - - name: Runtests gcc-7 - run: pythonx runtests.py -n - - name: Build gcc-8 - run: | - export CC=/usr/bin/gcc-8 - export CXX=/usr/bin/g++-8 - rm -rf build && pythonx setup.py install --user - - name: Runtests gcc-8 - run: pythonx runtests.py -n - - without_optimizations: - needs: [smoke_test] - runs-on: ubuntu-latest - if: github.event_name != 'push' - env: - MESON_ARGS: "-Dallow-noblas=true -Ddisable-optimization=true" - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - uses: ./.github/meson_actions - - with_baseline_only: - needs: [smoke_test] - runs-on: ubuntu-latest - if: github.event_name != 'push' - env: - MESON_ARGS: "-Dallow-noblas=true -Dcpu-dispatch=none" - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - uses: ./.github/meson_actions - - without_avx512: - needs: [smoke_test] - runs-on: ubuntu-latest - if: github.event_name != 'push' - env: - MESON_ARGS: "-Dallow-noblas=true -Dcpu-dispatch=SSSE3,SSE41,POPCNT,SSE42,AVX,F16C,AVX2,FMA3" - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - uses: ./.github/meson_actions - - without_avx512_avx2_fma3: - needs: [smoke_test] - runs-on: ubuntu-latest - if: github.event_name != 'push' - env: - MESON_ARGS: "-Dallow-noblas=true -Dcpu-dispatch=SSSE3,SSE41,POPCNT,SSE42,AVX,F16C" - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - uses: ./.github/meson_actions - - debug: - needs: [smoke_test] - runs-on: ubuntu-latest - if: github.event_name != 'push' - env: - USE_DEBUG: 1 - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - uses: ./.github/actions - - blas64: - needs: [smoke_test] - runs-on: ubuntu-latest - if: github.event_name != 'push' - env: - NPY_USE_BLAS_ILP64: 1 - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - uses: ./.github/actions - - full: - needs: [smoke_test] - runs-on: ubuntu-22.04 - env: - USE_WHEEL: 1 - RUN_FULL_TESTS: 1 - RUN_COVERAGE: 1 - INSTALL_PICKLE5: 1 - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - uses: ./.github/actions - - benchmark: - needs: [smoke_test] - runs-on: ubuntu-latest - if: github.event_name != 'push' - env: - PYTHONOPTIMIZE: 2 - BLAS: None - LAPACK: None - ATLAS: None - NPY_BLAS_ORDER: mkl,blis,openblas,atlas,blas - NPY_LAPACK_ORDER: MKL,OPENBLAS,ATLAS,LAPACK - USE_ASV: 1 - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - uses: ./.github/actions - - relaxed_strides_debug: - needs: [smoke_test] - runs-on: ubuntu-latest - if: github.event_name != 'push' - env: - CHECK_BLAS: 1 - NPY_USE_BLAS_ILP64: 1 - NPY_RELAXED_STRIDES_DEBUG: 1 - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - uses: ./.github/actions - - use_wheel: - needs: [smoke_test] - runs-on: ubuntu-latest - if: github.event_name != 'push' - env: - USE_WHEEL: 1 - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - uses: ./.github/actions - - no_openblas: - needs: [smoke_test] - runs-on: ubuntu-latest - if: github.event_name != 'push' - env: - BLAS: None - LAPACK: None - ATLAS: None - DOWNLOAD_OPENBLAS: '' - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - uses: ./.github/actions - - sdist: - needs: [smoke_test] - runs-on: ubuntu-latest - if: github.event_name != 'push' - env: - USE_SDIST: 1 - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - uses: ./.github/actions - - armv7_simd_test: - needs: [smoke_test] - # make sure this matches the base docker image below - runs-on: ubuntu-22.04 - if: github.event_name != 'push' - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - submodules: recursive - fetch-depth: 0 - - name: Initialize binfmt_misc for qemu-user-static - run: | - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - name: Creates new container - run: | - # use x86_64 cross-compiler to speed up the build - sudo apt update - sudo apt install -y gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf gfortran-arm-linux-gnueabihf - - docker run --name the_container --interactive -v /:/host -v $(pwd):/numpy arm32v7/ubuntu:22.04 /bin/bash -c " - apt update && - apt install -y git python3 python3-dev python3-pip && - python3 -m pip install -r /numpy/test_requirements.txt - ln -s /host/lib64 /lib64 && - ln -s /host/lib/x86_64-linux-gnu /lib/x86_64-linux-gnu && - ln -s /host/usr/arm-linux-gnueabihf /usr/arm-linux-gnueabihf && - rm -rf /usr/lib/gcc/arm-linux-gnueabihf && ln -s /host/usr/lib/gcc-cross/arm-linux-gnueabihf /usr/lib/gcc/arm-linux-gnueabihf && - rm -f /usr/bin/arm-linux-gnueabihf-gcc && ln -s /host/usr/bin/arm-linux-gnueabihf-gcc /usr/bin/arm-linux-gnueabihf-gcc && - rm -f /usr/bin/arm-linux-gnueabihf-g++ && ln -s /host/usr/bin/arm-linux-gnueabihf-g++ /usr/bin/arm-linux-gnueabihf-g++ && - rm -f /usr/bin/arm-linux-gnueabihf-gfortran && ln -s /host/usr/bin/arm-linux-gnueabihf-gfortran /usr/bin/arm-linux-gnueabihf-gfortran && - rm -f /usr/bin/arm-linux-gnueabihf-ar && ln -s /host/usr/bin/arm-linux-gnueabihf-ar /usr/bin/arm-linux-gnueabihf-ar && - rm -f /usr/bin/arm-linux-gnueabihf-as && ln -s /host/usr/bin/arm-linux-gnueabihf-as /usr/bin/arm-linux-gnueabihf-as && - rm -f /usr/bin/arm-linux-gnueabihf-ld && ln -s /host/usr/bin/arm-linux-gnueabihf-ld /usr/bin/arm-linux-gnueabihf-ld && - rm -f /usr/bin/arm-linux-gnueabihf-ld.bfd && ln -s /host/usr/bin/arm-linux-gnueabihf-ld.bfd /usr/bin/arm-linux-gnueabihf-ld.bfd - " - docker commit the_container the_container - - name: Build - run: | - sudo docker run --name the_build --interactive -v $(pwd):/numpy -v /:/host the_container /bin/bash -c " - uname -a && - gcc --version && - g++ --version && - arm-linux-gnueabihf-gfortran --version && - python3 --version && - git config --global --add safe.directory /numpy - cd /numpy && - python3 setup.py install - " - docker commit the_build the_build - - name: Run SIMD Tests - run: | - docker run --rm --interactive -v $(pwd):/numpy the_build /bin/bash -c " - cd /numpy && F90=arm-linux-gnueabihf-gfortran python3 runtests.py -n -v -- -k 'test_simd or test_kind' - " - - sde_simd_avx512_test: - # Intel Software Development Emulator (SDE) is used to run a given program - # on a specific instruction set architecture and capture various performance details. - # see https://www.intel.com/content/www/us/en/developer/articles/tool/software-development-emulator.html - needs: [smoke_test] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - name: Install Intel SDE - run: | - curl -o /tmp/sde.tar.xz https://downloadmirror.intel.com/732268/sde-external-9.7.0-2022-05-09-lin.tar.xz - mkdir /tmp/sde && tar -xvf /tmp/sde.tar.xz -C /tmp/sde/ - sudo mv /tmp/sde/* /opt/sde && sudo ln -s /opt/sde/sde64 /usr/bin/sde - - name: Install dependencies - run: python -m pip install -r test_requirements.txt - - name: Build - run: python setup.py build - --simd-test="\$werror AVX512F AVX512_KNL AVX512_KNM AVX512_SKX AVX512_CLX AVX512_CNL AVX512_ICL" - install - # KNM implies KNL - - name: Run SIMD tests (Xeon PHI) - run: sde -knm -- python runtests.py -n -v -- -k test_simd - # ICL implies SKX, CLX and CNL - - name: Run SIMD tests (Ice Lake) - run: sde -icl -- python runtests.py -n -v -- -k test_simd - - intel_spr_sde_test: - needs: [smoke_test] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 - with: - submodules: recursive - fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 - with: - python-version: ${{ env.PYTHON_VERSION }} - - name: Install Intel SDE - run: | - curl -o /tmp/sde.tar.xz https://downloadmirror.intel.com/784319/sde-external-9.24.0-2023-07-13-lin.tar.xz - mkdir /tmp/sde && tar -xvf /tmp/sde.tar.xz -C /tmp/sde/ - sudo mv /tmp/sde/* /opt/sde && sudo ln -s /opt/sde/sde64 /usr/bin/sde - - name: Install dependencies - run: | - python -m pip install -r test_requirements.txt - sudo apt install gcc-12 g++-12 - - name: Build and install NumPy - run: | - export CC=/usr/bin/gcc-12 - export CXX=/usr/bin/g++-12 - python setup.py develop - - name: Show config - run: | - python -c "import numpy as np; np.show_config()" - # Run only a few tests, running everything in an SDE takes a long time - # Using pytest directly, unable to use python runtests.py -n -t ... - - name: Run linalg/ufunc/umath tests - run: | - python -m pytest numpy/core/tests/test_umath* numpy/core/tests/test_ufunc.py numpy/linalg/tests/test_* - # Can't run on SDE just yet: see https://github.com/numpy/numpy/issues/23545#issuecomment-1659047365 - #sde -spr -- python -m pytest numpy/core/tests/test_umath* numpy/core/tests/test_ufunc.py numpy/linalg/tests/test_* diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml new file mode 100644 index 000000000000..ebceec365d7c --- /dev/null +++ b/.github/workflows/linux.yml @@ -0,0 +1,239 @@ +name: Linux tests + +# This file is meant for testing across supported Python versions, build types +# and interpreters (PyPy, python-dbg, a pre-release Python in summer time), +# build-via-sdist, run benchmarks, measure code coverage, and other build +# options like relaxed-strides. + +on: + push: + branches: + # coverage comparison in the "full" step needs to run on main after merges + - main + pull_request: + branches: + - main + - maintenance/** + +defaults: + run: + shell: bash + +env: + DOWNLOAD_OPENBLAS: 1 + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +jobs: + lint: + if: github.repository == 'numpy/numpy' && github.event_name != 'push' + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: '3.9' + - name: Install linter requirements + run: + python -m pip install -r linter_requirements.txt + - name: Run linter on PR diff + run: + python tools/linter.py --branch origin/${{ github.base_ref }} + + smoke_test: + if: "github.repository == 'numpy/numpy'" + runs-on: ubuntu-latest + env: + MESON_ARGS: "-Dallow-noblas=true -Dcpu-baseline=none -Dcpu-dispatch=none" + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: '3.9' + - uses: ./.github/meson_actions + + basic: + needs: [smoke_test] + runs-on: ubuntu-latest + if: github.event_name != 'push' + strategy: + matrix: + python-version: ["3.9", "pypy3.9-v7.3.12"] + env: + EXPECT_CPU_FEATURES: "SSE SSE2 SSE3 SSSE3 SSE41 POPCNT SSE42 AVX F16C FMA3 AVX2 AVX512F AVX512CD AVX512_KNL AVX512_KNM AVX512_SKX AVX512_CLX AVX512_CNL AVX512_ICL" + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: ${{ matrix.python-version }} + - uses: ./.github/actions + + debug: + needs: [smoke_test] + runs-on: ubuntu-latest + if: github.event_name != 'push' + env: + USE_DEBUG: 1 + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: '3.11' + + - uses: ./.github/actions + + full: + # Build a wheel, install it, then run the full test suite with code coverage + needs: [smoke_test] + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: '3.9' + - name: Install build and test dependencies from PyPI + run: | + pip install -r build_requirements.txt + pip install -r test_requirements.txt + - name: Install gfortran and OpenBLAS (MacPython build) + run: | + set -xe + sudo apt install gfortran libgfortran5 + target=$(python tools/openblas_support.py) + sudo cp -r $target/lib/* /usr/lib + sudo cp $target/include/* /usr/include + - name: Build a wheel + run: | + python -m build --wheel --no-isolation --skip-dependency-check + pip install dist/numpy*.whl + - name: Run full test suite + run: | + cd tools + pytest --pyargs numpy --cov-report=html:build/coverage + # TODO: gcov + + benchmark: + needs: [smoke_test] + runs-on: ubuntu-latest + if: github.event_name != 'push' + env: + PYTHONOPTIMIZE: 2 + BLAS: None + LAPACK: None + ATLAS: None + NPY_BLAS_ORDER: mkl,blis,openblas,atlas,blas + NPY_LAPACK_ORDER: MKL,OPENBLAS,ATLAS,LAPACK + USE_ASV: 1 + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: '3.9' + - uses: ./.github/actions + + relaxed_strides_debug: + needs: [smoke_test] + runs-on: ubuntu-latest + if: github.event_name != 'push' + env: + CHECK_BLAS: 1 + NPY_USE_BLAS_ILP64: 1 + NPY_RELAXED_STRIDES_DEBUG: 1 + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: '3.11' + - uses: ./.github/actions + + sdist: + needs: [smoke_test] + runs-on: ubuntu-latest + if: github.event_name != 'push' + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: '3.11' + - name: Install gfortran and OpenBLAS (MacPython build) + run: | + set -xe + sudo apt install gfortran libgfortran5 + target=$(python tools/openblas_support.py) + sudo cp -r $target/lib/* /usr/lib + sudo cp $target/include/* /usr/include + - name: Build a wheel via an sdist + run: | + pip install build + python -m build + pip install dist/numpy*.whl + - name: Install test dependencies + run: | + pip install -r test_requirements.txt + pip install ninja + - name: Run test suite + run: | + cd tools + pytest --pyargs numpy -m "not slow" + + custom_checks: + needs: [smoke_test] + runs-on: ubuntu-latest + if: github.event_name != 'push' + steps: + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: '3.11' + - name: Install build and test dependencies from PyPI + run: | + pip install -r build_requirements.txt + pip install -r test_requirements.txt + pip install vulture + - name: Build and install NumPy + run: | + # Install using the fastests way to build (no BLAS, no SIMD) + spin build -j2 -- -Dallow-noblas=true -Dcpu-baseline=none -Dcpu-dispatch=none + - name: Check build-internal dependencies + run: | + ninja -C build -t missingdeps + - name: Check installed test and stub files + run: | + python tools/check_installed_files.py $(find ./build-install -path '*/site-packages/numpy') + - name: Check for unreachable code paths in Python modules + run: | + # Need the explicit `bash -c` here because `grep` returns exit code 1 for no matches + bash -c "! vulture . --min-confidence 100 --exclude doc/,numpy/distutils/,vendored-meson/ | grep 'unreachable'" diff --git a/.github/workflows/linux_meson.yml b/.github/workflows/linux_blas.yml similarity index 73% rename from .github/workflows/linux_meson.yml rename to .github/workflows/linux_blas.yml index ebbd52997e42..aac37d205e9d 100644 --- a/.github/workflows/linux_meson.yml +++ b/.github/workflows/linux_blas.yml @@ -7,6 +7,10 @@ name: BLAS tests (Linux) # # Jobs and their purpose: # +# - openblas64_setuppy: +# This job uses the default 64-bit build of OpenBLAS with the +# `numpy.distutils`-based build. It can be removed once we remove +# support for those builds. # - openblas32_stable_nightly: # Uses the 32-bit OpenBLAS builds, both the latest stable release # and a nightly build. @@ -20,9 +24,12 @@ name: BLAS tests (Linux) # - openblas_cmake: # Tests whether OpenBLAS LP64 is detected correctly when only CMake # and not pkg-config is installed. -# - netlib: -# Installs vanilla blas/lapack, which is the last option tried in -# auto-detection. +# - netlib-debian: +# Installs libblas/liblapack, which in Debian contains libcblas within +# libblas. +# - netlib-split: +# Installs vanilla Netlib blas/lapack with separate libcblas, which is +# the last option tried in auto-detection. # - mkl: # Tests MKL installed from PyPI (because easiest/fastest, if broken) in # 3 ways: both LP64 and ILP64 via pkg-config, and then using the @@ -42,9 +49,6 @@ defaults: run: shell: bash -env: - PYTHON_VERSION: 3.11 - concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true @@ -53,7 +57,7 @@ permissions: contents: read # to fetch code (actions/checkout) jobs: - meson_spin: + openblas32_stable_nightly: if: "github.repository == 'numpy/numpy'" runs-on: ubuntu-latest strategy: @@ -63,13 +67,14 @@ jobs: USE_NIGHTLY_OPENBLAS: ${{ matrix.USE_NIGHTLY_OPENBLAS }} name: "Test Linux (${{ matrix.USE_NIGHTLY_OPENBLAS && 'nightly' || 'stable' }} OpenBLAS)" steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: submodules: recursive fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 with: - python-version: ${{ env.PYTHON_VERSION }} + python-version: '3.11' + - name: Install dependencies run: | pip install -r build_requirements.txt @@ -82,25 +87,29 @@ jobs: fi sudo cp -r $target/lib/* /usr/lib sudo cp $target/include/* /usr/include + - name: Build shell: 'script -q -e -c "bash --noprofile --norc -eo pipefail {0}"' env: TERM: xterm-256color run: - spin build -- --werror + spin build -- --werror -Dallow-noblas=false + - name: Check build-internal dependencies run: ninja -C build -t missingdeps + - name: Check installed test and stub files run: python tools/check_installed_files.py $(find ./build-install -path '*/site-packages/numpy') + - name: Test shell: 'script -q -e -c "bash --noprofile --norc -eo pipefail {0}"' env: TERM: xterm-256color LD_LIBRARY_PATH: "/usr/local/lib/" # to find libopenblas.so.0 + run: | - export NPY_RUN_MYPY_IN_TESTSUITE=1 pip install pytest pytest-xdist hypothesis typing_extensions spin test -j auto @@ -115,7 +124,7 @@ jobs: run: | dnf install git gcc-gfortran g++ python3-devel openblas-devel -y - - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: submodules: recursive fetch-depth: 0 @@ -126,7 +135,7 @@ jobs: pip install pytest hypothesis typing_extensions - name: Build (LP64) - run: spin build -- -Dblas=openblas -Dlapack=openblas -Ddisable-optimization=true + run: spin build -- -Dblas=openblas -Dlapack=openblas -Ddisable-optimization=true -Dallow-noblas=false - name: Test run: spin test -- numpy/linalg @@ -134,7 +143,7 @@ jobs: - name: Build (ILP64) run: | rm -rf build - spin build -- -Duse-ilp64=true -Ddisable-optimization=true + spin build -- -Duse-ilp64=true -Ddisable-optimization=true -Dallow-noblas=false - name: Test run: spin test -- numpy/linalg @@ -150,7 +159,7 @@ jobs: run: | dnf install git gcc-gfortran g++ python3-devel flexiblas-devel -y - - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: submodules: recursive fetch-depth: 0 @@ -161,7 +170,7 @@ jobs: pip install pytest hypothesis typing_extensions - name: Build - run: spin build -- -Ddisable-optimization=true + run: spin build -- -Ddisable-optimization=true -Dallow-noblas=false - name: Test run: spin test -- numpy/linalg @@ -169,7 +178,7 @@ jobs: - name: Build (ILP64) run: | rm -rf build - spin build -- -Ddisable-optimization=true -Duse-ilp64=true + spin build -- -Ddisable-optimization=true -Duse-ilp64=true -Dallow-noblas=false - name: Test (ILP64) run: spin test -- numpy/linalg @@ -180,11 +189,11 @@ jobs: runs-on: ubuntu-latest name: "OpenBLAS with CMake" steps: - - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: submodules: recursive fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 with: python-version: '3.11' @@ -196,22 +205,22 @@ jobs: sudo apt-get remove pkg-config - name: Build - run: spin build -- -Ddisable-optimization=true + run: spin build -- -Ddisable-optimization=true -Dallow-noblas=false - name: Test run: spin test -j auto -- numpy/linalg - netlib: + netlib-debian: if: "github.repository == 'numpy/numpy'" runs-on: ubuntu-latest - name: "Netlib BLAS/LAPACK" + name: "Debian libblas/liblapack" steps: - - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: submodules: recursive fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 with: python-version: '3.11' @@ -222,7 +231,7 @@ jobs: - name: Build run: | - spin build -- -Ddisable-optimization=true + spin build -- -Ddisable-optimization=true -Dallow-noblas=false - name: Test run: | @@ -230,16 +239,47 @@ jobs: spin test -j auto -- numpy/linalg + netlib-split: + if: "github.repository == 'numpy/numpy'" + runs-on: ubuntu-latest + container: opensuse/tumbleweed + name: "OpenSUSE Netlib BLAS/LAPACK" + steps: + - name: Install system dependencies + run: | + # No blas.pc on OpenSUSE as of Nov 2023, so no need to install pkg-config. + # If it is needed in the future, use install name `pkgconf-pkg-config` + zypper install -y git gcc-c++ python3-pip python3-devel blas cblas lapack + + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + submodules: recursive + fetch-depth: 0 + + - name: Install PyPI dependencies + run: | + pip install --break-system-packages -r build_requirements.txt + + - name: Build + run: | + spin build -- -Dblas=blas -Dlapack=lapack -Ddisable-optimization=true -Dallow-noblas=false + + - name: Test + run: | + pip install --break-system-packages pytest pytest-xdist hypothesis typing_extensions + spin test -j auto -- numpy/linalg + + mkl: if: "github.repository == 'numpy/numpy'" runs-on: ubuntu-latest name: "MKL (LP64, ILP64, SDL)" steps: - - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: submodules: recursive fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 with: python-version: '3.11' @@ -267,7 +307,7 @@ jobs: - name: Build with defaults (LP64) run: | pkg-config --libs mkl-dynamic-lp64-seq # check link flags - spin build -- -Ddisable-optimization=true + spin build -- -Ddisable-optimization=true -Dallow-noblas=false - name: Test run: spin test -- numpy/linalg @@ -276,7 +316,7 @@ jobs: run: | git clean -xdf > /dev/null pkg-config --libs mkl-dynamic-ilp64-seq - spin build -- -Duse-ilp64=true -Ddisable-optimization=true + spin build -- -Duse-ilp64=true -Ddisable-optimization=true -Dallow-noblas=false - name: Test run: spin test -- numpy/linalg @@ -288,7 +328,7 @@ jobs: rm mkl*.pc popd export MKLROOT=$Python3_ROOT_DIR - spin build -- -Ddisable-optimization=true + spin build -- -Ddisable-optimization=true -Dallow-noblas=false - name: Test run: spin test -- numpy/linalg @@ -298,11 +338,11 @@ jobs: runs-on: ubuntu-latest name: "BLIS" steps: - - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: submodules: recursive fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 with: python-version: '3.11' @@ -323,7 +363,7 @@ jobs: pkg-config --cflags blis - name: Build - run: spin build -- -Dblas=blis -Ddisable-optimization=true + run: spin build -- -Dblas=blis -Ddisable-optimization=true -Dallow-noblas=false - name: Test run: spin test -- numpy/linalg @@ -333,11 +373,11 @@ jobs: runs-on: ubuntu-latest name: "ATLAS" steps: - - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: submodules: recursive fetch-depth: 0 - - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 with: python-version: '3.11' @@ -348,7 +388,7 @@ jobs: sudo apt-get install libatlas-base-dev pkg-config - name: Build - run: spin build -- -Dblas=blas-atlas -Dlapack=lapack-atlas -Ddisable-optimization=true + run: spin build -- -Dblas=blas-atlas -Dlapack=lapack-atlas -Ddisable-optimization=true -Dallow-noblas=false - name: Test run: spin test -- numpy/linalg diff --git a/.github/workflows/linux_qemu.yml b/.github/workflows/linux_qemu.yml new file mode 100644 index 000000000000..31ceab851553 --- /dev/null +++ b/.github/workflows/linux_qemu.yml @@ -0,0 +1,157 @@ +# Meson's Python module doesn't support crosscompiling, +# and python dependencies may be another potential hurdle. +# There might also be a need to run runtime tests during configure time. +# +# The recommended practice is to rely on Docker to provide the x86_64 crosscompile toolchain, +# enabling native execution via binfmt. +# +# In simpler terms, everything except the crosscompile toolchain will be emulated. + +name: Linux Qemu tests + +on: + pull_request: + branches: + - main + - maintenance/** + +defaults: + run: + shell: bash + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + linux_qemu: + if: "github.repository == 'numpy/numpy'" + runs-on: ubuntu-22.04 + continue-on-error: true + strategy: + matrix: + BUILD_PROP: + - [ + "armhf", + "arm-linux-gnueabihf", + "arm32v7/ubuntu:22.04", + "-Dallow-noblas=true", + # test_unary_spurious_fpexception is currently skipped + # FIXME(@seiko2plus): Requires confirmation for the following issue: + # The presence of an FP invalid exception caused by sqrt. Unsure if this is a qemu bug or not. + "(test_kind or test_multiarray or test_simd or test_umath or test_ufunc) and not test_unary_spurious_fpexception" + ] + - [ + "ppc64le", + "powerpc64le-linux-gnu", + "ppc64le/ubuntu:22.04", + "-Dallow-noblas=true", + "test_kind or test_multiarray or test_simd or test_umath or test_ufunc", + ] + - [ + "ppc64le - baseline(Power9)", + "powerpc64le-linux-gnu", + "ppc64le/ubuntu:22.04", + "-Dallow-noblas=true -Dcpu-baseline=vsx3", + "test_kind or test_multiarray or test_simd or test_umath or test_ufunc", + ] + - [ + "s390x", + "s390x-linux-gnu", + "s390x/ubuntu:22.04", + "-Dallow-noblas=true", + # Skipping TestRationalFunctions.test_gcd_overflow test + # because of a possible qemu bug that appears to be related to int64 overflow in absolute operation. + # TODO(@seiko2plus): Confirm the bug and provide a minimal reproducer, then report it to upstream. + "(test_kind or test_multiarray or test_simd or test_umath or test_ufunc) and not test_gcd_overflow" + ] + - [ + "s390x - baseline(Z13)", + "s390x-linux-gnu", + "s390x/ubuntu:22.04", + "-Dallow-noblas=true -Dcpu-baseline=vx", + "(test_kind or test_multiarray or test_simd or test_umath or test_ufunc) and not test_gcd_overflow" + ] + env: + TOOLCHAIN_NAME: ${{ matrix.BUILD_PROP[1] }} + DOCKER_CONTAINER: ${{ matrix.BUILD_PROP[2] }} + MESON_OPTIONS: ${{ matrix.BUILD_PROP[3] }} + RUNTIME_TEST_FILTER: ${{ matrix.BUILD_PROP[4] }} + TERM: xterm-256color + + name: "${{ matrix.BUILD_PROP[0] }}" + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + submodules: recursive + fetch-depth: 0 + + - name: Initialize binfmt_misc for qemu-user-static + run: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + - name: Install GCC cross-compilers + run: | + sudo apt update + sudo apt install -y ninja-build gcc-${TOOLCHAIN_NAME} g++-${TOOLCHAIN_NAME} gfortran-${TOOLCHAIN_NAME} + + - name: Cache docker container + uses: actions/cache@v3 + id: container-cache + with: + path: ~/docker_${{ matrix.BUILD_PROP[1] }} + key: container-${{ runner.os }}-${{ matrix.BUILD_PROP[1] }}-${{ matrix.BUILD_PROP[2] }}-${{ hashFiles('build_requirements.txt') }} + + - name: Creates new container + if: steps.container-cache.outputs.cache-hit != 'true' + run: | + docker run --name the_container --interactive -v /:/host -v $(pwd):/numpy ${DOCKER_CONTAINER} /bin/bash -c " + apt update && + apt install -y cmake git python3 python-is-python3 python3-dev python3-pip && + mkdir -p /lib64 && ln -s /host/lib64/ld-* /lib64/ && + ln -s /host/lib/x86_64-linux-gnu /lib/x86_64-linux-gnu && + rm -rf /usr/${TOOLCHAIN_NAME} && ln -s /host/usr/${TOOLCHAIN_NAME} /usr/${TOOLCHAIN_NAME} && + rm -rf /usr/lib/gcc/${TOOLCHAIN_NAME} && ln -s /host/usr/lib/gcc-cross/${TOOLCHAIN_NAME} /usr/lib/gcc/${TOOLCHAIN_NAME} && + rm -f /usr/bin/gcc && ln -s /host/usr/bin/${TOOLCHAIN_NAME}-gcc /usr/bin/gcc && + rm -f /usr/bin/g++ && ln -s /host/usr/bin/${TOOLCHAIN_NAME}-g++ /usr/bin/g++ && + rm -f /usr/bin/gfortran && ln -s /host/usr/bin/${TOOLCHAIN_NAME}-gfortran /usr/bin/gfortran && + rm -f /usr/bin/ar && ln -s /host/usr/bin/${TOOLCHAIN_NAME}-ar /usr/bin/ar && + rm -f /usr/bin/as && ln -s /host/usr/bin/${TOOLCHAIN_NAME}-as /usr/bin/as && + rm -f /usr/bin/ld && ln -s /host/usr/bin/${TOOLCHAIN_NAME}-ld /usr/bin/ld && + rm -f /usr/bin/ld.bfd && ln -s /host/usr/bin/${TOOLCHAIN_NAME}-ld.bfd /usr/bin/ld.bfd && + rm -f /usr/bin/ninja && ln -s /host/usr/bin/ninja /usr/bin/ninja && + git config --global --add safe.directory /numpy && + python -m pip install -r /numpy/build_requirements.txt && + python -m pip install pytest pytest-xdist hypothesis typing_extensions && + rm -f /usr/local/bin/ninja && mkdir -p /usr/local/bin && ln -s /host/usr/bin/ninja /usr/local/bin/ninja + " + docker commit the_container the_container + mkdir -p "~/docker_${TOOLCHAIN_NAME}" + docker save -o "~/docker_${TOOLCHAIN_NAME}/the_container.tar" the_container + + - name: Load container from cache + if: steps.container-cache.outputs.cache-hit == 'true' + run: docker load -i "~/docker_${TOOLCHAIN_NAME}/the_container.tar" + + - name: Meson Build + run: | + docker run --rm -e "TERM=xterm-256color" -v $(pwd):/numpy -v /:/host the_container \ + /bin/script -e -q -c "/bin/bash --noprofile --norc -eo pipefail -c ' + cd /numpy && spin build --clean -- ${MESON_OPTIONS} + '" + + - name: Meson Log + if: always() + run: 'cat build/meson-logs/meson-log.txt' + + - name: Run Tests + run: | + docker run --rm -e "TERM=xterm-256color" -v $(pwd):/numpy -v /:/host the_container \ + /bin/script -e -q -c "/bin/bash --noprofile --norc -eo pipefail -c ' + export F90=/usr/bin/gfortran + cd /numpy && spin test -- -k \"${RUNTIME_TEST_FILTER}\" + '" + diff --git a/.github/workflows/linux_simd.yml b/.github/workflows/linux_simd.yml new file mode 100644 index 000000000000..f8b694124d41 --- /dev/null +++ b/.github/workflows/linux_simd.yml @@ -0,0 +1,212 @@ +name: Linux SIMD tests + +# This file is meant for testing different SIMD-related build options and +# optimization levels. See `meson_options.txt` for the available build options. +# +# Jobs and their purposes: +# +# - baseline_only: +# Focuses on completing as quickly as possible and acts as a filter for other, more resource-intensive jobs. +# Utilizes only the default baseline targets (e.g., SSE3 on X86_64) without enabling any runtime dispatched features. +# +# - old_gcc: +# Tests the oldest supported GCC version with default CPU/baseline/dispatch settings. +# +# - without_optimizations: +# Completely disables all SIMD optimizations and other compiler optimizations such as loop unrolling. +# +# - native: +# Tests against the host CPU features set as the baseline without enabling any runtime dispatched features. +# Intended to assess the entire NumPy codebase against host flags, even for code sections lacking handwritten SIMD intrincis. +# +# - without_avx512/avx2/fma3: +# Uses runtime SIMD dispatching but disables AVX2, FMA3, and AVX512. +# Intended to evaluate 128-bit SIMD extensions without FMA support. +# +# - without_avx512: +# Uses runtime SIMD dispatching but disables AVX512. +# Intended to evaluate 128-bit/256-bit SIMD extensions. +# +# - intel_sde: +# Executes only the SIMD tests for various AVX512 SIMD extensions under the Intel Software Development Emulator (SDE). +# +on: + pull_request: + branches: + - main + - maintenance/** + +defaults: + run: + shell: 'script -q -e -c "bash --noprofile --norc -eo pipefail {0}"' + +env: + TERM: xterm-256color + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +jobs: + baseline_only: + if: "github.repository == 'numpy/numpy'" + runs-on: ubuntu-latest + env: + MESON_ARGS: "-Dallow-noblas=true -Dcpu-dispatch=none" + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 + with: + python-version: '3.9' + - uses: ./.github/meson_actions + name: Build/Test + + old_gcc: + if: github.event_name != 'push' + needs: [baseline_only] + runs-on: ubuntu-latest + env: + MESON_ARGS: "-Dallow-noblas=true" + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 + with: + python-version: '3.9' + + - name: Install GCC/8/9 + run: | + echo "deb http://archive.ubuntu.com/ubuntu focal main universe" | sudo tee /etc/apt/sources.list.d/focal.list + sudo apt update + sudo apt install -y g++-8 g++-9 + + - name: Enable gcc-8 + run: | + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 1 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 1 + + - uses: ./.github/meson_actions + name: Build/Test against gcc-8 + + - name: Enable gcc-9 + run: | + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 2 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 2 + + - uses: ./.github/meson_actions + name: Build/Test against gcc-9 + + specialize: + needs: [baseline_only] + runs-on: ubuntu-latest + if: github.event_name != 'push' + continue-on-error: true + strategy: + matrix: + BUILD_PROP: + #- [ + #"without optimizations", + #"-Dallow-noblas=true -Ddisable-optimization=true", + #"3.12" + #] + - [ + "native", + "-Dallow-noblas=true -Dcpu-baseline=native -Dcpu-dispatch=none", + "3.11" + ] + - [ + "without avx512", + "-Dallow-noblas=true -Dcpu-dispatch=SSSE3,SSE41,POPCNT,SSE42,AVX,F16C,AVX2,FMA3", + "3.10" + ] + - [ + "without avx512/avx2/fma3", + "-Dallow-noblas=true -Dcpu-dispatch=SSSE3,SSE41,POPCNT,SSE42,AVX,F16C", + "3.9" + ] + + env: + MESON_ARGS: ${{ matrix.BUILD_PROP[1] }} + + name: "${{ matrix.BUILD_PROP[0] }}" + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 + with: + python-version: "${{ matrix.BUILD_PROP[2] }}" + - uses: ./.github/meson_actions + name: Build/Test + + intel_sde: + needs: [baseline_only] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 + with: + python-version: '3.11' + + - name: Install Intel SDE + run: | + curl -o /tmp/sde.tar.xz https://downloadmirror.intel.com/784319/sde-external-9.24.0-2023-07-13-lin.tar.xz + mkdir /tmp/sde && tar -xvf /tmp/sde.tar.xz -C /tmp/sde/ + sudo mv /tmp/sde/* /opt/sde && sudo ln -s /opt/sde/sde64 /usr/bin/sde + + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y g++-13 + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 1 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 1 + python -m pip install -r build_requirements.txt + python -m pip install pytest pytest-xdist hypothesis typing_extensions + + - name: Build + run: spin build -- -Dallow-noblas=true -Dcpu-baseline=avx512f -Dtest-simd='BASELINE,AVX512_KNL,AVX512_KNM,AVX512_SKX,AVX512_CLX,AVX512_CNL,AVX512_ICL,AVX512_SPR' + + - name: Meson Log + if: always() + run: cat build/meson-logs/meson-log.txt + + - name: SIMD tests (KNM) + run: | + export NUMPY_SITE=$(realpath build-install/usr/lib/python*/site-packages/) + export PYTHONPATH="$PYTHONPATH:$NUMPY_SITE" + cd build-install && + sde -knm -- python -c "import numpy; numpy.show_config()" && + sde -knm -- python -m pytest $NUMPY_SITE/numpy/core/tests/test_simd* + + - name: SIMD tests (SPR) + run: | + export NUMPY_SITE=$(realpath build-install/usr/lib/python*/site-packages/) + export PYTHONPATH="$PYTHONPATH:$NUMPY_SITE" + cd build-install && + sde -spr -- python -c "import numpy; numpy.show_config()" && + sde -spr -- python -m pytest $NUMPY_SITE/numpy/core/tests/test_simd* + + # Can't run on SDE just yet: see https://github.com/numpy/numpy/issues/23545#issuecomment-1659047365 + # + #- name: linalg/ufunc/umath tests (SPR) + # run: | + # export NUMPY_SITE=$(realpath build-install/usr/lib/python*/site-packages/) + # export PYTHONPATH="$PYTHONPATH:$NUMPY_SITE" + # cd build-install && + # sde -spr -- python -c "import numpy; numpy.show_config()" && + # sde -spr -- python -m pytest $NUMPY_SITE/numpy/core/tests/test_umath* \ + # $NUMPY_SITE/numpy/core/tests/test_ufunc.py \ + # $NUMPY_SITE/numpy/linalg/tests/test_* + diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index b34c86bec901..959b51e0626b 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -84,7 +84,7 @@ jobs: shell: bash -l {0} run: | conda activate numpy-dev - CC="ccache $CC" spin build -j2 + CC="ccache $CC" spin build -j2 -- -Dallow-noblas=false - name: Run test suite (full) shell: bash -l {0} @@ -123,14 +123,14 @@ jobs: pip install pytest pytest-xdist hypothesis - name: Build against Accelerate (LP64) - run: spin build -- -Ddisable-optimization=true + run: spin build -- -Ddisable-optimization=true -Dallow-noblas=false - name: Test (linalg only) run: spin test -j2 -- numpy/linalg - name: Build NumPy against Accelerate (ILP64) run: | - spin build -- -Duse-ilp64=true + spin build -- -Duse-ilp64=true -Dallow-noblas=false - name: Test (fast tests) run: spin test -j2 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml new file mode 100644 index 000000000000..c89b8e11ec95 --- /dev/null +++ b/.github/workflows/mypy.yml @@ -0,0 +1,67 @@ +name: Run MyPy + +# Mypy is too slow to run as part of regular CI. The purpose of the jobs in +# this file is to cover running Mypy across: +# +# - OSes: Linux, Windows and macOS +# - Python versions: lowest/highest supported versions, and an intermediate one +# +# The build matrix aims for sparse coverage across those two dimensions. +# Use of BLAS/LAPACK and SIMD is disabled on purpose, because those things +# don't matter for static typing and this speeds up the builds. +# +# This is a separate job file so it's easy to trigger by hand. + +on: + pull_request: + branches: + - main + - maintenance/** + paths-ignore: + - 'benchmarks/' + - '.circlecl/' + - 'docs/' + - 'meson_cpu/' + - 'tools/' + workflow_dispatch: + +defaults: + run: + shell: bash + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +jobs: + mypy: + if: "github.repository == 'numpy/numpy'" + name: "MyPy" + runs-on: ${{ matrix.os_python[0] }} + strategy: + matrix: + os_python: + - [ubuntu-latest, '3.10'] # switch to 3.12-dev after mypy is upgraded (see gh-23764) + - [windows-2019, '3.11'] + - [macos-12, '3.9'] + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + with: + python-version: ${{ matrix.os_python[1] }} + - name: Install dependencies + run: | + pip install -r build_requirements.txt + pip install -r test_requirements.txt + - name: Build + run: | + spin build -j2 -- -Dallow-noblas=true -Ddisable-optimization=true --vsenv + - name: Run Mypy + run: | + spin mypy diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index fbde86092b92..988a55b761db 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -23,13 +23,13 @@ jobs: compiler: ["MSVC", "Clang-cl"] steps: - name: Checkout - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: submodules: recursive fetch-depth: 0 - name: Setup Python - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 with: python-version: '3.11' @@ -37,16 +37,9 @@ jobs: run: | python -m pip install spin Cython - - name: Install OpenBLAS (MacPython build) + - name: Install pkg-config run: | - # Download and install pre-built OpenBLAS library with 32-bit - # interfaces. Unpack it in the pkg-config hardcoded path - choco install unzip -y - choco install wget -y choco install -y --checksum 6004DF17818F5A6DBF19CB335CC92702 pkgconfiglite - wget https://anaconda.org/multibuild-wheels-staging/openblas-libs/v0.3.21/download/openblas-v0.3.21-win_amd64-gcc_10_3_0.zip - unzip -d c:\opt openblas-v0.3.21-win_amd64-gcc_10_3_0.zip - echo "PKG_CONFIG_PATH=c:\opt\64\lib\pkgconfig;" >> $env:GITHUB_ENV - name: Install Clang-cl if: matrix.compiler == 'Clang-cl' @@ -55,26 +48,26 @@ jobs: - name: Install NumPy (MSVC) if: matrix.compiler == 'MSVC' + env: + PKG_CONFIG_PATH: ${{ github.workspace }}/.openblas run: | - spin build -j2 -- --vsenv + python -m pip install scipy-openblas32 + spin build --with-scipy-openblas=32 -j2 -- --vsenv - name: Install NumPy (Clang-cl) if: matrix.compiler == 'Clang-cl' + env: + PKG_CONFIG_PATH: ${{ github.workspace }}/.openblas run: | "[binaries]","c = 'clang-cl'","cpp = 'clang-cl'","ar = 'llvm-lib'","c_ld = 'lld-link'","cpp_ld = 'lld-link'" | Out-File $PWD/clang-cl-build.ini -Encoding ascii - spin build -j2 -- --vsenv --native-file=$PWD/clang-cl-build.ini + python -m pip install scipy-openblas32 + spin build --with-scipy-openblas=32 -j2 -- --vsenv --native-file=$PWD/clang-cl-build.ini - - name: Copy OpenBLAS DLL, write _distributor_init.py + - name: Meson Log + shell: bash + if: ${{ failure() }} run: | - # Getting the OpenBLAS DLL to the right place so it loads - $installed_path = "$PWD\build-install\usr\Lib\site-packages" - $numpy_path = "${installed_path}\numpy" - $libs_path = "${installed_path}\numpy.libs" - mkdir ${libs_path} - $ob_path = "C:/opt/64/bin/" - cp $ob_path/*.dll $libs_path - # Write _distributor_init.py to load .libs DLLs. - python -c "from tools import openblas_support; openblas_support.make_init(r'${numpy_path}')" + cat build/meson-logs/meson-log.txt - name: Install test dependencies run: | @@ -85,19 +78,19 @@ jobs: run: | spin test - msvc_32bit_python_openblas: + msvc_32bit_python_no_openblas: name: MSVC, 32-bit Python, no BLAS runs-on: windows-2019 if: "github.repository == 'numpy/numpy'" steps: - name: Checkout - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: submodules: recursive fetch-depth: 0 - name: Setup Python (32-bit) - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0 + uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 with: python-version: '3.10' architecture: 'x86' diff --git a/.mailmap b/.mailmap index c34bd81bc90a..21f4af76ecc3 100644 --- a/.mailmap +++ b/.mailmap @@ -29,7 +29,6 @@ @luzpaz @luzpaz @partev -@pierreloicq @pkubaj @pmvz @pratiklp00 diff --git a/.spin/cmds.py b/.spin/cmds.py index 09556e67bbce..ea44329b1346 100644 --- a/.spin/cmds.py +++ b/.spin/cmds.py @@ -7,6 +7,8 @@ import shutil import json import pathlib +import importlib +import subprocess import click from spin import util @@ -23,6 +25,61 @@ ) +def _get_numpy_tools(filename): + filepath = pathlib.Path('tools', filename) + spec = importlib.util.spec_from_file_location(filename.stem, filepath) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +@click.command() +@click.option( + "-t", "--token", + help="GitHub access token", + required=True +) +@click.option( + "--revision-range", + help="..", + required=True +) +@click.pass_context +def changelog(ctx, token, revision_range): + """👩 Get change log for provided revision range + + \b + Example: + + \b + $ spin authors -t $GH_TOKEN --revision-range v1.25.0..v1.26.0 + """ + try: + from github.GithubException import GithubException + from git.exc import GitError + changelog = _get_numpy_tools(pathlib.Path('changelog.py')) + except ModuleNotFoundError as e: + raise click.ClickException( + f"{e.msg}. Install the missing packages to use this command." + ) + click.secho( + f"Generating change log for range {revision_range}", + bold=True, fg="bright_green", + ) + try: + changelog.main(token, revision_range) + except GithubException as e: + raise click.ClickException( + f"GithubException raised with status: {e.status} " + f"and message: {e.data['message']}" + ) + except GitError as e: + raise click.ClickException( + f"Git error in command `{' '.join(e.command)}` " + f"with error message: {e.stderr}" + ) + + @click.command() @click.option( "-j", "--jobs", @@ -263,6 +320,47 @@ def _run_asv(cmd): util.run(cmd, cwd='benchmarks', env=env) +@click.command() +@click.option( + "-b", "--branch", + metavar='branch', + default="main", +) +@click.option( + '--uncommitted', + is_flag=True, + default=False, + required=False, +) +@click.pass_context +def lint(ctx, branch, uncommitted): + """🔦 Run lint checks on diffs. + Provide target branch name or `uncommitted` to check changes before committing: + + \b + Examples: + + \b + For lint checks of your development brach with `main` or a custom branch: + + \b + $ spin lint # defaults to main + $ spin lint --branch custom_branch + + \b + To check just the uncommitted changes before committing + + \b + $ spin lint --uncommitted + """ + try: + linter = _get_numpy_tools(pathlib.Path('linter.py')) + except ModuleNotFoundError as e: + raise click.ClickException( + f"{e.msg}. Install using linter_requirements.txt" + ) + + linter.DiffLinter(branch).run_lint(uncommitted) @click.command() @click.option( @@ -470,3 +568,78 @@ def _config_openblas(blas_variant): os.makedirs(openblas_dir, exist_ok=True) with open(pkg_config_fname, "wt", encoding="utf8") as fid: fid.write(openblas.get_pkg_config().replace("\\", "/")) + + +@click.command() +@click.option( + "-v", "--version-override", + help="NumPy version of release", + required=False +) +@click.pass_context +def notes(ctx, version_override): + """🎉 Generate release notes and validate + + \b + Example: + + \b + $ spin notes --version-override 2.0 + + \b + To automatically pick the version + + \b + $ spin notes + """ + project_config = util.get_config() + version = version_override or project_config['project.version'] + + click.secho( + f"Generating release notes for NumPy {version}", + bold=True, fg="bright_green", + ) + + # Check if `towncrier` is installed + if not shutil.which("towncrier"): + raise click.ClickException( + f"please install `towncrier` to use this command" + ) + + click.secho( + f"Reading upcoming changes from {project_config['tool.towncrier.directory']}", + bold=True, fg="bright_yellow" + ) + # towncrier build --version 2.1 --yes + cmd = ["towncrier", "build", "--version", version, "--yes"] + try: + p = util.run( + cmd=cmd, + sys_exit=False, + output=True, + encoding="utf-8" + ) + except subprocess.SubprocessError as e: + raise click.ClickException( + f"`towncrier` failed returned {e.returncode} with error `{e.stderr}`" + ) + + output_path = project_config['tool.towncrier.filename'].format(version=version) + click.secho( + f"Release notes successfully written to {output_path}", + bold=True, fg="bright_yellow" + ) + + click.secho( + "Verifying consumption of all news fragments", + bold=True, fg="bright_green", + ) + + try: + test_notes = _get_numpy_tools(pathlib.Path('ci', 'test_all_newsfragments_used.py')) + except ModuleNotFoundError as e: + raise click.ClickException( + f"{e.msg}. Install the missing packages to use this command." + ) + + test_notes.main() diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 72845eefac09..000000000000 --- a/.travis.yml +++ /dev/null @@ -1,59 +0,0 @@ -# After changing this file, check it on: -# http://lint.travis-ci.org/ -language: python -group: travis_latest -os: linux -dist: focal - -# Travis allows these packages, additions can be requested -# https://github.com/travis-ci/apt-package-safelist -addons: - apt: - packages: &common_packages - - gfortran - - libgfortran5 - - libatlas-base-dev - # Speedup builds, particularly when USE_CHROOT=1 - - eatmydata - -# Disable clone depth -git: - depth: false - -cache: - directories: - - $HOME/.cache/pip - -jobs: - include: -# The ppc64le for these tests is usually missing, resulting in -# test failure most of the time. Let's not do that. -# - python: "3.9" -# os: linux -# arch: ppc64le -# env: -# # use OpenBLAS build, not system ATLAS -# - DOWNLOAD_OPENBLAS=1 -# # - NPY_USE_BLAS_ILP64=1 # the openblas build fails -# - ATLAS=None -# # VSX4 still not supported by ubuntu/gcc-11 -# - EXPECT_CPU_FEATURES="VSX VSX2 VSX3" - - - python: "3.9" - os: linux - arch: s390x - # fixes VX assembler ambiguous errors - # due to compiler incompatibility - install: sudo apt update && sudo apt -y --only-upgrade install binutils - env: - # use OpenBLAS build, not system ATLAS - - DOWNLOAD_OPENBLAS=1 - - NPY_USE_BLAS_ILP64=1 - - ATLAS=None - - EXPECT_CPU_FEATURES="VX VXE VXE2" - -before_install: - - ./tools/travis-before-install.sh - -script: - - ./tools/travis-test.sh diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d14e73b27edc..ce78d07777fa 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -84,126 +84,6 @@ stages: python3 -m pytest --pyargs numpy" displayName: 'Run 32-bit manylinux2014 Docker Build / Tests' - - - job: macOS - pool: - vmImage: 'macOS-11' - strategy: - maxParallel: 3 - matrix: - Python39: - PYTHON_VERSION: '3.9' - USE_OPENBLAS: '1' - Python39-ILP64: - PYTHON_VERSION: '3.9' - NPY_USE_BLAS_ILP64: '1' - USE_OPENBLAS: '1' - steps: - - script: | - git submodule update --init - displayName: 'Fetch submodules' - # the @0 refers to the (major) version of the *task* on Microsoft's - # end, not the order in the build matrix nor anything to do - # with version of Python selected - - task: UsePythonVersion@0 - inputs: - versionSpec: $(PYTHON_VERSION) - addToPath: true - architecture: 'x64' - - script: | - set -xe - [ -n "$USE_XCODE_10" ] && /bin/bash -c "sudo xcode-select -s /Applications/Xcode_10.app/Contents/Developer" - clang --version - displayName: 'report clang version' - - - script: | - if [[ $PLATFORM == "macosx-arm64" ]]; then - PLAT="arm64" - fi - source tools/wheels/gfortran_utils.sh - install_gfortran - displayName: 'install gfortran' - # use the pre-built openblas binary that most closely - # matches our MacOS wheel builds -- currently based - # primarily on file size / name details - - script: | - set -xe - target=$(python tools/openblas_support.py) - ls -lR $target - # manually link to appropriate system paths - cp $target/lib/lib* /usr/local/lib/ - cp $target/include/* /usr/local/include/ - otool -L /usr/local/lib/libopenblas* - displayName: 'install pre-built openblas' - condition: eq(variables['USE_OPENBLAS'], '1') - - script: python -m pip install --upgrade pip 'setuptools<49.2.0' wheel - displayName: 'Install tools' - - script: | - python -m pip install -r test_requirements.txt - # Don't use doc_requirements.txt since that messes up tests - python -m pip install vulture sphinx==4.3.0 numpydoc==1.4.0 ninja - displayName: 'Install dependencies; some are optional to avoid test skips' - - script: /bin/bash -c "! vulture . --min-confidence 100 --exclude doc/,numpy/distutils/ | grep 'unreachable'" - displayName: 'Check for unreachable code paths in Python modules' - - - script: git submodule update --init - displayName: 'Fetch submodules' - - # prefer usage of clang over gcc proper - # to match likely scenario on many user mac machines - - script: python setup.py build -j 4 build_src --verbose-cfg install - displayName: 'Build NumPy' - env: - BLAS: None - LAPACK: None - ATLAS: None - CC: /usr/bin/clang - - # wait until after dev build of NumPy to pip - # install matplotlib to avoid pip install of older numpy - - script: python -m pip install matplotlib - displayName: 'Install matplotlib before refguide run' - - - script: python runtests.py -g --refguide-check - displayName: 'Run Refguide Check' - condition: eq(variables['USE_OPENBLAS'], '1') - - - script: | - echo LIBRARY_PATH ${LIBRARY_PATH} - python runtests.py -n --mode=full -- -rsx --junitxml=junit/test-results.xml - displayName: 'Run Full NumPy Test Suite' - condition: eq(variables['USE_OPENBLAS'], '1') - env: - # gfortran installed above adds -lSystem, so this is needed to find it (gh-22043) - LIBRARY_PATH: /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib - - - bash: | - python -m pip install threadpoolctl - python tools/openblas_support.py --check_version - displayName: 'Verify OpenBLAS version' - condition: eq(variables['USE_OPENBLAS'], '1') - - # import doesn't work when in numpy src directory , so do a pip dev install of build lib to test - - script: | - #!/bin/bash -v - set +e - python -c "import numpy as np" > test_output.log 2>&1 - check_output_code=$? - cat test_output.log - grep "buggy Accelerate backend" test_output.log - check_message=$? - if [ $check_output_code == 1 ] && [ $check_message == 0 ]; then exit 0; else exit 1;fi - displayName: "Check if numpy import fails with accelerate" - condition: eq(variables['USE_OPENBLAS'], '0') - - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-*.xml' - failTaskOnFailedTests: true - testRunTitle: 'Publish test results for Python 3.9 64-bit full Mac OS' - - - job: Windows pool: vmImage: 'windows-2019' @@ -232,35 +112,3 @@ stages: steps: - template: azure-steps-windows.yml - - - - job: Linux_conda - pool: - vmImage: 'ubuntu-20.04' - steps: - - script: | - git submodule update --init - displayName: 'Fetch submodules' - - script: | - # create and activate conda environment - conda env create -f environment.yml - displayName: 'Create conda environment.' - - script: | - # >>> conda initialize >>> - # !! Contents within this block are 'conda init' !! - # see https://github.com/conda/conda/issues/7980 - __conda_setup="$('conda' 'shell.bash' 'hook' 2> /dev/null)" - eval "$__conda_setup" - unset __conda_setup - # <<< conda initialize <<< - conda activate numpy-dev - # Run native baseline Build / Tests - python runtests.py --show-build-log --cpu-baseline=native --cpu-dispatch=none \ - --debug-info --mode=full -- -rsx --junitxml=junit/test-results.xml - displayName: 'Run native baseline Build / Tests in conda.' - - task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-*.xml' - failTaskOnFailedTests: true - testRunTitle: 'Publish test results for conda installation' diff --git a/doc/changelog/1.26.2-changelog.rst b/doc/changelog/1.26.2-changelog.rst new file mode 100644 index 000000000000..8715f2f0bb58 --- /dev/null +++ b/doc/changelog/1.26.2-changelog.rst @@ -0,0 +1,51 @@ + +Contributors +============ + +A total of 13 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* @stefan6419846 +* @thalassemia + +* Andrew Nelson +* Charles Bousseau + +* Charles Harris +* Marcel Bargull + +* Mark Mentovai + +* Matti Picus +* Nathan Goldbaum +* Ralf Gommers +* Sayed Adel +* Sebastian Berg +* William Ayd + + +Pull requests merged +==================== + +A total of 25 pull requests were merged for this release. + +* `#24814 `__: MAINT: align test_dispatcher s390x targets with _umath_tests_mtargets +* `#24929 `__: MAINT: prepare 1.26.x for further development +* `#24955 `__: ENH: Add Cython enumeration for NPY_FR_GENERIC +* `#24962 `__: REL: Remove Python upper version from the release branch +* `#24971 `__: BLD: Use the correct Python interpreter when running tempita.py +* `#24972 `__: MAINT: Remove unhelpful error replacements from ``import_array()`` +* `#24977 `__: BLD: use classic linker on macOS, the new one in XCode 15 has... +* `#25003 `__: BLD: musllinux_aarch64 [wheel build] +* `#25043 `__: MAINT: Update mailmap +* `#25049 `__: MAINT: Update meson build infrastructure. +* `#25071 `__: MAINT: Split up .github/workflows to match main +* `#25083 `__: BUG: Backport fix build on ppc64 when the baseline set to Power9... +* `#25093 `__: BLD: Fix features.h detection for Meson builds [1.26.x Backport] +* `#25095 `__: BUG: Avoid intp conversion regression in Cython 3 (backport) +* `#25107 `__: CI: remove obsolete jobs, and move macOS and conda Azure jobs... +* `#25108 `__: CI: Add linux_qemu action and remove travis testing. +* `#25112 `__: MAINT: Update .spin/cmds.py from main. +* `#25113 `__: DOC: Visually divide main license and bundled licenses in wheels +* `#25115 `__: MAINT: Add missing ``noexcept`` to shuffle helpers +* `#25116 `__: DOC: Fix license identifier for OpenBLAS +* `#25117 `__: BLD: improve detection of Netlib libblas/libcblas/liblapack +* `#25118 `__: MAINT: Make bitfield integers unsigned +* `#25119 `__: BUG: Make n a long int for np.random.multinomial +* `#25120 `__: BLD: change default of the ``allow-noblas`` option to true. +* `#25121 `__: BUG: ensure passing ``np.dtype`` to itself doesn't crash diff --git a/doc/source/release.rst b/doc/source/release.rst index 2c2947ee6685..0e0d477ce06b 100644 --- a/doc/source/release.rst +++ b/doc/source/release.rst @@ -5,6 +5,7 @@ Release notes .. toctree:: :maxdepth: 3 + 1.26.2 1.26.1 1.26.0 1.25.2 diff --git a/doc/source/release/1.26.2-notes.rst b/doc/source/release/1.26.2-notes.rst new file mode 100644 index 000000000000..6f32bcf48206 --- /dev/null +++ b/doc/source/release/1.26.2-notes.rst @@ -0,0 +1,64 @@ +.. currentmodule:: numpy + +========================== +NumPy 1.26.2 Release Notes +========================== + +NumPy 1.26.2 is a maintenance release that fixes bugs and regressions +discovered after the 1.26.1 release. The 1.26.release series is the last +planned minor release series before NumPy 2.0. The Python versions supported by +this release are 3.9-3.12. + + +Contributors +============ + +A total of 13 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* @stefan6419846 +* @thalassemia + +* Andrew Nelson +* Charles Bousseau + +* Charles Harris +* Marcel Bargull + +* Mark Mentovai + +* Matti Picus +* Nathan Goldbaum +* Ralf Gommers +* Sayed Adel +* Sebastian Berg +* William Ayd + + + +Pull requests merged +==================== + +A total of 25 pull requests were merged for this release. + +* `#24814 `__: MAINT: align test_dispatcher s390x targets with _umath_tests_mtargets +* `#24929 `__: MAINT: prepare 1.26.x for further development +* `#24955 `__: ENH: Add Cython enumeration for NPY_FR_GENERIC +* `#24962 `__: REL: Remove Python upper version from the release branch +* `#24971 `__: BLD: Use the correct Python interpreter when running tempita.py +* `#24972 `__: MAINT: Remove unhelpful error replacements from ``import_array()`` +* `#24977 `__: BLD: use classic linker on macOS, the new one in XCode 15 has... +* `#25003 `__: BLD: musllinux_aarch64 [wheel build] +* `#25043 `__: MAINT: Update mailmap +* `#25049 `__: MAINT: Update meson build infrastructure. +* `#25071 `__: MAINT: Split up .github/workflows to match main +* `#25083 `__: BUG: Backport fix build on ppc64 when the baseline set to Power9... +* `#25093 `__: BLD: Fix features.h detection for Meson builds [1.26.x Backport] +* `#25095 `__: BUG: Avoid intp conversion regression in Cython 3 (backport) +* `#25107 `__: CI: remove obsolete jobs, and move macOS and conda Azure jobs... +* `#25108 `__: CI: Add linux_qemu action and remove travis testing. +* `#25112 `__: MAINT: Update .spin/cmds.py from main. +* `#25113 `__: DOC: Visually divide main license and bundled licenses in wheels +* `#25115 `__: MAINT: Add missing ``noexcept`` to shuffle helpers +* `#25116 `__: DOC: Fix license identifier for OpenBLAS +* `#25117 `__: BLD: improve detection of Netlib libblas/libcblas/liblapack +* `#25118 `__: MAINT: Make bitfield integers unsigned +* `#25119 `__: BUG: Make n a long int for np.random.multinomial +* `#25120 `__: BLD: change default of the ``allow-noblas`` option to true. +* `#25121 `__: BUG: ensure passing ``np.dtype`` to itself doesn't crash + diff --git a/meson.build b/meson.build index 264160d132eb..a8cec6d104d4 100644 --- a/meson.build +++ b/meson.build @@ -82,5 +82,11 @@ if cc_id.startswith('clang') endif endif +if host_machine.system() == 'darwin' and cc.has_link_argument('-Wl,-ld_classic') + # New linker introduced in macOS 14 not working yet with at least OpenBLAS in Spack, + # see gh-24964 (and linked scipy issue from there). + add_project_link_arguments('-Wl,-ld_classic', language : ['c', 'cpp']) +endif + subdir('meson_cpu') subdir('numpy') diff --git a/meson_cpu/ppc64/meson.build b/meson_cpu/ppc64/meson.build index d14b23703fe3..bad95257ca95 100644 --- a/meson_cpu/ppc64/meson.build +++ b/meson_cpu/ppc64/meson.build @@ -26,11 +26,14 @@ VSX3 = mod_features.new( 'VSX3', 3, implies: VSX2, args: {'val': '-mcpu=power9', 'match': '.*[mcpu=|vsx].*'}, detect: {'val': 'VSX3', 'match': 'VSX.*'}, test_code: files(source_root + '/numpy/distutils/checks/cpu_vsx3.c')[0], + extra_tests: { + 'VSX3_HALF_DOUBLE': files(source_root + '/numpy/distutils/checks/extra_vsx3_half_double.c')[0] + } ) VSX4 = mod_features.new( 'VSX4', 4, implies: VSX3, args: {'val': '-mcpu=power10', 'match': '.*[mcpu=|vsx].*'}, detect: {'val': 'VSX4', 'match': 'VSX.*'}, - test_code: files(source_root + '/numpy/distutils/checks/cpu_vsx3.c')[0], + test_code: files(source_root + '/numpy/distutils/checks/cpu_vsx4.c')[0], extra_tests: { 'VSX4_MMA': files(source_root + '/numpy/distutils/checks/extra_vsx4_mma.c')[0] } diff --git a/meson_options.txt b/meson_options.txt index 05e9e733578e..0e8fd0b9d92d 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -2,7 +2,7 @@ option('blas', type: 'string', value: 'auto', description: 'Option for BLAS library selection. By default, try to find any in the order given by `blas-order`') option('lapack', type: 'string', value: 'auto', description: 'Option for LAPACK library selection. By default, try to find any in the order given by `lapack-order`') -option('allow-noblas', type: 'boolean', value: false, +option('allow-noblas', type: 'boolean', value: true, description: 'If set to true, allow building with (slow!) internal fallback routines') option('blas-order', type: 'array', value: ['mkl', 'accelerate', 'openblas', 'flexiblas', 'blis', 'blas']) diff --git a/numpy/__init__.cython-30.pxd b/numpy/__init__.cython-30.pxd index 0dd2fff2b87c..1409514f7a84 100644 --- a/numpy/__init__.cython-30.pxd +++ b/numpy/__init__.cython-30.pxd @@ -21,7 +21,7 @@ cdef extern from *: cdef extern from "Python.h": - ctypedef Py_ssize_t Py_intptr_t + ctypedef int Py_intptr_t cdef extern from "numpy/arrayobject.h": ctypedef Py_intptr_t npy_intp @@ -852,6 +852,7 @@ cdef extern from "numpy/arrayscalars.h": NPY_FR_ps NPY_FR_fs NPY_FR_as + NPY_FR_GENERIC # diff --git a/numpy/__init__.pxd b/numpy/__init__.pxd index 47d9294c1c4b..ca0a3a6c5288 100644 --- a/numpy/__init__.pxd +++ b/numpy/__init__.pxd @@ -810,6 +810,7 @@ cdef extern from "numpy/arrayscalars.h": NPY_FR_ps NPY_FR_fs NPY_FR_as + NPY_FR_GENERIC # diff --git a/numpy/_build_utils/tempita.py b/numpy/_build_utils/tempita.py old mode 100755 new mode 100644 diff --git a/numpy/core/code_generators/generate_numpy_api.py b/numpy/core/code_generators/generate_numpy_api.py index bfcb0d0e5444..ae38c4efc2e3 100644 --- a/numpy/core/code_generators/generate_numpy_api.py +++ b/numpy/core/code_generators/generate_numpy_api.py @@ -55,7 +55,6 @@ c_api = PyObject_GetAttrString(numpy, "_ARRAY_API"); Py_DECREF(numpy); if (c_api == NULL) { - PyErr_SetString(PyExc_AttributeError, "_ARRAY_API not found"); return -1; } diff --git a/numpy/core/config.h.in b/numpy/core/config.h.in index e3b559753521..cd63a59b2afc 100644 --- a/numpy/core/config.h.in +++ b/numpy/core/config.h.in @@ -12,6 +12,7 @@ #mesondefine HAVE___DECLSPEC_THREAD_ /* Optional headers */ +#mesondefine HAVE_FEATURES_H #mesondefine HAVE_XLOCALE_H #mesondefine HAVE_DLFCN_H #mesondefine HAVE_EXECINFO_H diff --git a/numpy/core/include/numpy/npy_common.h b/numpy/core/include/numpy/npy_common.h index fb976aa6ae09..9e98f8ef5edd 100644 --- a/numpy/core/include/numpy/npy_common.h +++ b/numpy/core/include/numpy/npy_common.h @@ -168,6 +168,9 @@ #define npy_ftell ftell #endif #include + #ifndef _WIN32 + #include + #endif #define npy_lseek lseek #define npy_off_t off_t diff --git a/numpy/core/meson.build b/numpy/core/meson.build index a1fc7e9d8a84..0823524d6a3f 100644 --- a/numpy/core/meson.build +++ b/numpy/core/meson.build @@ -257,37 +257,6 @@ foreach filefunc_maybe: optional_file_funcs endif endforeach -# Optional locale function -have_strtold_l = cc.has_function('strtold_l', include_directories: inc_curdir, - prefix:''' - #include - #include - #include "feature_detection_locale.h" -''') -if not have_strtold_l - # Retry with locale.h, seems to vary across Linux distros - have_strtold_l = cc.has_function('strtold_l', include_directories: inc_curdir, - prefix:''' - #include - #include - #include "feature_detection_locale.h" - ''') -endif -if have_strtold_l - cdata.set10('HAVE_STRTOLD_L', true) -else - # FIXME: this is wrong! the HAVE_ define should not exist, or it'll be - # interpreted as the function being available (true/false does nothing, see - # note on HAVE_ defines higher up). This is necessary though in order to make - # the Linux CI job pass. So either the check is wrong somehow, or this - # function is not available in CI. For the latter there is a fallback path, - # but that is broken because we don't have the exact long double - # representation checks. - if cc.get_argument_syntax() != 'msvc' - cdata.set10('HAVE_STRTOLD_L', false) - endif -endif - # Other optional functions optional_misc_funcs = [ 'backtrace', @@ -305,7 +274,7 @@ endforeach # SSE headers only enabled automatically on amd64/x32 builds optional_headers = [ 'features.h', # for glibc version linux - 'xlocale.h', # see GH#8367 + 'xlocale.h', # removed in glibc 2.26, but may still be useful - see gh-8367 'dlfcn.h', # dladdr 'execinfo.h', # backtrace 'libunwind.h', # backtrace for LLVM/Clang using libunwind @@ -317,6 +286,19 @@ foreach header: optional_headers endif endforeach +# Optional locale function - GNU-specific +_strtold_prefix = ''' +#define _GNU_SOURCE +#include +#include +''' +if cdata.get('HAVE_XLOCALE_H', 0) == 1 + _strtold_prefix += '#include ' +endif +if cc.has_function('strtold_l', include_directories: inc_curdir, prefix: _strtold_prefix) + cdata.set10('HAVE_STRTOLD_L', true) +endif + # Optional compiler attributes # TODO: this doesn't work with cc.has_function_attribute, see # https://github.com/mesonbuild/meson/issues/10732 diff --git a/numpy/core/src/_simd/_simd_inc.h.src b/numpy/core/src/_simd/_simd_inc.h.src index 887545414b9b..a023848831ed 100644 --- a/numpy/core/src/_simd/_simd_inc.h.src +++ b/numpy/core/src/_simd/_simd_inc.h.src @@ -105,19 +105,19 @@ typedef struct // type name compatible with python style const char *pyname; // returns '1' if the type represent a unsigned integer - int is_unsigned:1; + unsigned int is_unsigned:1; // returns '1' if the type represent a signed integer - int is_signed:1; + unsigned int is_signed:1; // returns '1' if the type represent a single or double precision - int is_float:1; + unsigned int is_float:1; // returns '1' if the type represent a boolean - int is_bool:1; + unsigned int is_bool:1; // returns '1' if the type represent a sequence - int is_sequence:1; + unsigned int is_sequence:1; // returns '1' if the type represent a scalar - int is_scalar:1; + unsigned int is_scalar:1; // returns '1' if the type represent a vector - int is_vector:1; + unsigned int is_vector:1; // returns the len of multi-vector if the type represent x2 or x3 vector // otherwise returns 0, e.g. returns 2 if data type is simd_data_vu8x2 int is_vectorx; diff --git a/numpy/core/src/common/half.hpp b/numpy/core/src/common/half.hpp index 4d16e3bcc1c2..ff9a547766d3 100644 --- a/numpy/core/src/common/half.hpp +++ b/numpy/core/src/common/half.hpp @@ -36,7 +36,7 @@ class Half final { #endif ) || ( std::is_same_v && - #if defined(NPY_HAVE_AVX512FP16) || defined(NPY_HAVE_VSX3) + #if defined(NPY_HAVE_AVX512FP16) || (defined(NPY_HAVE_VSX3) && defined(NPY_HAVE_VSX3_HALF_DOUBLE)) true #else false @@ -47,7 +47,7 @@ class Half final { /// Default constructor. initialize nothing. Half() = default; - /// Constract from float + /// Construct from float /// If there are no hardware optimization available, rounding will always /// be set to ties to even. explicit Half(float f) @@ -73,11 +73,8 @@ class Half final { #if defined(NPY_HAVE_AVX512FP16) __m128d md = _mm_load_sd(&f); bits_ = static_cast(_mm_cvtsi128_si32(_mm_castph_si128(_mm_cvtpd_ph(md)))); - #elif defined(NPY_HAVE_VSX3) && defined(NPY_HAVE_VSX_ASM) - __vector double vf64 = vec_splats(f); - __vector unsigned short vf16; - __asm__ __volatile__ ("xvcvdphp %x0,%x1" : "=wa" (vf16) : "wa" (vf64)); - bits_ = vec_extract(vf16, 0); + #elif defined(NPY_HAVE_VSX3) && defined(NPY_HAVE_VSX3_HALF_DOUBLE) + __asm__ __volatile__ ("xscvdphp %x0,%x1" : "=wa" (bits_) : "wa" (f)); #else bits_ = half_private::FromDoubleBits(BitCast(f)); #endif @@ -96,7 +93,7 @@ class Half final { __vector float vf32; __asm__ __volatile__("xvcvhpsp %x0,%x1" : "=wa"(vf32) - : "wa"(vec_splats(bits_.u))); + : "wa"(vec_splats(bits_))); return vec_extract(vf32, 0); #else return BitCast(half_private::ToFloatBits(bits_)); @@ -110,18 +107,18 @@ class Half final { double ret; _mm_store_sd(&ret, _mm_cvtph_pd(_mm_castsi128_ph(_mm_cvtsi32_si128(bits_)))); return ret; - #elif defined(NPY_HAVE_VSX3) && defined(NPY_HAVE_VSX_ASM) - __vector float vf64; - __asm__ __volatile__("xvcvhpdp %x0,%x1" - : "=wa"(vf32) - : "wa"(vec_splats(bits_))); - return vec_extract(vf64, 0); + #elif defined(NPY_HAVE_VSX3) && defined(NPY_HAVE_VSX3_HALF_DOUBLE) + double f64; + __asm__ __volatile__("xscvhpdp %x0,%x1" + : "=wa"(f64) + : "wa"(bits_)); + return f64; #else return BitCast(half_private::ToDoubleBits(bits_)); #endif } - /// Returns a new Half constracted from the IEEE 754 binary16. + /// Returns a new Half constructed from the IEEE 754 binary16. static constexpr Half FromBits(uint16_t bits) { Half h{}; @@ -134,7 +131,7 @@ class Half final { return bits_; } - /// @name Comparison operators (orderd) + /// @name Comparison operators (ordered) /// @{ constexpr bool operator==(Half r) const { @@ -158,7 +155,7 @@ class Half final { } /// @} - /// @name Comparison operators (unorderd) + /// @name Comparison operators (unordered) /// @{ constexpr bool operator!=(Half r) const { diff --git a/numpy/core/src/common/npy_config.h b/numpy/core/src/common/npy_config.h index 715b17777aed..e590366888aa 100644 --- a/numpy/core/src/common/npy_config.h +++ b/numpy/core/src/common/npy_config.h @@ -160,8 +160,29 @@ #undef HAVE_CACOSHL #endif /* __GLIBC_PREREQ(2, 18) */ -#endif /* defined(__GLIBC_PREREQ) */ +#else /* defined(__GLIBC) */ +/* musl linux?, see issue #25092 */ +#undef HAVE_CASIN +#undef HAVE_CASINF +#undef HAVE_CASINL +#undef HAVE_CASINH +#undef HAVE_CASINHF +#undef HAVE_CASINHL +#undef HAVE_CATAN +#undef HAVE_CATANF +#undef HAVE_CATANL +#undef HAVE_CATANH +#undef HAVE_CATANHF +#undef HAVE_CATANHL +#undef HAVE_CACOS +#undef HAVE_CACOSF +#undef HAVE_CACOSL +#undef HAVE_CACOSH +#undef HAVE_CACOSHF +#undef HAVE_CACOSHL + +#endif /* defined(__GLIBC) */ #endif /* defined(HAVE_FEATURES_H) */ #endif /* NUMPY_CORE_SRC_COMMON_NPY_CONFIG_H_ */ diff --git a/numpy/core/src/common/numpyos.c b/numpy/core/src/common/numpyos.c index 2fec06e1c564..19f6be0d47b7 100644 --- a/numpy/core/src/common/numpyos.c +++ b/numpy/core/src/common/numpyos.c @@ -11,6 +11,9 @@ #include "npy_pycompat.h" +#if defined(HAVE_STRTOLD_L) && !defined(_GNU_SOURCE) +# define _GNU_SOURCE +#endif #include #include diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index 68d398d64806..f8712d4d3bc3 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -1478,6 +1478,11 @@ PyArray_DTypeOrDescrConverterRequired(PyObject *obj, npy_dtype_info *dt_info) dt_info->descr = NULL; if (PyObject_TypeCheck(obj, &PyArrayDTypeMeta_Type)) { + if (obj == (PyObject *)&PyArrayDescr_Type) { + PyErr_SetString(PyExc_TypeError, + "Cannot convert np.dtype into a dtype."); + return NPY_FAIL; + } Py_INCREF(obj); dt_info->dtype = (PyArray_DTypeMeta *)obj; dt_info->descr = NULL; diff --git a/numpy/core/src/umath/_umath_tests.dispatch.c b/numpy/core/src/umath/_umath_tests.dispatch.c index 9d8df4c86d36..73a59103c1a0 100644 --- a/numpy/core/src/umath/_umath_tests.dispatch.c +++ b/numpy/core/src/umath/_umath_tests.dispatch.c @@ -5,6 +5,7 @@ * SSE2 SSE41 AVX2 * VSX VSX2 VSX3 * NEON ASIMD ASIMDHP + * VX VXE */ #define PY_SSIZE_T_CLEAN #include diff --git a/numpy/core/tests/examples/cython/checks.pyx b/numpy/core/tests/examples/cython/checks.pyx index e41c6d657351..c5529ee8fcaf 100644 --- a/numpy/core/tests/examples/cython/checks.pyx +++ b/numpy/core/tests/examples/cython/checks.pyx @@ -30,3 +30,6 @@ def get_dt64_unit(obj): def is_integer(obj): return isinstance(obj, (cnp.integer, int)) + +def conv_intp(cnp.intp_t val): + return val diff --git a/numpy/core/tests/test_cpu_dispatcher.py b/numpy/core/tests/test_cpu_dispatcher.py index 2f7eac7e8e90..41a60d5c395c 100644 --- a/numpy/core/tests/test_cpu_dispatcher.py +++ b/numpy/core/tests/test_cpu_dispatcher.py @@ -9,7 +9,8 @@ def test_dispatcher(): targets = ( "SSE2", "SSE41", "AVX2", "VSX", "VSX2", "VSX3", - "NEON", "ASIMD", "ASIMDHP" + "NEON", "ASIMD", "ASIMDHP", + "VX", "VXE" ) highest_sfx = "" # no suffix for the baseline all_sfx = [] diff --git a/numpy/core/tests/test_cython.py b/numpy/core/tests/test_cython.py index 29473f5ba424..99dd57e4c62d 100644 --- a/numpy/core/tests/test_cython.py +++ b/numpy/core/tests/test_cython.py @@ -122,3 +122,14 @@ def test_abstract_scalars(install_temp): assert checks.is_integer(1) assert checks.is_integer(np.int8(1)) assert checks.is_integer(np.uint64(1)) + +def test_conv_intp(install_temp): + import checks + + class myint: + def __int__(self): + return 3 + + # These conversion passes via `__int__`, not `__index__`: + assert checks.conv_intp(3.) == 3 + assert checks.conv_intp(myint()) == 3 diff --git a/numpy/core/tests/test_dtype.py b/numpy/core/tests/test_dtype.py index 81692015fc5e..ac155b67baf0 100644 --- a/numpy/core/tests/test_dtype.py +++ b/numpy/core/tests/test_dtype.py @@ -1898,3 +1898,9 @@ def test_result_type_integers_and_unitless_timedelta64(): td = np.timedelta64(4) result = np.result_type(0, td) assert_dtype_equal(result, td.dtype) + + +def test_creating_dtype_with_dtype_class_errors(): + # Regression test for #25031, calling `np.dtype` with itself segfaulted. + with pytest.raises(TypeError, match="Cannot convert np.dtype into a"): + np.array(np.ones(10), dtype=np.dtype) diff --git a/numpy/core/tests/test_half.py b/numpy/core/tests/test_half.py index ca849ad52ead..fbc1bf6a0a6d 100644 --- a/numpy/core/tests/test_half.py +++ b/numpy/core/tests/test_half.py @@ -21,8 +21,11 @@ def setup_method(self): # An array of all possible float16 values self.all_f16 = np.arange(0x10000, dtype=uint16) self.all_f16.dtype = float16 - self.all_f32 = np.array(self.all_f16, dtype=float32) - self.all_f64 = np.array(self.all_f16, dtype=float64) + + # NaN value can cause an invalid FP exception if HW is been used + with np.errstate(invalid='ignore'): + self.all_f32 = np.array(self.all_f16, dtype=float32) + self.all_f64 = np.array(self.all_f16, dtype=float64) # An array of all non-NaN float16 values, in sorted order self.nonan_f16 = np.concatenate( @@ -44,14 +47,19 @@ def test_half_conversions(self): # value is preserved when converting to/from other floats. # Convert from float32 back to float16 - b = np.array(self.all_f32, dtype=float16) - assert_equal(self.all_f16.view(dtype=uint16), - b.view(dtype=uint16)) + with np.errstate(invalid='ignore'): + b = np.array(self.all_f32, dtype=float16) + # avoid testing NaNs due to differ bits wither Q/SNaNs + b_nn = b == b + assert_equal(self.all_f16[b_nn].view(dtype=uint16), + b[b_nn].view(dtype=uint16)) # Convert from float64 back to float16 - b = np.array(self.all_f64, dtype=float16) - assert_equal(self.all_f16.view(dtype=uint16), - b.view(dtype=uint16)) + with np.errstate(invalid='ignore'): + b = np.array(self.all_f64, dtype=float16) + b_nn = b == b + assert_equal(self.all_f16[b_nn].view(dtype=uint16), + b[b_nn].view(dtype=uint16)) # Convert float16 to longdouble and back # This doesn't necessarily preserve the extra NaN bits, @@ -266,8 +274,8 @@ def test_half_correctness(self): if len(a32_fail) != 0: bad_index = a32_fail[0] assert_equal(self.finite_f32, a_manual, - "First non-equal is half value %x -> %g != %g" % - (self.finite_f16[bad_index], + "First non-equal is half value 0x%x -> %g != %g" % + (a_bits[bad_index], self.finite_f32[bad_index], a_manual[bad_index])) @@ -275,8 +283,8 @@ def test_half_correctness(self): if len(a64_fail) != 0: bad_index = a64_fail[0] assert_equal(self.finite_f64, a_manual, - "First non-equal is half value %x -> %g != %g" % - (self.finite_f16[bad_index], + "First non-equal is half value 0x%x -> %g != %g" % + (a_bits[bad_index], self.finite_f64[bad_index], a_manual[bad_index])) @@ -319,7 +327,8 @@ def test_half_funcs(self): a = np.array([0, 0, -1, -1/1e20, 0, 2.0**-24, 7.629e-6], dtype=float16) assert_equal(a.nonzero()[0], [2, 5, 6]) - a = a.byteswap().newbyteorder() + a = a.byteswap() + a = a.view(a.dtype.newbyteorder()) assert_equal(a.nonzero()[0], [2, 5, 6]) diff --git a/numpy/core/tests/test_ufunc.py b/numpy/core/tests/test_ufunc.py index 02c437021fe9..9fbc4b2dc57b 100644 --- a/numpy/core/tests/test_ufunc.py +++ b/numpy/core/tests/test_ufunc.py @@ -1617,6 +1617,8 @@ def check_identityless_reduction(self, a): assert_equal(np.minimum.reduce(a, axis=()), a) @requires_memory(6 * 1024**3) + @pytest.mark.skipif(sys.maxsize < 2**32, + reason="test array too large for 32bit platform") def test_identityless_reduction_huge_array(self): # Regression test for gh-20921 (copying identity incorrectly failed) arr = np.zeros((2, 2**31), 'uint8') diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index 5567244d14a9..59c670ffed29 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -1712,6 +1712,9 @@ def test_arctanh(self): assert_raises(FloatingPointError, np.arctanh, np.array(value, dtype=dt)) + # Make sure glibc < 2.18 atanh is not used, issue 25087 + assert np.signbit(np.arctanh(-1j).real) + # See: https://github.com/numpy/numpy/issues/20448 @pytest.mark.xfail( _glibc_older_than("2.17"), diff --git a/numpy/distutils/ccompiler_opt.py b/numpy/distutils/ccompiler_opt.py index 1e9de3c45bc0..37a5368b0b82 100644 --- a/numpy/distutils/ccompiler_opt.py +++ b/numpy/distutils/ccompiler_opt.py @@ -301,7 +301,8 @@ class _Config: ## Power8/ISA 2.07 VSX2 = dict(interest=2, implies="VSX", implies_detect=False), ## Power9/ISA 3.00 - VSX3 = dict(interest=3, implies="VSX2", implies_detect=False), + VSX3 = dict(interest=3, implies="VSX2", implies_detect=False, + extra_checks="VSX3_HALF_DOUBLE"), ## Power10/ISA 3.1 VSX4 = dict(interest=4, implies="VSX3", implies_detect=False, extra_checks="VSX4_MMA"), @@ -1308,7 +1309,7 @@ def feature_names(self, names=None, force_flags=None, macros=[]): def feature_is_exist(self, name): """ Returns True if a certain feature is exist and covered within - `_Config.conf_features`. + ``_Config.conf_features``. Parameters ---------- diff --git a/numpy/distutils/checks/extra_vsx3_half_double.c b/numpy/distutils/checks/extra_vsx3_half_double.c new file mode 100644 index 000000000000..514a2b18f96c --- /dev/null +++ b/numpy/distutils/checks/extra_vsx3_half_double.c @@ -0,0 +1,12 @@ +/** + * Assembler may not fully support the following VSX3 scalar + * instructions, even though compilers report VSX3 support. + */ +int main(void) +{ + unsigned short bits = 0xFF; + double f; + __asm__ __volatile__("xscvhpdp %x0,%x1" : "=wa"(f) : "wa"(bits)); + __asm__ __volatile__ ("xscvdphp %x0,%x1" : "=wa" (bits) : "wa" (f)); + return bits; +} diff --git a/numpy/lib/tests/test_histograms.py b/numpy/lib/tests/test_histograms.py index 38b3d3dcbf3f..8c55f16db98e 100644 --- a/numpy/lib/tests/test_histograms.py +++ b/numpy/lib/tests/test_histograms.py @@ -398,8 +398,9 @@ def test_histogram_bin_edges(self): edges = histogram_bin_edges(arr, bins='auto', range=(0, 1)) assert_array_equal(edges, e) - @requires_memory(free_bytes=1e10) - @pytest.mark.slow + # @requires_memory(free_bytes=1e10) + # @pytest.mark.slow + @pytest.mark.skip(reason="Bad memory reports lead to OOM in ci testing") def test_big_arrays(self): sample = np.zeros([100000000, 3]) xbins = 400 diff --git a/numpy/meson.build b/numpy/meson.build index 8da83ce61643..a8f810c8e559 100644 --- a/numpy/meson.build +++ b/numpy/meson.build @@ -10,15 +10,20 @@ endif # Platform detection is_windows = host_machine.system() == 'windows' -is_mingw = is_windows and cc.get_id() == 'gcc' +is_mingw = is_windows and cc.get_define('__MINGW32__') != '' if is_mingw - # For mingw-w64, link statically against the UCRT. - gcc_link_args = ['-lucrt', '-static'] - add_project_link_arguments(gcc_link_args, language: ['c', 'cpp']) - # Force gcc to float64 long doubles for compatibility with MSVC - # builds, for C only. - add_project_arguments('-mlong-double-64', language: 'c') + is_mingw_built_python = run_command( + py, ['-c', 'import sysconfig; print(sysconfig.get_platform())'], + check: true).stdout().strip().startswith('mingw') + if not is_mingw_built_python + # For mingw-w64, link statically against the UCRT. + gcc_link_args = ['-lucrt', '-static'] + add_project_link_arguments(gcc_link_args, language: ['c', 'cpp']) + # Force gcc to float64 long doubles for compatibility with MSVC + # builds, for C only. + add_project_arguments('-mlong-double-64', language: 'c') + endif # Make fprintf("%zd") work (see https://github.com/rgommers/scipy/issues/118) add_project_arguments('-D__USE_MINGW_ANSI_STDIO=1', language: ['c', 'cpp']) endif @@ -93,7 +98,7 @@ mkl_may_use_sdl = not use_ilp64 and _threading_opt in ['auto', 'iomp'] # First try scipy-openblas, and if found don't look for cblas or lapack, we # know what's inside the scipy-openblas wheels already. if blas_name == 'openblas' or blas_name == 'auto' - blas = dependency('scipy-openblas', required: false) + blas = dependency('scipy-openblas', method: 'pkg-config', required: false) if blas.found() blas_name = 'scipy-openblas' endif diff --git a/numpy/random/_generator.pyx b/numpy/random/_generator.pyx index 05061aa2dc3b..1bdba3356557 100644 --- a/numpy/random/_generator.pyx +++ b/numpy/random/_generator.pyx @@ -63,7 +63,7 @@ cdef int64_t _safe_sum_nonneg_int64(size_t num_colors, int64_t *colors): cdef inline void _shuffle_raw_wrap(bitgen_t *bitgen, np.npy_intp n, np.npy_intp first, np.npy_intp itemsize, np.npy_intp stride, - char* data, char* buf) nogil: + char* data, char* buf) noexcept nogil: # We trick gcc into providing a specialized implementation for # the most common case, yielding a ~33% performance improvement. # Note that apparently, only one branch can ever be specialized. @@ -76,7 +76,7 @@ cdef inline void _shuffle_raw_wrap(bitgen_t *bitgen, np.npy_intp n, cdef inline void _shuffle_raw(bitgen_t *bitgen, np.npy_intp n, np.npy_intp first, np.npy_intp itemsize, np.npy_intp stride, - char* data, char* buf) nogil: + char* data, char* buf) noexcept nogil: """ Parameters ---------- @@ -107,7 +107,7 @@ cdef inline void _shuffle_raw(bitgen_t *bitgen, np.npy_intp n, cdef inline void _shuffle_int(bitgen_t *bitgen, np.npy_intp n, - np.npy_intp first, int64_t* data) nogil: + np.npy_intp first, int64_t* data) noexcept nogil: """ Parameters ---------- diff --git a/numpy/random/mtrand.pyx b/numpy/random/mtrand.pyx index eb9e3ddd1dd0..9ffaa572d87e 100644 --- a/numpy/random/mtrand.pyx +++ b/numpy/random/mtrand.pyx @@ -4253,7 +4253,7 @@ cdef class RandomState: x.shape = tuple(final_shape) return x - def multinomial(self, np.npy_intp n, object pvals, size=None): + def multinomial(self, long n, object pvals, size=None): """ multinomial(n, pvals, size=None) diff --git a/numpy/random/tests/test_randomstate.py b/numpy/random/tests/test_randomstate.py index 524ac7b7c5e0..c77bfce883ae 100644 --- a/numpy/random/tests/test_randomstate.py +++ b/numpy/random/tests/test_randomstate.py @@ -183,6 +183,9 @@ def test_multinomial_pvals_float32(self): with pytest.raises(ValueError, match=match): random.multinomial(1, pvals) + def test_multinomial_n_float(self): + # Non-index integer types should gracefully truncate floats + random.multinomial(100.5, [0.2, 0.8]) class TestSetState: def setup_method(self): diff --git a/numpy/typing/tests/data/reveal/warnings_and_errors.pyi b/numpy/typing/tests/data/reveal/warnings_and_errors.pyi index 16492c2fb41b..b026e4f6e3b0 100644 --- a/numpy/typing/tests/data/reveal/warnings_and_errors.pyi +++ b/numpy/typing/tests/data/reveal/warnings_and_errors.pyi @@ -5,7 +5,7 @@ import numpy as np if sys.version_info >= (3, 11): from typing import assert_type else: - from typing_nptensions import assert_type + from typing_extensions import assert_type assert_type(np.ModuleDeprecationWarning(), np.ModuleDeprecationWarning) assert_type(np.VisibleDeprecationWarning(), np.VisibleDeprecationWarning) diff --git a/numpy/typing/tests/test_typing.py b/numpy/typing/tests/test_typing.py index cd009bb6e7f2..68c6f5d03fab 100644 --- a/numpy/typing/tests/test_typing.py +++ b/numpy/typing/tests/test_typing.py @@ -26,6 +26,20 @@ ) +# Only trigger a full `mypy` run if this environment variable is set +# Note that these tests tend to take over a minute even on a macOS M1 CPU, +# and more than that in CI. +RUN_MYPY = "NPY_RUN_MYPY_IN_TESTSUITE" in os.environ +if RUN_MYPY and RUN_MYPY not in ('0', '', 'false'): + RUN_MYPY = True + +# Skips all functions in this file +pytestmark = pytest.mark.skipif( + not RUN_MYPY, + reason="`NPY_RUN_MYPY_IN_TESTSUITE` not set" +) + + try: from mypy import api except ImportError: diff --git a/pavement.py b/pavement.py index 70a5e7657dde..41b04bc108fc 100644 --- a/pavement.py +++ b/pavement.py @@ -38,7 +38,7 @@ #----------------------------------- # Path to the release notes -RELEASE_NOTES = 'doc/source/release/1.26.1-notes.rst' +RELEASE_NOTES = 'doc/source/release/1.26.2-notes.rst' #------------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index ec8da79d3963..c8dc1a72eeba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ requires = [ [project] name = "numpy" -version = "1.26.1" +version = "1.26.2" # TODO: add `license-files` once PEP 639 is accepted (see meson-python#88) license = {file = "LICENSE.txt"} @@ -26,7 +26,7 @@ authors = [{name = "Travis E. Oliphant et al."}] maintainers = [ {name = "NumPy Developers", email="numpy-discussion@python.org"}, ] -requires-python = ">=3.9,<3.13" +requires-python = ">=3.9" readme = "README.md" classifiers = [ 'Development Status :: 5 - Production/Stable', @@ -150,10 +150,10 @@ tracker = "https://github.com/numpy/numpy/issues" # Note: the below skip command doesn't do much currently, the platforms to # build wheels for in CI are controlled in `.github/workflows/wheels.yml` and # `tools/ci/cirrus_wheels.yml`. -skip = "cp36-* cp37-* cp-38* pp37-* pp38-* *-manylinux_i686 *_ppc64le *_s390x *-musllinux_aarch64" +skip = "cp36-* cp37-* cp-38* pp37-* pp38-* *-manylinux_i686 *_ppc64le *_s390x" build-verbosity = "3" before-build = "bash {project}/tools/wheels/cibw_before_build.sh {project}" -config-settings = "setup-args=-Duse-ilp64=true setup-args=-Dblas=openblas setup-args=-Dlapack=openblas setup-args=-Dblas-symbol-suffix=64_" +config-settings = "setup-args=-Duse-ilp64=true setup-args=-Dblas=openblas setup-args=-Dlapack=openblas setup-args=-Dblas-symbol-suffix=64_ setup-args=-Dallow-noblas=false" # meson has a hard dependency on ninja, and we need meson to build # c-extensions in tests. There is a ninja PyPI package used in # build_requirements.txt for macOS, windows, linux but it cannot be in @@ -203,10 +203,16 @@ cli = 'vendored-meson/meson/meson.py' ".spin/cmds.py:test", ".spin/cmds.py:mypy", ".spin/cmds.py:config_openblas", + ".spin/cmds.py:lint", ] "Environments" = [ "spin.cmds.meson.run", ".spin/cmds.py:ipython", ".spin/cmds.py:python", "spin.cmds.meson.gdb" ] -"Documentation" = [".spin/cmds.py:docs"] +"Documentation" = [ + ".spin/cmds.py:docs", + ".spin/cmds.py:changelog", + ".spin/cmds.py:notes", +] + "Metrics" = [".spin/cmds.py:bench"] diff --git a/pyproject.toml.setuppy b/pyproject.toml.setuppy index bfb093ba7648..02a7d7c57de5 100644 --- a/pyproject.toml.setuppy +++ b/pyproject.toml.setuppy @@ -3,7 +3,7 @@ # to avoid building with Meson (e.g., in the Emscripten/Pyodide CI job) [project] name = "numpy" -version = "1.26.1" +version = "1.26.2" [build-system] requires = [ diff --git a/tools/ci/cirrus_arm.yml b/tools/ci/cirrus_arm.yml index a28f182ae8ba..7e394f55442b 100644 --- a/tools/ci/cirrus_arm.yml +++ b/tools/ci/cirrus_arm.yml @@ -127,7 +127,7 @@ freebsd_test_task: memory: 4G install_devtools_script: | - pkg install -y git bash ninja ccache + pkg install -y git bash ninja ccache blas cblas lapack pkgconf <<: *MODIFIED_CLONE @@ -149,7 +149,7 @@ freebsd_test_task: build_script: | chsh -s /usr/local/bin/bash source .venv/bin/activate - python -m pip install . --no-build-isolation -v -Csetup-args="-Dallow-noblas=true" + python -m pip install . --no-build-isolation -v -Csetup-args="-Dallow-noblas=false" test_script: | chsh -s /usr/local/bin/bash @@ -157,3 +157,7 @@ freebsd_test_task: cd tools python -m pytest --pyargs numpy -m "not slow" ccache -s + + on_failure: + debug_script: | + cat build/meson-logs/meson-log.txt diff --git a/tools/ci/cirrus_wheels.yml b/tools/ci/cirrus_wheels.yml index ea290a8b23c8..2cc1f6f69884 100644 --- a/tools/ci/cirrus_wheels.yml +++ b/tools/ci/cirrus_wheels.yml @@ -17,7 +17,7 @@ linux_aarch64_task: image: family/docker-builder-arm64 architecture: arm64 platform: linux - cpu: 2 + cpu: 1 memory: 8G matrix: # build in a matrix because building and testing all four wheels in a diff --git a/tools/wheels/LICENSE_linux.txt b/tools/wheels/LICENSE_linux.txt index 97596fe61f28..a5b5ae5c22e6 100644 --- a/tools/wheels/LICENSE_linux.txt +++ b/tools/wheels/LICENSE_linux.txt @@ -8,7 +8,7 @@ Name: OpenBLAS Files: numpy.libs/libopenblas*.so Description: bundled as a dynamically linked library Availability: https://github.com/OpenMathLib/OpenBLAS/ -License: BSD-3-Clause-Attribution +License: BSD-3-Clause Copyright (c) 2011-2014, The OpenBLAS Project All rights reserved. diff --git a/tools/wheels/LICENSE_osx.txt b/tools/wheels/LICENSE_osx.txt index dd221a34d49f..1ebd5663d02c 100644 --- a/tools/wheels/LICENSE_osx.txt +++ b/tools/wheels/LICENSE_osx.txt @@ -7,7 +7,7 @@ Name: OpenBLAS Files: numpy/.dylibs/libopenblas*.so Description: bundled as a dynamically linked library Availability: https://github.com/OpenMathLib/OpenBLAS/ -License: BSD-3-Clause-Attribution +License: BSD-3-Clause Copyright (c) 2011-2014, The OpenBLAS Project All rights reserved. diff --git a/tools/wheels/LICENSE_win32.txt b/tools/wheels/LICENSE_win32.txt index ef954a6ea781..2740fc6165a7 100644 --- a/tools/wheels/LICENSE_win32.txt +++ b/tools/wheels/LICENSE_win32.txt @@ -8,7 +8,7 @@ Name: OpenBLAS Files: numpy.libs\libopenblas*.dll Description: bundled as a dynamically linked library Availability: https://github.com/OpenMathLib/OpenBLAS/ -License: BSD-3-Clause-Attribution +License: BSD-3-Clause Copyright (c) 2011-2014, The OpenBLAS Project All rights reserved. diff --git a/tools/wheels/cibw_before_build.sh b/tools/wheels/cibw_before_build.sh index 23471a43285e..3217ce488c1b 100644 --- a/tools/wheels/cibw_before_build.sh +++ b/tools/wheels/cibw_before_build.sh @@ -4,6 +4,9 @@ PROJECT_DIR="$1" PLATFORM=$(PYTHONPATH=tools python -c "import openblas_support; print(openblas_support.get_plat())") # Update license +echo "" >> $PROJECT_DIR/LICENSE.txt +echo "----" >> $PROJECT_DIR/LICENSE.txt +echo "" >> $PROJECT_DIR/LICENSE.txt cat $PROJECT_DIR/LICENSES_bundled.txt >> $PROJECT_DIR/LICENSE.txt if [[ $RUNNER_OS == "Linux" ]] ; then cat $PROJECT_DIR/tools/wheels/LICENSE_linux.txt >> $PROJECT_DIR/LICENSE.txt diff --git a/vendored-meson/meson b/vendored-meson/meson index 66ba7dbbfe28..ea5809096404 160000 --- a/vendored-meson/meson +++ b/vendored-meson/meson @@ -1 +1 @@ -Subproject commit 66ba7dbbfe2838983f65ad8fe16da1535ebf5b9d +Subproject commit ea580909640457450e03d8b84d1fec9f035d7acb