From e26006972c58f066215d998ac2c0e70258404d9a Mon Sep 17 00:00:00 2001 From: Janakarajan Natarajan Date: Tue, 9 Feb 2021 17:42:15 +0000 Subject: [PATCH 1/5] Build aarch64 wheels cibuildwheel allows to build non-native architectures using CIBW_ARCHS. --- .github/workflows/cibuildwheel.yml | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cibuildwheel.yml b/.github/workflows/cibuildwheel.yml index ac542bf2ab77..1899197bb927 100644 --- a/.github/workflows/cibuildwheel.yml +++ b/.github/workflows/cibuildwheel.yml @@ -15,8 +15,18 @@ jobs: strategy: matrix: os: [ubuntu-18.04, windows-latest, macos-latest] + cibw_archs: ["auto"] + include: + - os: ubuntu-18.04 + cibw_archs: "aarch64" steps: + - name: Set up QEMU + if: matrix.cibw_archs == 'aarch64' + uses: docker/setup-qemu-action@v1 + with: + platforms: arm64 + - uses: actions/checkout@v2 with: fetch-depth: 0 @@ -28,7 +38,7 @@ jobs: - name: Install cibuildwheel run: | - python -m pip install cibuildwheel==1.6.3 + python -m pip install cibuildwheel==1.9.0 - name: Copy setup.cfg to configure wheel run: | @@ -43,17 +53,18 @@ jobs: CIBW_MANYLINUX_I686_IMAGE: manylinux1 CIBW_BEFORE_BUILD: pip install certifi numpy==1.19.3 MPL_DISABLE_FH4: "yes" + CIBW_ARCHS: ${{ matrix.cibw_archs }} - name: Build wheels for CPython run: | python -m cibuildwheel --output-dir dist env: - CIBW_BUILD: "cp3?-*" - CIBW_SKIP: "cp35-* cp36-* cp39-*" + CIBW_BUILD: "cp37-* cp38-*" CIBW_MANYLINUX_X86_64_IMAGE: manylinux1 CIBW_MANYLINUX_I686_IMAGE: manylinux1 - CIBW_BEFORE_BUILD: pip install certifi numpy==1.16 + CIBW_BEFORE_BUILD: pip install certifi oldest-supported-numpy MPL_DISABLE_FH4: "yes" + CIBW_ARCHS: ${{ matrix.cibw_archs }} - name: Build wheels for CPython 3.6 run: | @@ -62,8 +73,9 @@ jobs: CIBW_BUILD: "cp36-*" CIBW_MANYLINUX_X86_64_IMAGE: manylinux1 CIBW_MANYLINUX_I686_IMAGE: manylinux1 - CIBW_BEFORE_BUILD: pip install certifi numpy==1.16 + CIBW_BEFORE_BUILD: pip install certifi oldest-supported-numpy MPL_DISABLE_FH4: "yes" + CIBW_ARCHS: ${{ matrix.cibw_archs }} if: > startsWith(github.ref, 'refs/heads/v3.3') || startsWith(github.ref, 'refs/tags/v3.3') @@ -73,7 +85,8 @@ jobs: python -m cibuildwheel --output-dir dist env: CIBW_BUILD: "pp3?-*" - CIBW_BEFORE_BUILD: pip install certifi numpy==1.16 + CIBW_BEFORE_BUILD: pip install certifi oldest-supported-numpy + CIBW_ARCHS: ${{ matrix.cibw_archs }} if: > runner.os != 'Windows' && ( startsWith(github.ref, 'refs/heads/v3.3') || From 3748039cc21edfe54072ec25f6ceaaedcb05ffae Mon Sep 17 00:00:00 2001 From: Janakarajan Natarajan Date: Fri, 19 Feb 2021 22:44:55 +0000 Subject: [PATCH 2/5] Build and cache numpy=1.16 aarch64 wheels --- .github/workflows/cibuildwheel.yml | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cibuildwheel.yml b/.github/workflows/cibuildwheel.yml index 1899197bb927..ab010360aea3 100644 --- a/.github/workflows/cibuildwheel.yml +++ b/.github/workflows/cibuildwheel.yml @@ -36,10 +36,27 @@ jobs: with: python-version: '3.7' + - uses: actions/cache@v2 + id: numpy-aarch64-cache + with: + path: numpy-aarch64-cache/ + key: numpy-aarch64-cache + - name: Install cibuildwheel run: | python -m pip install cibuildwheel==1.9.0 + - name: Build numpy 1.16 for aarch64 + if: matrix.cibw_archs == 'aarch64' && steps.numpy-aarch64-cache.outputs.cache-hit != 'true' + run: | + wget https://files.pythonhosted.org/packages/04/b6/d7faa70a3e3eac39f943cc6a6a64ce378259677de516bd899dd9eb8f9b32/numpy-1.16.0.zip + unzip numpy-1.16.0.zip + cd numpy-1.16.0 + python -m cibuildwheel --output-dir ../numpy-aarch64-cache + env: + CIBW_BUILD: "cp36-* cp37-* cp38-*" + CIBW_ARCHS: aarch64 + - name: Copy setup.cfg to configure wheel run: | cp setup.cfg.template setup.cfg @@ -62,7 +79,7 @@ jobs: CIBW_BUILD: "cp37-* cp38-*" CIBW_MANYLINUX_X86_64_IMAGE: manylinux1 CIBW_MANYLINUX_I686_IMAGE: manylinux1 - CIBW_BEFORE_BUILD: pip install certifi oldest-supported-numpy + CIBW_BEFORE_BUILD: pip install certifi; pip install --find-links=numpy-aarch64-cache/ numpy==1.16 MPL_DISABLE_FH4: "yes" CIBW_ARCHS: ${{ matrix.cibw_archs }} @@ -73,7 +90,7 @@ jobs: CIBW_BUILD: "cp36-*" CIBW_MANYLINUX_X86_64_IMAGE: manylinux1 CIBW_MANYLINUX_I686_IMAGE: manylinux1 - CIBW_BEFORE_BUILD: pip install certifi oldest-supported-numpy + CIBW_BEFORE_BUILD: pip install certifi; pip install --find-links=numpy-aarch64-cache/ numpy==1.16 MPL_DISABLE_FH4: "yes" CIBW_ARCHS: ${{ matrix.cibw_archs }} if: > @@ -85,7 +102,7 @@ jobs: python -m cibuildwheel --output-dir dist env: CIBW_BUILD: "pp3?-*" - CIBW_BEFORE_BUILD: pip install certifi oldest-supported-numpy + CIBW_BEFORE_BUILD: pip install certifi numpy==1.16 CIBW_ARCHS: ${{ matrix.cibw_archs }} if: > runner.os != 'Windows' && ( From 47263cc5da5c0b1ffd0d4e1f3e3eaabc59357c0b Mon Sep 17 00:00:00 2001 From: Janakarajan Natarajan Date: Mon, 8 Mar 2021 18:26:11 +0000 Subject: [PATCH 3/5] Use global var for numpy version --- .github/workflows/cibuildwheel.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cibuildwheel.yml b/.github/workflows/cibuildwheel.yml index ab010360aea3..f89f969e0434 100644 --- a/.github/workflows/cibuildwheel.yml +++ b/.github/workflows/cibuildwheel.yml @@ -19,6 +19,7 @@ jobs: include: - os: ubuntu-18.04 cibw_archs: "aarch64" + numpy-version: "1.16" steps: - name: Set up QEMU @@ -40,7 +41,7 @@ jobs: id: numpy-aarch64-cache with: path: numpy-aarch64-cache/ - key: numpy-aarch64-cache + key: numpy-aarch64-cache-${{ matrix.numpy-version }} - name: Install cibuildwheel run: | From 3907180daf6c986373b2d4e11b6712b790220dcc Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Tue, 9 Mar 2021 00:37:24 -0500 Subject: [PATCH 4/5] ci: Move NumPy version into a variable. --- .github/workflows/cibuildwheel.yml | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/cibuildwheel.yml b/.github/workflows/cibuildwheel.yml index f89f969e0434..51fd13d1a169 100644 --- a/.github/workflows/cibuildwheel.yml +++ b/.github/workflows/cibuildwheel.yml @@ -12,6 +12,9 @@ jobs: build_wheels: name: Build wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} + env: + min-numpy-version: "1.16.0" + min-numpy-hash: "04/b6/d7faa70a3e3eac39f943cc6a6a64ce378259677de516bd899dd9eb8f9b32" strategy: matrix: os: [ubuntu-18.04, windows-latest, macos-latest] @@ -19,7 +22,6 @@ jobs: include: - os: ubuntu-18.04 cibw_archs: "aarch64" - numpy-version: "1.16" steps: - name: Set up QEMU @@ -41,18 +43,18 @@ jobs: id: numpy-aarch64-cache with: path: numpy-aarch64-cache/ - key: numpy-aarch64-cache-${{ matrix.numpy-version }} + key: numpy-aarch64-cache-${{ env.min-numpy-version }} - name: Install cibuildwheel run: | python -m pip install cibuildwheel==1.9.0 - - name: Build numpy 1.16 for aarch64 + - name: Build minimum NumPy for aarch64 if: matrix.cibw_archs == 'aarch64' && steps.numpy-aarch64-cache.outputs.cache-hit != 'true' run: | - wget https://files.pythonhosted.org/packages/04/b6/d7faa70a3e3eac39f943cc6a6a64ce378259677de516bd899dd9eb8f9b32/numpy-1.16.0.zip - unzip numpy-1.16.0.zip - cd numpy-1.16.0 + wget https://files.pythonhosted.org/packages/${{ env.min-numpy-hash }}/numpy-${{ env.min-numpy-version }}.zip + unzip numpy-${{ env.min-numpy-version }}.zip + cd numpy-${{ env.min-numpy-version }} python -m cibuildwheel --output-dir ../numpy-aarch64-cache env: CIBW_BUILD: "cp36-* cp37-* cp38-*" @@ -80,7 +82,7 @@ jobs: CIBW_BUILD: "cp37-* cp38-*" CIBW_MANYLINUX_X86_64_IMAGE: manylinux1 CIBW_MANYLINUX_I686_IMAGE: manylinux1 - CIBW_BEFORE_BUILD: pip install certifi; pip install --find-links=numpy-aarch64-cache/ numpy==1.16 + CIBW_BEFORE_BUILD: pip install certifi; pip install --find-links=numpy-aarch64-cache/ numpy==${{ env.min-numpy-version }} MPL_DISABLE_FH4: "yes" CIBW_ARCHS: ${{ matrix.cibw_archs }} @@ -91,7 +93,7 @@ jobs: CIBW_BUILD: "cp36-*" CIBW_MANYLINUX_X86_64_IMAGE: manylinux1 CIBW_MANYLINUX_I686_IMAGE: manylinux1 - CIBW_BEFORE_BUILD: pip install certifi; pip install --find-links=numpy-aarch64-cache/ numpy==1.16 + CIBW_BEFORE_BUILD: pip install certifi; pip install --find-links=numpy-aarch64-cache/ numpy==${{ env.min-numpy-version }} MPL_DISABLE_FH4: "yes" CIBW_ARCHS: ${{ matrix.cibw_archs }} if: > @@ -103,7 +105,7 @@ jobs: python -m cibuildwheel --output-dir dist env: CIBW_BUILD: "pp3?-*" - CIBW_BEFORE_BUILD: pip install certifi numpy==1.16 + CIBW_BEFORE_BUILD: pip install certifi numpy==${{ env.min-numpy-version }} CIBW_ARCHS: ${{ matrix.cibw_archs }} if: > runner.os != 'Windows' && ( From c842a29d1b6005c0b910905c3beba3fc05d9849d Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Tue, 9 Mar 2021 04:14:51 -0500 Subject: [PATCH 5/5] ci: Fix cache key for NumPy wheels. It must be unique across jobs, or one of the others (which are undoubtedly faster, since they aren't building NumPy) will finish first and fill the cache with an empty directory. --- .github/workflows/cibuildwheel.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cibuildwheel.yml b/.github/workflows/cibuildwheel.yml index 51fd13d1a169..2100d15fe000 100644 --- a/.github/workflows/cibuildwheel.yml +++ b/.github/workflows/cibuildwheel.yml @@ -40,17 +40,17 @@ jobs: python-version: '3.7' - uses: actions/cache@v2 - id: numpy-aarch64-cache + id: numpy-cache with: path: numpy-aarch64-cache/ - key: numpy-aarch64-cache-${{ env.min-numpy-version }} + key: numpy-${{ matrix.cibw_archs }}-cache-${{ env.min-numpy-version }} - name: Install cibuildwheel run: | python -m pip install cibuildwheel==1.9.0 - name: Build minimum NumPy for aarch64 - if: matrix.cibw_archs == 'aarch64' && steps.numpy-aarch64-cache.outputs.cache-hit != 'true' + if: matrix.cibw_archs == 'aarch64' && steps.numpy-cache.outputs.cache-hit != 'true' run: | wget https://files.pythonhosted.org/packages/${{ env.min-numpy-hash }}/numpy-${{ env.min-numpy-version }}.zip unzip numpy-${{ env.min-numpy-version }}.zip