diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index f2fda55..7d59053 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -4,36 +4,165 @@ on: release: types: [created] + # Enable Run Workflow button in GitHub UI + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: - wheels: + build_sdist: + name: Build SDist + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Build SDist + run: pipx run build --sdist + + - name: Check metadata + run: pipx run twine check dist/* + + - uses: actions/upload-artifact@v3 + with: + path: dist/*.tar.gz + + + build_wheels: + name: Wheels on ${{ matrix.platform_id }} - ${{ matrix.os }} runs-on: ${{ matrix.os }} - defaults: - run: - shell: bash -l {0} strategy: fail-fast: false matrix: - os: ["ubuntu-latest"] + # Loosely based on scikit-learn's config: + # https://github.com/scikit-learn/scikit-learn/blob/main/.github/workflows/wheels.yml + include: + - os: windows-latest + python-version: "3.8" + platform_id: win_amd64 + + # Linux 64 bit manylinux2014 + - os: ubuntu-latest + python-version: "3.8" + platform_id: manylinux_x86_64 + manylinux_image: manylinux2014 + + # Use x86 macOS runner to build both x86 and ARM. GitHub does not offer M1/M2 yet (only self-hosted). + - os: macos-latest + python-version: "3.8" + platform_id: macosx_x86_64 + steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - - name: Upgrade pip - run: | - python -m pip install --upgrade pip - - name: Build manylinux Python wheels - uses: RalfG/python-wheels-manylinux-build@v0.4.2-manylinux2014_x86_64 - with: - python-versions: 'cp38-cp38 cp39-cp39' - build-requirements: 'cffi numpy>=1.19,<1.20 cython' - pre-build-command: ${{ format('sh suitesparse.sh {0}', github.ref) }} - - name: Publish wheels to PyPI - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - run: | - pip install twine - twine upload dist/*-manylinux*.whl + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + +# - name: Install tools (macOS) +# if: contains(matrix.os, 'macos') +# # Install coreutils which includes `nproc` used by `make -j` in suitesparse.sh +# # +# # GitHub actions comes with libomp already installed, but for its native arch only. Must build universal one +# # manually so that both x86 and arm builds can be built. +# run: | +# brew install coreutils +# brew install libomp +# sh add_arm_to_libomp_dylib.sh + + - name: Build Wheels + env: + # very verbose + CIBW_BUILD_VERBOSITY: 3 + + # Build SuiteSparse + CIBW_BEFORE_ALL: bash suitesparse.sh ${{ github.ref }} + + # CMAKE_GNUtoMS=ON asks suitesparse.sh to build libraries in MSVC style on Windows. + CIBW_ENVIRONMENT_WINDOWS: CMAKE_GNUtoMS=ON GRAPHBLAS_PREFIX="C:/GraphBLAS" + + # macOS libomp requires special configs. BREW_LIBOMP=1 asks suitesparse.sh to include them. + CIBW_ENVIRONMENT_MACOS: BREW_LIBOMP="1" + + # Uncomment to only build CPython wheels +# CIBW_BUILD: "cp*" + + # macOS: build x86_64 and arm64 + #CIBW_ARCHS_MACOS: "x86_64 arm64" + + # No 32-bit builds + CIBW_SKIP: "*-win32 *_i686 *musl*" + + # Use delvewheel on Windows. + # This copies graphblas.dll into the wheel. "repair" in cibuildwheel parlance includes copying any shared + # libraries from the build host into the wheel to make the wheel self-contained. + # Cibuildwheel includes tools for this for Linux and macOS, and they recommend delvewheel for Windows. + # Note: Currently using a workaround: --no-mangle instead of stripping graphblas.dll + # see https://github.com/adang1345/delvewheel/issues/33 + CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel" + CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair --add-path \"C:\\GraphBLAS\\bin\" --no-mangle \"libgomp-1.dll;libgcc_s_seh-1.dll\" -w {dest_dir} {wheel}" + + # make cibuildwheel install test dependencies from pyproject.toml + CIBW_TEST_EXTRAS: "test" + + # run tests + CIBW_TEST_COMMAND: "pytest {project}/suitesparse_graphblas/tests" + + # GitHub Actions macOS Intel runner cannot run ARM tests. + CIBW_TEST_SKIP: "*-macosx_arm64" + + run: | + python -m pip install cibuildwheel + python -m cibuildwheel --output-dir wheelhouse . + shell: bash + + - uses: actions/upload-artifact@v3 + id: uploadAttempt1 + continue-on-error: true + with: + path: wheelhouse/*.whl + if-no-files-found: error + + # Retry upload if first attempt failed. This happens somewhat randomly and for irregular reasons. + # Logic is a duplicate of previous step. + - uses: actions/upload-artifact@v3 + id: uploadAttempt2 + if: steps.uploadAttempt1.outcome == 'failure' + continue-on-error: false + with: + path: wheelhouse/*.whl + if-no-files-found: error + + upload_all: + name: Upload to PyPI + needs: [build_wheels, build_sdist] + runs-on: ubuntu-latest + if: github.repository == 'GraphBLAS/python-suitesparse-graphblas' +# if: github.event_name == 'release' && github.event.action == 'published' + + steps: + - uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - uses: actions/download-artifact@v3 + with: + name: artifact + path: dist + + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + # PyPI does not allow replacing a file. Without this flag the entire action fails if even a single duplicate exists. + skip_existing: true + verbose: true + # Real PyPI: + password: ${{ secrets.PYPI_TOKEN }} + + # Test PyPI: +# password: ${{ secrets.TEST_PYPI_API_TOKEN }} +# repository_url: https://test.pypi.org/legacy/ diff --git a/.gitignore b/.gitignore index 609ca66..e382cda 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,9 @@ share/python-wheels/ MANIFEST wheelhouse +# Wheel building stuff +GraphBLAS-*/ + # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. diff --git a/add_arm_to_libomp_dylib.sh b/add_arm_to_libomp_dylib.sh new file mode 100755 index 0000000..8492c7c --- /dev/null +++ b/add_arm_to_libomp_dylib.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +#mkdir x86lib +mkdir armlib + +# download and unzip both x86 and arm libomp tarballs +#brew fetch --force --bottle-tag=x86_64_monterey libomp +brew fetch --force --bottle-tag=arm64_big_sur libomp + +# untar +#tar -xzf $(brew --cache --bottle-tag=x86_64_monterey libomp) --strip-components 2 -C x86lib +tar -xzf $(brew --cache --bottle-tag=arm64_big_sur libomp) --strip-components 2 -C armlib + +# merge +lipo armlib/lib/libomp.dylib $(brew --prefix libomp)/lib/libomp.dylib -output libomp.dylib -create +cp -f libomp.dylib $(brew --prefix libomp)/lib +rm libomp.dylib +rm -rf armlib diff --git a/build_graphblas_cffi.py b/build_graphblas_cffi.py index 0de462b..dff21e6 100644 --- a/build_graphblas_cffi.py +++ b/build_graphblas_cffi.py @@ -16,6 +16,12 @@ include_dirs.append(os.path.join(sys.prefix, "Library", "include")) library_dirs.append(os.path.join(sys.prefix, "Library", "lib")) + # wheels.yml configures suitesparse.sh to install GraphBLAS here. + prefix = "C:\\GraphBLAS" + include_dirs.append(os.path.join(prefix, "include")) + library_dirs.append(os.path.join(prefix, "lib")) + library_dirs.append(os.path.join(prefix, "bin")) + ffibuilder.set_source( "suitesparse_graphblas._graphblas", (ss_g / "source.c").read_text(), diff --git a/pyproject.toml b/pyproject.toml index 4059c4e..beccc7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ requires = [ "setuptools >=64", "setuptools-git-versioning", "wheel", - "cffi", + "cffi>=1.11", "cython", "oldest-supported-numpy", ] @@ -27,7 +27,7 @@ maintainers = [ {name = "Michel Pelletier", email = "michel@graphegon.com"}, ] classifiers = [ - "Development Status :: 4 - Beta", + "Development Status :: 5 - Production/Stable", "License :: OSI Approved :: Apache Software License", "Operating System :: MacOS :: MacOS X", "Operating System :: POSIX :: Linux", @@ -45,7 +45,7 @@ classifiers = [ ] dependencies = [ # These are super-old; can/should we update them? - "cffi>=1.0.0", + "cffi>=1.11", "numpy>=1.19", ] [project.urls] diff --git a/setup.py b/setup.py index 53d4d58..860cd3a 100644 --- a/setup.py +++ b/setup.py @@ -58,8 +58,10 @@ if use_cython: ext_modules = cythonize(ext_modules, include_path=include_dirs) -ext_modules.append(build_graphblas_cffi.get_extension(extra_compile_args=extra_compile_args)) +if build_graphblas_cffi.is_win: + ext_modules.append(build_graphblas_cffi.get_extension(extra_compile_args=extra_compile_args)) setup( ext_modules=ext_modules, + cffi_modules=None if build_graphblas_cffi.is_win else ["build_graphblas_cffi.py:ffibuilder"], ) diff --git a/suitesparse.sh b/suitesparse.sh old mode 100644 new mode 100755 index 77410fc..72d805f --- a/suitesparse.sh +++ b/suitesparse.sh @@ -1,14 +1,87 @@ +#!/bin/bash +# parse SuiteSparse version from first argument, a git tag that ends in the version (no leading v) if [[ $1 =~ refs/tags/([0-9]*\.[0-9]*\.[0-9]*)\..*$ ]]; then VERSION=${BASH_REMATCH[1]} else + echo "Specify a SuiteSparse version, such as: $0 refs/tags/7.4.3.0" exit -1 fi echo VERSION: $VERSION +NPROC="$(nproc)" +if [ -z "${NPROC}" ]; then + # Default for platforms that don't have nproc. Mostly Windows. + NPROC="2" +fi + +cmake_params=() +if [ -n "${BREW_LIBOMP}" ]; then + # macOS OpenMP flags. + # FindOpenMP doesn't find brew's libomp, so set the necessary configs manually. + cmake_params+=(-DOpenMP_C_FLAGS="-Xclang -fopenmp -I$(brew --prefix libomp)/include") + cmake_params+=(-DOpenMP_C_LIB_NAMES="libomp") + cmake_params+=(-DOpenMP_libomp_LIBRARY="omp") + export LDFLAGS="-L$(brew --prefix libomp)/lib" + + export CFLAGS="-arch x86_64" +# # build both x86 and ARM +# export CFLAGS="-arch x86_64 -arch arm64" +fi + +if [ -n "${CMAKE_GNUtoMS}" ]; then + # Windows needs .lib libraries, not .a + cmake_params+=(-DCMAKE_GNUtoMS=ON) + # Windows expects 'graphblas.lib', not 'libgraphblas.lib' + cmake_params+=(-DCMAKE_SHARED_LIBRARY_PREFIX=) + cmake_params+=(-DCMAKE_STATIC_LIBRARY_PREFIX=) +fi + +if [ -n "${GRAPHBLAS_PREFIX}" ]; then + echo "GRAPHBLAS_PREFIX=${GRAPHBLAS_PREFIX}" + cmake_params+=(-DCMAKE_INSTALL_PREFIX="${GRAPHBLAS_PREFIX}") +fi + curl -L https://github.com/DrTimothyAldenDavis/GraphBLAS/archive/refs/tags/v${VERSION}.tar.gz | tar xzf - cd GraphBLAS-${VERSION}/build -cmake .. -DCMAKE_BUILD_TYPE=Release -make -j$(nproc) + +# Disable optimizing some rarely-used types for significantly faster builds and significantly smaller wheel size. +# Also the build with all types enabled sometimes stalls on GitHub Actions. Probably due to exceeded resource limits. +# These can still be used, they'll just have reduced performance (AFAIK similar to UDTs). +# echo "#define GxB_NO_BOOL 1" >> ../Source/GB_control.h # +# echo "#define GxB_NO_FP32 1" >> ../Source/GB_control.h # +# echo "#define GxB_NO_FP64 1" >> ../Source/GB_control.h # +echo "#define GxB_NO_FC32 1" >> ../Source/GB_control.h # +echo "#define GxB_NO_FC64 1" >> ../Source/GB_control.h # +# echo "#define GxB_NO_INT16 1" >> ../Source/GB_control.h +# echo "#define GxB_NO_INT32 1" >> ../Source/GB_control.h +# echo "#define GxB_NO_INT64 1" >> ../Source/GB_control.h # +# echo "#define GxB_NO_INT8 1" >> ../Source/GB_control.h +echo "#define GxB_NO_UINT16 1" >> ../Source/GB_control.h +echo "#define GxB_NO_UINT32 1" >> ../Source/GB_control.h +# echo "#define GxB_NO_UINT64 1" >> ../Source/GB_control.h +echo "#define GxB_NO_UINT8 1" >> ../Source/GB_control.h + +# Disable all Source/Generated2 kernels. For workflow development only. +#cmake_params+=(-DCMAKE_CUDA_DEV=1) + +cmake .. -DCMAKE_BUILD_TYPE=Release -G 'Unix Makefiles' "${cmake_params[@]}" +make -j$NPROC make install + +if [ -n "${CMAKE_GNUtoMS}" ]; then + if [ -z "${GRAPHBLAS_PREFIX}" ]; then + # Windows default + GRAPHBLAS_PREFIX="C:/Program Files (x86)" + fi + + # Windows: + # CMAKE_STATIC_LIBRARY_PREFIX is sometimes ignored, possibly when the MinGW toolchain is selected. + # Drop the 'lib' prefix manually. + echo "manually removing lib prefix" + mv "${GRAPHBLAS_PREFIX}/lib/libgraphblas.lib" "${GRAPHBLAS_PREFIX}/lib/graphblas.lib" + mv "${GRAPHBLAS_PREFIX}/lib/libgraphblas.dll.a" "${GRAPHBLAS_PREFIX}/lib/graphblas.dll.a" + # cp instead of mv because the GNU tools expect libgraphblas.dll and the MS tools expect graphblas.dll. + cp "${GRAPHBLAS_PREFIX}/bin/libgraphblas.dll" "${GRAPHBLAS_PREFIX}/bin/graphblas.dll" +fi diff --git a/suitesparse_graphblas/tests/__init__.py b/suitesparse_graphblas/tests/__init__.py deleted file mode 100644 index e69de29..0000000