From 7b2dac4d253d7ec96dbe919b5a42a31d70d5870c Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 18 Oct 2023 19:59:27 -0500 Subject: [PATCH 1/8] Try using `sysconfig` to configure the SS JIT --- .github/workflows/test_and_build.yml | 2 +- graphblas/tests/test_ssjit.py | 8 ++++++++ pyproject.toml | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index d4504e2fd..76d73b399 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -85,7 +85,7 @@ jobs: shell: bash -l {0} strategy: # To "stress test" in CI, set `fail-fast` to `false` and perhaps add more items to `matrix.slowtask` - fail-fast: true + fail-fast: false # The build matrix is [os]x[slowtask] and then randomly chooses [pyver] and [sourcetype]. # This should ensure we'll have full code coverage (i.e., no chance of getting unlucky), # since we need to run all slow tests on Windows and non-Windoes OSes. diff --git a/graphblas/tests/test_ssjit.py b/graphblas/tests/test_ssjit.py index 3c974c50d..14225d39f 100644 --- a/graphblas/tests/test_ssjit.py +++ b/graphblas/tests/test_ssjit.py @@ -1,6 +1,7 @@ import os import pathlib import sys +import sysconfig import numpy as np import pytest @@ -26,6 +27,13 @@ @pytest.fixture(scope="module", autouse=True) def _setup_jit(): + gb.ss.config["jit_c_control"] = "on" + gb.ss.config["jit_c_compiler_name"] = sysconfig.get_config_var("CC") + gb.ss.config["jit_c_compiler_flags"] = sysconfig.get_config_var("CFLAGS") + gb.ss.config["jit_c_libraries"] = sysconfig.get_config_var("LIBS") + print(gb.ss.config) + return + # Configuration values below were obtained from the output of the JIT config # in CI, but with paths changed to use `{conda_prefix}` where appropriate. if "CONDA_PREFIX" not in os.environ or _IS_SSGB7: diff --git a/pyproject.toml b/pyproject.toml index 9579b1c16..04ef28645 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,7 +87,7 @@ pandas = [ "pandas >=1.2", ] scipy = [ - "scipy >=1.8", + "scipy >=1.9", ] suitesparse-udf = [ # udf requires numba "python-graphblas[suitesparse,numba]", From 681ab1598f153ee50555b3338efb1b5e8aa7bb30 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 18 Oct 2023 20:14:45 -0500 Subject: [PATCH 2/8] moar please --- .github/workflows/test_and_build.yml | 2 +- graphblas/tests/test_ssjit.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index 76d73b399..48e63b7a1 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -354,7 +354,7 @@ jobs: G=${{ needs.rngs.outputs.backend == 'G' || '' }} ; H=${{ needs.rngs.outputs.backend == 'H' || '' }} normal=${{ matrix.slowtask == 'pytest_normal' || '' }} ; bizarro=${{ matrix.slowtask == 'pytest_bizarro' || '' }} ubuntu=${{ matrix.os == 'ubuntu-latest' || '' }} ; windows=${{ matrix.os == 'windows-latest' || '' }} ; macos=${{ matrix.os == 'macos-latest' || '' }} - mapnumpy='--mapnumpy' ; nomapnumpy='--no-mapnumpy' ; suitesparse='--backend=suitesparse' ; vanilla='--backend=suitesparse-vanilla' + mapnumpy='--mapnumpy' ; nomapnumpy='--no-mapnumpy' ; suitesparse='--backend=suitesparse' ; vanilla='--backend=suitesparse' args=$( if [[ $A && $normal ]] ; then if [[ $ubuntu ]] ; then echo " $nomapnumpy" ; elif [[ $windows ]] ; then echo " $mapnumpy" ; fi ; fi)$( \ if [[ $A && $bizarro ]] ; then if [[ $ubuntu ]] ; then echo " $mapnumpy" ; elif [[ $windows ]] ; then echo " $nomapnumpy" ; fi ; fi)$( \ diff --git a/graphblas/tests/test_ssjit.py b/graphblas/tests/test_ssjit.py index 14225d39f..582323b08 100644 --- a/graphblas/tests/test_ssjit.py +++ b/graphblas/tests/test_ssjit.py @@ -28,9 +28,15 @@ @pytest.fixture(scope="module", autouse=True) def _setup_jit(): gb.ss.config["jit_c_control"] = "on" - gb.ss.config["jit_c_compiler_name"] = sysconfig.get_config_var("CC") - gb.ss.config["jit_c_compiler_flags"] = sysconfig.get_config_var("CFLAGS") - gb.ss.config["jit_c_libraries"] = sysconfig.get_config_var("LIBS") + if val := sysconfig.get_config_var("CC"): + gb.ss.config["jit_c_compiler_name"] = val + if val := sysconfig.get_config_var("CFLAGS"): + gb.ss.config["jit_c_compiler_flags"] = val + if val := sysconfig.get_config_var("LIBS"): + gb.ss.config["jit_c_libraries"] = val + # if val := sysconfig.get_config_var('LDFLAGS'): + if False: + gb.ss.config["jit_c_linker_flags"] = val print(gb.ss.config) return From 45a35778fd3f635bc02a7206d54e636e0bbb98f8 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 18 Oct 2023 20:22:25 -0500 Subject: [PATCH 3/8] Fix; more; less --- .github/workflows/test_and_build.yml | 3 ++- graphblas/tests/test_ssjit.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index 48e63b7a1..a63711560 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -220,6 +220,7 @@ jobs: # These should be exact versions psgver=$(python -c 'import random ; print(random.choice(["==7.4.0.0", "==7.4.1.0", "==7.4.2.0", "==7.4.3.0", "==7.4.3.1", "==7.4.3.2", "==8.0.2.1", "==8.2.0.1", "==8.2.1.0", ""]))') fi + psg="" # XXX if [[ ${npver} == "=1.26" ]] ; then numbaver="" if [[ ${spver} == "=1.8" || ${spver} == "=1.9" ]] ; then @@ -339,7 +340,7 @@ jobs: echo ${args} set -x # echo on coverage run -m pytest --color=yes --randomly -v ${args} \ - ${{ matrix.slowtask == 'pytest_normal' && '--runslow' || '' }} + ${{ matrix.slowtask == 'pytest_normal' && '--runslow' || '' }} graphblas/tests/test_ssjit.py - name: Unit tests (bizarro scalars) run: | # Run tests again with Scalars being C scalars by default diff --git a/graphblas/tests/test_ssjit.py b/graphblas/tests/test_ssjit.py index 582323b08..0aec90033 100644 --- a/graphblas/tests/test_ssjit.py +++ b/graphblas/tests/test_ssjit.py @@ -27,6 +27,8 @@ @pytest.fixture(scope="module", autouse=True) def _setup_jit(): + if _IS_SSGB7: + return gb.ss.config["jit_c_control"] = "on" if val := sysconfig.get_config_var("CC"): gb.ss.config["jit_c_compiler_name"] = val From a2bb07ec4211102b85c08cc63a135ec98c5e03dc Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 18 Oct 2023 20:29:58 -0500 Subject: [PATCH 4/8] oops --- .github/workflows/test_and_build.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index a63711560..f184649af 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -213,14 +213,16 @@ jobs: fi elif [[ ${{ steps.sourcetype.outputs.selected}} == "conda-forge" ]] ; then psgver=$(python -c 'import random ; print(random.choice(["=7.4.0", "=7.4.1", "=7.4.2", "=7.4.3.0", "=7.4.3.1", "=7.4.3.2", "=8.0.2.1", "=8.2.0.1", "=8.2.1.0", ""]))') + psgver="" # XXX psg=python-suitesparse-graphblas${psgver} elif [[ ${{ steps.sourcetype.outputs.selected}} == "wheel" ]] ; then psgver=$(python -c 'import random ; print(random.choice(["==7.4.3.2", "==8.0.2.1", "==8.2.0.1", "==8.2.1.0", ""]))') + psgver="" # XXX elif [[ ${{ steps.sourcetype.outputs.selected}} == "source" ]] ; then # These should be exact versions psgver=$(python -c 'import random ; print(random.choice(["==7.4.0.0", "==7.4.1.0", "==7.4.2.0", "==7.4.3.0", "==7.4.3.1", "==7.4.3.2", "==8.0.2.1", "==8.2.0.1", "==8.2.1.0", ""]))') + psgver="" # XXX fi - psg="" # XXX if [[ ${npver} == "=1.26" ]] ; then numbaver="" if [[ ${spver} == "=1.8" || ${spver} == "=1.9" ]] ; then @@ -319,7 +321,7 @@ jobs: G=${{ needs.rngs.outputs.backend == 'G' || '' }} ; H=${{ needs.rngs.outputs.backend == 'H' || '' }} normal=${{ matrix.slowtask == 'pytest_normal' || '' }} ; bizarro=${{ matrix.slowtask == 'pytest_bizarro' || '' }} ubuntu=${{ matrix.os == 'ubuntu-latest' || '' }} ; windows=${{ matrix.os == 'windows-latest' || '' }} ; macos=${{ matrix.os == 'macos-latest' || '' }} - mapnumpy='--mapnumpy' ; nomapnumpy='--no-mapnumpy' ; suitesparse='--backend=suitesparse' ; vanilla='--backend=suitesparse-vanilla' + mapnumpy='--mapnumpy' ; nomapnumpy='--no-mapnumpy' ; suitesparse='--backend=suitesparse' ; vanilla='--backend=suitesparse' args=$( if [[ $A && $normal ]] ; then if [[ $ubuntu ]] ; then echo " $mapnumpy" ; elif [[ $windows ]] ; then echo " $nomapnumpy" ; fi ; fi)$( \ if [[ $A && $bizarro ]] ; then if [[ $ubuntu ]] ; then echo " $nomapnumpy" ; elif [[ $windows ]] ; then echo " $mapnumpy" ; fi ; fi)$( \ From 36c40e309336fc8f9c00dda926d21d0bea2f2589 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 18 Oct 2023 20:36:12 -0500 Subject: [PATCH 5/8] better for testing --- .github/workflows/test_and_build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index f184649af..d37486926 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -321,7 +321,7 @@ jobs: G=${{ needs.rngs.outputs.backend == 'G' || '' }} ; H=${{ needs.rngs.outputs.backend == 'H' || '' }} normal=${{ matrix.slowtask == 'pytest_normal' || '' }} ; bizarro=${{ matrix.slowtask == 'pytest_bizarro' || '' }} ubuntu=${{ matrix.os == 'ubuntu-latest' || '' }} ; windows=${{ matrix.os == 'windows-latest' || '' }} ; macos=${{ matrix.os == 'macos-latest' || '' }} - mapnumpy='--mapnumpy' ; nomapnumpy='--no-mapnumpy' ; suitesparse='--backend=suitesparse' ; vanilla='--backend=suitesparse' + mapnumpy='--mapnumpy' ; nomapnumpy='--no-mapnumpy' ; suitesparse='' ; vanilla='' args=$( if [[ $A && $normal ]] ; then if [[ $ubuntu ]] ; then echo " $mapnumpy" ; elif [[ $windows ]] ; then echo " $nomapnumpy" ; fi ; fi)$( \ if [[ $A && $bizarro ]] ; then if [[ $ubuntu ]] ; then echo " $nomapnumpy" ; elif [[ $windows ]] ; then echo " $mapnumpy" ; fi ; fi)$( \ @@ -341,7 +341,7 @@ jobs: if [[ $H && $bizarro ]] ; then if [[ $macos ]] ; then echo " $suitesparse" ; elif [[ $windows ]] ; then echo " $vanilla" ; fi ; fi) echo ${args} set -x # echo on - coverage run -m pytest --color=yes --randomly -v ${args} \ + coverage run -m pytest --color=yes --randomly -v ${args} --backend=suitesparse \ ${{ matrix.slowtask == 'pytest_normal' && '--runslow' || '' }} graphblas/tests/test_ssjit.py - name: Unit tests (bizarro scalars) run: | @@ -357,7 +357,7 @@ jobs: G=${{ needs.rngs.outputs.backend == 'G' || '' }} ; H=${{ needs.rngs.outputs.backend == 'H' || '' }} normal=${{ matrix.slowtask == 'pytest_normal' || '' }} ; bizarro=${{ matrix.slowtask == 'pytest_bizarro' || '' }} ubuntu=${{ matrix.os == 'ubuntu-latest' || '' }} ; windows=${{ matrix.os == 'windows-latest' || '' }} ; macos=${{ matrix.os == 'macos-latest' || '' }} - mapnumpy='--mapnumpy' ; nomapnumpy='--no-mapnumpy' ; suitesparse='--backend=suitesparse' ; vanilla='--backend=suitesparse' + mapnumpy='--mapnumpy' ; nomapnumpy='--no-mapnumpy' ; suitesparse='' ; vanilla='' args=$( if [[ $A && $normal ]] ; then if [[ $ubuntu ]] ; then echo " $nomapnumpy" ; elif [[ $windows ]] ; then echo " $mapnumpy" ; fi ; fi)$( \ if [[ $A && $bizarro ]] ; then if [[ $ubuntu ]] ; then echo " $mapnumpy" ; elif [[ $windows ]] ; then echo " $nomapnumpy" ; fi ; fi)$( \ @@ -377,8 +377,8 @@ jobs: if [[ $H && $bizarro ]] ; then if [[ $macos ]] ; then echo " $vanilla" ; elif [[ $windows ]] ; then echo " $suitesparse" ; fi ; fi) echo ${args} set -x # echo on - coverage run -a -m pytest --color=yes --randomly -v ${args} \ - ${{ matrix.slowtask == 'pytest_bizarro' && '--runslow' || '' }} + coverage run -a -m pytest --color=yes --randomly -v ${args} --backend=suitesparse \ + ${{ matrix.slowtask == 'pytest_bizarro' && '--runslow' || '' }} graphblas/tests/test_ssjit.py git checkout . # Undo changes to scalar default - name: Miscellaneous tests if: matrix.slowtask == 'pytest_normal' From 0a0a7ebe2c217c7bc2dbd1ea11158dfce84c3ac4 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 18 Oct 2023 20:42:02 -0500 Subject: [PATCH 6/8] double your pleasure, double your fun --- .github/workflows/test_and_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index d37486926..acf2269ae 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -90,7 +90,7 @@ jobs: # This should ensure we'll have full code coverage (i.e., no chance of getting unlucky), # since we need to run all slow tests on Windows and non-Windoes OSes. matrix: - os: ["ubuntu-latest", "macos-latest", "windows-latest"] + os: ["ubuntu-latest", "macos-latest", "windows-latest", "ubuntu-latest", "macos-latest", "windows-latest"] slowtask: ["pytest_normal", "pytest_bizarro", "notebooks"] env: # Wheels on OS X come with an OpenMP that conflicts with OpenMP from conda-forge. From 0f958bff2e7ec60b468e23c018cb07ee82119f3f Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 18 Oct 2023 21:08:07 -0500 Subject: [PATCH 7/8] Try setting jit_c_linker_flags to see what happens --- graphblas/tests/test_ssjit.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/graphblas/tests/test_ssjit.py b/graphblas/tests/test_ssjit.py index 0aec90033..f28ffdd54 100644 --- a/graphblas/tests/test_ssjit.py +++ b/graphblas/tests/test_ssjit.py @@ -36,8 +36,8 @@ def _setup_jit(): gb.ss.config["jit_c_compiler_flags"] = val if val := sysconfig.get_config_var("LIBS"): gb.ss.config["jit_c_libraries"] = val - # if val := sysconfig.get_config_var('LDFLAGS'): - if False: + # if False: + if val := sysconfig.get_config_var("LDFLAGS"): gb.ss.config["jit_c_linker_flags"] = val print(gb.ss.config) return From 72d6154bc8a78a6d35c0546c74149d2e798b37e3 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Wed, 18 Oct 2023 21:13:04 -0500 Subject: [PATCH 8/8] haha, nevermind --- graphblas/tests/test_ssjit.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/graphblas/tests/test_ssjit.py b/graphblas/tests/test_ssjit.py index f28ffdd54..5aa35793f 100644 --- a/graphblas/tests/test_ssjit.py +++ b/graphblas/tests/test_ssjit.py @@ -36,9 +36,6 @@ def _setup_jit(): gb.ss.config["jit_c_compiler_flags"] = val if val := sysconfig.get_config_var("LIBS"): gb.ss.config["jit_c_libraries"] = val - # if False: - if val := sysconfig.get_config_var("LDFLAGS"): - gb.ss.config["jit_c_linker_flags"] = val print(gb.ss.config) return