From 4632af55654a15bf642f5e1bf366c9e2c05210e3 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 27 Jun 2023 00:41:52 -0500 Subject: [PATCH 1/9] Backport many of the SS:GB 8 changes to run on 7 --- .github/workflows/imports.yml | 4 +- .github/workflows/test_and_build.yml | 16 +- .pre-commit-config.yaml | 16 +- docs/env.yml | 2 +- graphblas/binary/ss.py | 2 + graphblas/core/base.py | 6 +- graphblas/core/descriptor.py | 1 + graphblas/core/expr.py | 2 +- graphblas/core/matrix.py | 4 +- graphblas/core/operator/binary.py | 6 +- graphblas/core/operator/indexunary.py | 7 +- graphblas/core/operator/semiring.py | 4 +- graphblas/core/operator/unary.py | 6 +- graphblas/core/ss/config.py | 57 +++++-- graphblas/core/ss/descriptor.py | 1 - graphblas/core/ss/matrix.py | 3 +- graphblas/core/ss/vector.py | 3 +- graphblas/core/vector.py | 7 +- graphblas/dtypes/__init__.py | 43 +++++ graphblas/{dtypes.py => dtypes/_core.py} | 196 +++++++++++------------ graphblas/dtypes/ss.py | 0 graphblas/indexunary/__init__.py | 14 +- graphblas/indexunary/ss.py | 5 + graphblas/io/_matrixmarket.py | 3 +- graphblas/op/ss.py | 2 + graphblas/select/__init__.py | 14 +- graphblas/select/ss.py | 5 + graphblas/semiring/ss.py | 2 + graphblas/ss/_core.py | 6 +- graphblas/tests/conftest.py | 19 +++ graphblas/tests/test_dtype.py | 10 +- graphblas/tests/test_io.py | 18 ++- graphblas/tests/test_matrix.py | 2 +- graphblas/tests/test_op.py | 1 + graphblas/unary/ss.py | 2 + pyproject.toml | 4 + scripts/check_versions.sh | 10 +- 37 files changed, 333 insertions(+), 170 deletions(-) create mode 100644 graphblas/dtypes/__init__.py rename graphblas/{dtypes.py => dtypes/_core.py} (69%) create mode 100644 graphblas/dtypes/ss.py create mode 100644 graphblas/indexunary/ss.py create mode 100644 graphblas/select/ss.py diff --git a/.github/workflows/imports.yml b/.github/workflows/imports.yml index 2b0b0ed9f..18e6f637c 100644 --- a/.github/workflows/imports.yml +++ b/.github/workflows/imports.yml @@ -54,5 +54,7 @@ jobs: python-version: ${{ needs.rngs.outputs.pyver }} # python-version: ${{ matrix.python-version }} - run: python -m pip install --upgrade pip + # - run: pip install --pre suitesparse-graphblas # Use if we need pre-release - run: pip install -e .[default] - - run: ./scripts/test_imports.sh + - name: Run test imports + run: ./scripts/test_imports.sh diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index d0c3f71fb..5630b8d35 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -170,7 +170,7 @@ jobs: nxver=$(python -c 'import random ; print(random.choice(["=2.7", "=2.8", "=3.0", "=3.1", ""]))') yamlver=$(python -c 'import random ; print(random.choice(["=5.4", "=6.0", ""]))') sparsever=$(python -c 'import random ; print(random.choice(["=0.13", "=0.14", ""]))') - fmmver=$(python -c 'import random ; print(random.choice(["=1.4", "=1.5", "=1.6", ""]))') + fmmver=$(python -c 'import random ; print(random.choice(["=1.4", "=1.5", "=1.6", "=1.7", ""]))') if [[ ${{ startsWith(steps.pyver.outputs.selected, '3.8') }} == true ]]; then npver=$(python -c 'import random ; print(random.choice(["=1.21", "=1.22", "=1.23", "=1.24", ""]))') spver=$(python -c 'import random ; print(random.choice(["=1.8", "=1.9", "=1.10", ""]))') @@ -178,17 +178,17 @@ jobs: akver=$(python -c 'import random ; print(random.choice(["=1.9", "=1.10", "=2.0", "=2.1", "=2.2", ""]))') elif [[ ${{ startsWith(steps.pyver.outputs.selected, '3.9') }} == true ]]; then npver=$(python -c 'import random ; print(random.choice(["=1.21", "=1.22", "=1.23", "=1.24", "=1.25", ""]))') - spver=$(python -c 'import random ; print(random.choice(["=1.8", "=1.9", "=1.10", ""]))') + spver=$(python -c 'import random ; print(random.choice(["=1.8", "=1.9", "=1.10", "=1.11", ""]))') pdver=$(python -c 'import random ; print(random.choice(["=1.2", "=1.3", "=1.4", "=1.5", "=2.0", ""]))') akver=$(python -c 'import random ; print(random.choice(["=1.9", "=1.10", "=2.0", "=2.1", "=2.2", ""]))') elif [[ ${{ startsWith(steps.pyver.outputs.selected, '3.10') }} == true ]]; then npver=$(python -c 'import random ; print(random.choice(["=1.21", "=1.22", "=1.23", "=1.24", "=1.25", ""]))') - spver=$(python -c 'import random ; print(random.choice(["=1.8", "=1.9", "=1.10", ""]))') + spver=$(python -c 'import random ; print(random.choice(["=1.8", "=1.9", "=1.10", "=1.11", ""]))') pdver=$(python -c 'import random ; print(random.choice(["=1.3", "=1.4", "=1.5", "=2.0", ""]))') akver=$(python -c 'import random ; print(random.choice(["=1.9", "=1.10", "=2.0", "=2.1", "=2.2", ""]))') else # Python 3.11 npver=$(python -c 'import random ; print(random.choice(["=1.23", "=1.24", "=1.25", ""]))') - spver=$(python -c 'import random ; print(random.choice(["=1.9", "=1.10", ""]))') + spver=$(python -c 'import random ; print(random.choice(["=1.9", "=1.10", "=1.11", ""]))') pdver=$(python -c 'import random ; print(random.choice(["=1.5", "=2.0", ""]))') akver=$(python -c 'import random ; print(random.choice(["=1.10", "=2.0", "=2.1", "=2.2", ""]))') fi @@ -204,20 +204,20 @@ jobs: # But, it's still useful for us to test with different versions! psg="" if [[ ${{ steps.sourcetype.outputs.selected}} == "conda-forge" ]] ; then - psgver=$(python -c 'import random ; print(random.choice(["=7.4.0", "=7.4.1", "=7.4.2", "=7.4.3.0", "=7.4.3.1", "=7.4.3.2", ""]))') + psgver=$(python -c 'import random ; print(random.choice(["=7.4.0", "=7.4.1", "=7.4.2", "=7.4.3.0", "=7.4.3.1", "=7.4.3.2"]))') psg=python-suitesparse-graphblas${psgver} elif [[ ${{ steps.sourcetype.outputs.selected}} == "wheel" ]] ; then - psgver=$(python -c 'import random ; print(random.choice(["==7.4.3.2", ""]))') + psgver=$(python -c 'import random ; print(random.choice(["==7.4.3.2"]))') elif [[ ${{ steps.sourcetype.outputs.selected}} == "source" ]] ; then # These should be exact versions - psgver=$(python -c 'import random ; print(random.choice(["==7.4.0.0", "==7.4.1.0", "==7.4.2.0", "==7.4.3.0", "==7.4.3.1", "==7.4.3.2", ""]))') + psgver=$(python -c 'import random ; print(random.choice(["==7.4.0.0", "==7.4.1.0", "==7.4.2.0", "==7.4.3.0", "==7.4.3.1", "==7.4.3.2"]))') else psgver="" fi if [[ ${npver} == "=1.25" ]] ; then numbaver="" if [[ ${spver} == "=1.8" ]] ; then - spver=$(python -c 'import random ; print(random.choice(["=1.9", "=1.10", ""]))') + spver=$(python -c 'import random ; print(random.choice(["=1.9", "=1.10", "=1.11", ""]))') fi elif [[ ${npver} == "=1.24" || ${{ startsWith(steps.pyver.outputs.selected, '3.11') }} == true ]] ; then numbaver=$(python -c 'import random ; print(random.choice(["=0.57", ""]))') diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4d0e5c0b6..f0ca307e8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,7 +21,7 @@ repos: - id: check-added-large-files - id: check-case-conflict - id: check-merge-conflict - - id: check-symlinks + # - id: check-symlinks - id: check-ast - id: check-toml - id: check-yaml @@ -39,7 +39,7 @@ repos: name: Validate pyproject.toml # I don't yet trust ruff to do what autoflake does - repo: https://github.com/PyCQA/autoflake - rev: v2.1.1 + rev: v2.2.0 hooks: - id: autoflake args: [--in-place] @@ -51,7 +51,7 @@ repos: - id: isort # Let's keep `pyupgrade` even though `ruff --fix` probably does most of it - repo: https://github.com/asottile/pyupgrade - rev: v3.4.0 + rev: v3.7.0 hooks: - id: pyupgrade args: [--py38-plus] @@ -66,7 +66,7 @@ repos: - id: black - id: black-jupyter - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.270 + rev: v0.0.275 hooks: - id: ruff args: [--fix-only, --show-fixes] @@ -79,22 +79,22 @@ repos: additional_dependencies: &flake8_dependencies # These versions need updated manually - flake8==6.0.0 - - flake8-bugbear==23.5.9 + - flake8-bugbear==23.6.5 - flake8-simplify==0.20.0 - repo: https://github.com/asottile/yesqa - rev: v1.4.0 + rev: v1.5.0 hooks: - id: yesqa additional_dependencies: *flake8_dependencies - repo: https://github.com/codespell-project/codespell - rev: v2.2.4 + rev: v2.2.5 hooks: - id: codespell types_or: [python, rst, markdown] additional_dependencies: [tomli] files: ^(graphblas|docs)/ - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.0.270 + rev: v0.0.275 hooks: - id: ruff - repo: https://github.com/sphinx-contrib/sphinx-lint diff --git a/docs/env.yml b/docs/env.yml index c0c4c8999..3636cfa2d 100644 --- a/docs/env.yml +++ b/docs/env.yml @@ -8,7 +8,7 @@ dependencies: # python-graphblas dependencies - donfig - numba - - python-suitesparse-graphblas>=7.4.0.0 + - python-suitesparse-graphblas>=7.4.0.0,<8 - pyyaml # extra dependencies - matplotlib diff --git a/graphblas/binary/ss.py b/graphblas/binary/ss.py index e45cbcda0..97852fc12 100644 --- a/graphblas/binary/ss.py +++ b/graphblas/binary/ss.py @@ -1,3 +1,5 @@ from ..core import operator +_delayed = {} + del operator diff --git a/graphblas/core/base.py b/graphblas/core/base.py index a4e48b612..42a4de9a1 100644 --- a/graphblas/core/base.py +++ b/graphblas/core/base.py @@ -348,7 +348,7 @@ def _update(self, expr, mask=None, accum=None, replace=False, input_mask=None, * return if opts: # Ignore opts for now - descriptor_lookup(**opts) + desc = descriptor_lookup(**opts) self.value = expr return @@ -371,7 +371,7 @@ def _update(self, expr, mask=None, accum=None, replace=False, input_mask=None, * else: if opts: # Ignore opts for now - descriptor_lookup(**opts) + desc = descriptor_lookup(**opts) self.value = expr return else: @@ -571,7 +571,7 @@ def _new(self, dtype, mask, name, is_cscalar=None, **opts): ): if opts: # Ignore opts for now - descriptor_lookup(**opts) + desc = descriptor_lookup(**opts) # noqa: F841 (keep desc in scope for context) if self._is_scalar and self._value._is_cscalar != is_cscalar: return self._value.dup(is_cscalar=is_cscalar, name=name) rv = self._value diff --git a/graphblas/core/descriptor.py b/graphblas/core/descriptor.py index 1e195e3fe..9ba3ae466 100644 --- a/graphblas/core/descriptor.py +++ b/graphblas/core/descriptor.py @@ -26,6 +26,7 @@ def __init__( self.mask_structure = mask_structure self.transpose_first = transpose_first self.transpose_second = transpose_second + self._context = None # Used by SuiteSparse:GraphBLAS @property def _carg(self): diff --git a/graphblas/core/expr.py b/graphblas/core/expr.py index 48839bcff..d803939a5 100644 --- a/graphblas/core/expr.py +++ b/graphblas/core/expr.py @@ -421,7 +421,7 @@ def _setitem(self, resolved_indexes, obj, *, is_submask): # Fast path using assignElement if self.opts: # Ignore opts for now - descriptor_lookup(**self.opts) + desc = descriptor_lookup(**self.opts) # noqa: F841 (keep desc in scope for context) self.parent._assign_element(resolved_indexes, obj) else: mask = self.kwargs.get("mask") diff --git a/graphblas/core/matrix.py b/graphblas/core/matrix.py index 2542ad00e..4696d8ead 100644 --- a/graphblas/core/matrix.py +++ b/graphblas/core/matrix.py @@ -665,7 +665,7 @@ def dup(self, dtype=None, *, clear=False, mask=None, name=None, **opts): else: if opts: # Ignore opts for now - descriptor_lookup(**opts) + desc = descriptor_lookup(**opts) # noqa: F841 (keep desc in scope for context) new_mat = ffi_new("GrB_Matrix*") rv = Matrix._from_obj(new_mat, self.dtype, self._nrows, self._ncols, name=name) call("GrB_Matrix_dup", [_Pointer(rv), self]) @@ -2707,7 +2707,7 @@ def _extract_element( result = Scalar(dtype, is_cscalar=is_cscalar, name=name) if opts: # Ignore opts for now - descriptor_lookup(**opts) + desc = descriptor_lookup(**opts) # noqa: F841 (keep desc in scope for context) if is_cscalar: dtype_name = "UDT" if dtype._is_udt else dtype.name if ( diff --git a/graphblas/core/operator/binary.py b/graphblas/core/operator/binary.py index 406405a80..434ad91cb 100644 --- a/graphblas/core/operator/binary.py +++ b/graphblas/core/operator/binary.py @@ -19,10 +19,10 @@ UINT16, UINT32, UINT64, - _sample_values, _supports_complex, lookup_dtype, ) +from ...dtypes._core import _sample_values from ...exceptions import UdfParseError, check_status_carg from .. import _has_numba, _supports_udfs, ffi, lib from ..expr import InfixExprBase @@ -506,7 +506,7 @@ def binary_wrapper(z, x, y): # pragma: no cover (numba) type_.gb_obj, ), "BinaryOp", - new_binary, + new_binary[0], ) op = TypedUserBinaryOp(new_type_obj, name, type_, ret_type, new_binary[0]) new_type_obj._add(op) @@ -611,7 +611,7 @@ def binary_wrapper(z_ptr, x_ptr, y_ptr): # pragma: no cover (numba) new_binary, binary_wrapper.cffi, ret_type._carg, dtype._carg, dtype2._carg ), "BinaryOp", - new_binary, + new_binary[0], ) op = TypedUserBinaryOp( self, diff --git a/graphblas/core/operator/indexunary.py b/graphblas/core/operator/indexunary.py index f6637ae6d..8b1211258 100644 --- a/graphblas/core/operator/indexunary.py +++ b/graphblas/core/operator/indexunary.py @@ -3,7 +3,8 @@ from types import FunctionType from ... import _STANDARD_OPERATOR_NAMES, indexunary, select -from ...dtypes import BOOL, FP64, INT8, INT64, UINT64, _sample_values, lookup_dtype +from ...dtypes import BOOL, FP64, INT8, INT64, UINT64, lookup_dtype +from ...dtypes._core import _sample_values from ...exceptions import UdfParseError, check_status_carg from .. import _has_numba, ffi, lib from .base import OpBase, ParameterizedUdf, TypedOpBase, _call_op, _deserialize_parameterized @@ -193,7 +194,7 @@ def indexunary_wrapper(z, x, row, col, y): # pragma: no cover (numba) type_.gb_obj, ), "IndexUnaryOp", - new_indexunary, + new_indexunary[0], ) op = cls._typed_user_class(new_type_obj, name, type_, ret_type, new_indexunary[0]) new_type_obj._add(op) @@ -225,7 +226,7 @@ def _compile_udt(self, dtype, dtype2): new_indexunary, indexunary_wrapper.cffi, ret_type._carg, dtype._carg, dtype2._carg ), "IndexUnaryOp", - new_indexunary, + new_indexunary[0], ) op = TypedUserIndexUnaryOp( self, diff --git a/graphblas/core/operator/semiring.py b/graphblas/core/operator/semiring.py index 035a1c43b..d367461f6 100644 --- a/graphblas/core/operator/semiring.py +++ b/graphblas/core/operator/semiring.py @@ -228,7 +228,7 @@ def _build(cls, name, monoid, binaryop, *, anonymous=False): check_status_carg( lib.GrB_Semiring_new(new_semiring, monoid[binary_out].gb_obj, binary_func.gb_obj), "Semiring", - new_semiring, + new_semiring[0], ) ret_type = monoid[binary_out].return_type op = TypedUserSemiring( @@ -254,7 +254,7 @@ def _compile_udt(self, dtype, dtype2): ret_type = monoid.return_type new_semiring = ffi_new("GrB_Semiring*") status = lib.GrB_Semiring_new(new_semiring, monoid.gb_obj, binaryop.gb_obj) - check_status_carg(status, "Semiring", new_semiring) + check_status_carg(status, "Semiring", new_semiring[0]) op = TypedUserSemiring( new_semiring, self.name, diff --git a/graphblas/core/operator/unary.py b/graphblas/core/operator/unary.py index a02445836..11ada4e48 100644 --- a/graphblas/core/operator/unary.py +++ b/graphblas/core/operator/unary.py @@ -15,10 +15,10 @@ UINT16, UINT32, UINT64, - _sample_values, _supports_complex, lookup_dtype, ) +from ...dtypes._core import _sample_values from ...exceptions import UdfParseError, check_status_carg from .. import _has_numba, ffi, lib from ..utils import output_type @@ -239,7 +239,7 @@ def unary_wrapper(z, x): new_unary, unary_wrapper.cffi, ret_type.gb_obj, type_.gb_obj ), "UnaryOp", - new_unary, + new_unary[0], ) op = TypedUserUnaryOp(new_type_obj, name, type_, ret_type, new_unary[0]) new_type_obj._add(op) @@ -264,7 +264,7 @@ def _compile_udt(self, dtype, dtype2): check_status_carg( lib.GrB_UnaryOp_new(new_unary, unary_wrapper.cffi, ret_type._carg, dtype._carg), "UnaryOp", - new_unary, + new_unary[0], ) op = TypedUserUnaryOp(self, self.name, dtype, ret_type, new_unary[0]) self._udt_types[dtype] = ret_type diff --git a/graphblas/core/ss/config.py b/graphblas/core/ss/config.py index ca91cc198..433716bb3 100644 --- a/graphblas/core/ss/config.py +++ b/graphblas/core/ss/config.py @@ -12,6 +12,9 @@ class BaseConfig(MutableMapping): # Subclasses should redefine these _get_function = None _set_function = None + _context_get_function = "GxB_Context_get" + _context_set_function = "GxB_Context_set" + _context_keys = set() _null_valid = {} _options = {} _defaults = {} @@ -28,7 +31,7 @@ class BaseConfig(MutableMapping): "GxB_Format_Value", } - def __init__(self, parent=None): + def __init__(self, parent=None, context=None): cls = type(self) if not cls._initialized: cls._reverse_enumerations = {} @@ -51,6 +54,7 @@ def __init__(self, parent=None): rd[k] = k cls._initialized = True self._parent = parent + self._context = context def __delitem__(self, key): raise TypeError("Configuration options can't be deleted.") @@ -61,19 +65,27 @@ def __getitem__(self, key): raise KeyError(key) key_obj, ctype = self._options[key] is_bool = ctype == "bool" + if is_context := (key in self._context_keys): + get_function_base = self._context_get_function + else: + get_function_base = self._get_function if ctype in self._int32_ctypes: ctype = "int32_t" - get_function_name = f"{self._get_function}_INT32" + get_function_name = f"{get_function_base}_INT32" elif ctype.startswith("int64_t"): - get_function_name = f"{self._get_function}_INT64" + get_function_name = f"{get_function_base}_INT64" elif ctype.startswith("double"): - get_function_name = f"{self._get_function}_FP64" + get_function_name = f"{get_function_base}_FP64" + elif ctype.startswith("char"): + get_function_name = f"{get_function_base}_CHAR" else: # pragma: no cover (sanity) raise ValueError(ctype) get_function = getattr(lib, get_function_name) is_array = "[" in ctype val_ptr = ffi.new(ctype if is_array else f"{ctype}*") - if self._parent is None: + if is_context: + info = get_function(self._context._carg, key_obj, val_ptr) + elif self._parent is None: info = get_function(key_obj, val_ptr) else: info = get_function(self._parent._carg, key_obj, val_ptr) @@ -93,6 +105,8 @@ def __getitem__(self, key): return rv if is_bool: return bool(val_ptr[0]) + if ctype.startswith("char"): + return ffi.string(val_ptr[0]).decode() return val_ptr[0] raise _error_code_lookup[info](f"Failed to get info for {key!r}") # pragma: no cover @@ -103,15 +117,21 @@ def __setitem__(self, key, val): if key in self._read_only: raise ValueError(f"Config option {key!r} is read-only") key_obj, ctype = self._options[key] + if is_context := (key in self._context_keys): + set_function_base = self._context_set_function + else: + set_function_base = self._set_function if ctype in self._int32_ctypes: ctype = "int32_t" - set_function_name = f"{self._set_function}_INT32" + set_function_name = f"{set_function_base}_INT32" elif ctype == "double": - set_function_name = f"{self._set_function}_FP64" + set_function_name = f"{set_function_base}_FP64" elif ctype.startswith("int64_t["): - set_function_name = f"{self._set_function}_INT64_ARRAY" + set_function_name = f"{set_function_base}_INT64_ARRAY" elif ctype.startswith("double["): - set_function_name = f"{self._set_function}_FP64_ARRAY" + set_function_name = f"{set_function_base}_FP64_ARRAY" + elif ctype.startswith("char"): + set_function_name = f"{set_function_base}_CHAR" else: # pragma: no cover (sanity) raise ValueError(ctype) set_function = getattr(lib, set_function_name) @@ -154,9 +174,19 @@ def __setitem__(self, key, val): f"expected {size}, got {vals.size}: {val}" ) val_obj = ffi.from_buffer(ctype, vals) + elif ctype.startswith("char"): + val_obj = ffi.new("char[]", val.encode()) else: val_obj = ffi.cast(ctype, val) - if self._parent is None: + if is_context: + if self._context is None: + from .context import Context + + self._context = Context(engage=False) + self._context._engage() # Disengage when context goes out of scope + self._parent._context = self._context # Set context to descriptor + info = set_function(self._context._carg, key_obj, val_obj) + elif self._parent is None: info = set_function(key_obj, val_obj) else: info = set_function(self._parent._carg, key_obj, val_obj) @@ -174,7 +204,12 @@ def __len__(self): return len(self._options) def __repr__(self): - return "{" + ",\n ".join(f"{k!r}: {v!r}" for k, v in self.items()) + "}" + return ( + type(self).__name__ + + "({" + + ",\n ".join(f"{k!r}: {v!r}" for k, v in self.items()) + + "})" + ) def _ipython_key_completions_(self): # pragma: no cover (ipython) return list(self) diff --git a/graphblas/core/ss/descriptor.py b/graphblas/core/ss/descriptor.py index 2f7d11ffa..43553f5ea 100644 --- a/graphblas/core/ss/descriptor.py +++ b/graphblas/core/ss/descriptor.py @@ -90,7 +90,6 @@ class _DescriptorConfig(BaseConfig): "sort": False, "secure_import": False, } - _count = 0 def __init__(self): gb_obj = ffi_new("GrB_Descriptor*") diff --git a/graphblas/core/ss/matrix.py b/graphblas/core/ss/matrix.py index 64aa43a96..990d692b9 100644 --- a/graphblas/core/ss/matrix.py +++ b/graphblas/core/ss/matrix.py @@ -7,7 +7,8 @@ import graphblas as gb from ... import binary, monoid -from ...dtypes import _INDEX, BOOL, INT64, UINT64, _string_to_dtype, lookup_dtype +from ...dtypes import _INDEX, BOOL, INT64, UINT64, lookup_dtype +from ...dtypes._core import _string_to_dtype from ...exceptions import _error_code_lookup, check_status, check_status_carg from .. import NULL, _has_numba, ffi, lib from ..base import call diff --git a/graphblas/core/ss/vector.py b/graphblas/core/ss/vector.py index 1babc556e..ff9e233eb 100644 --- a/graphblas/core/ss/vector.py +++ b/graphblas/core/ss/vector.py @@ -6,7 +6,8 @@ import graphblas as gb from ... import binary, monoid -from ...dtypes import _INDEX, INT64, UINT64, _string_to_dtype, lookup_dtype +from ...dtypes import _INDEX, INT64, UINT64, lookup_dtype +from ...dtypes._core import _string_to_dtype from ...exceptions import _error_code_lookup, check_status, check_status_carg from .. import NULL, ffi, lib from ..base import call diff --git a/graphblas/core/vector.py b/graphblas/core/vector.py index d2ddee372..cd5b992ba 100644 --- a/graphblas/core/vector.py +++ b/graphblas/core/vector.py @@ -612,7 +612,7 @@ def dup(self, dtype=None, *, clear=False, mask=None, name=None, **opts): else: if opts: # Ignore opts for now - descriptor_lookup(**opts) + desc = descriptor_lookup(**opts) # noqa: F841 (keep desc in scope for context) rv = Vector._from_obj(ffi_new("GrB_Vector*"), self.dtype, self._size, name=name) call("GrB_Vector_dup", [_Pointer(rv), self]) return rv @@ -1757,7 +1757,7 @@ def _extract_element( result = Scalar(dtype, is_cscalar=is_cscalar, name=name) if opts: # Ignore opts for now - descriptor_lookup(**opts) + desc = descriptor_lookup(**opts) # noqa: F841 (keep desc in scope for context) if is_cscalar: dtype_name = "UDT" if dtype._is_udt else dtype.name if ( @@ -2177,6 +2177,9 @@ def dup(self, dtype=None, *, clear=False, mask=None, name=None, **opts): if clear: if dtype is None: dtype = self.dtype + if opts: + # Ignore opts for now + desc = descriptor_lookup(**opts) # noqa: F841 (keep desc in scope for context) return self.output_type(dtype, *self.shape, name=name) return self.new(dtype, mask=mask, name=name, **opts) diff --git a/graphblas/dtypes/__init__.py b/graphblas/dtypes/__init__.py new file mode 100644 index 000000000..0d26a44a0 --- /dev/null +++ b/graphblas/dtypes/__init__.py @@ -0,0 +1,43 @@ +from ._core import ( + _INDEX, + BOOL, + FP32, + FP64, + INT8, + INT16, + INT32, + INT64, + UINT8, + UINT16, + UINT32, + UINT64, + DataType, + _supports_complex, + lookup_dtype, + register_anonymous, + register_new, + unify, +) + +if _supports_complex: + from ._core import FC32, FC64 + + +def __dir__(): + return globals().keys() | {"ss"} + + +def __getattr__(key): + if key == "ss": + from .. import backend + + if backend != "suitesparse": + raise AttributeError( + f'module {__name__!r} only has attribute "ss" when backend is "suitesparse"' + ) + from importlib import import_module + + ss = import_module(".ss", __name__) + globals()["ss"] = ss + return ss + raise AttributeError(f"module {__name__!r} has no attribute {key!r}") diff --git a/graphblas/dtypes.py b/graphblas/dtypes/_core.py similarity index 69% rename from graphblas/dtypes.py rename to graphblas/dtypes/_core.py index 61b297c13..345c1be81 100644 --- a/graphblas/dtypes.py +++ b/graphblas/dtypes/_core.py @@ -1,20 +1,16 @@ -import warnings as _warnings +import warnings -import numpy as _np -from numpy import promote_types as _promote_types -from numpy import result_type as _result_type +import numpy as np +from numpy import promote_types, result_type -from . import backend -from .core import NULL as _NULL -from .core import _has_numba -from .core import ffi as _ffi -from .core import lib as _lib +from .. import backend, dtypes +from ..core import NULL, _has_numba, ffi, lib if _has_numba: - import numba as _numba + import numba # Default assumption unless FC32/FC64 are found in lib -_supports_complex = hasattr(_lib, "GrB_FC64") or hasattr(_lib, "GxB_FC64") +_supports_complex = hasattr(lib, "GrB_FC64") or hasattr(lib, "GxB_FC64") class DataType: @@ -26,7 +22,7 @@ def __init__(self, name, gb_obj, gb_name, c_type, numba_type, np_type): self.gb_name = gb_name self.c_type = c_type self.numba_type = numba_type - self.np_type = _np.dtype(np_type) + self.np_type = np.dtype(np_type) def __repr__(self): return self.name @@ -62,7 +58,7 @@ def _carg(self): @property def _is_anonymous(self): - return globals().get(self.name) is not self + return getattr(dtypes, self.name, None) is not self @property def _is_udt(self): @@ -80,27 +76,29 @@ def _deserialize(name, dtype, is_anonymous): def register_new(name, dtype): if not name.isidentifier(): raise ValueError(f"`name` argument must be a valid Python identifier; got: {name!r}") - if name in _registry or name in globals(): + if name in _registry or hasattr(dtypes, name): raise ValueError(f"{name!r} name for dtype is unavailable") rv = register_anonymous(dtype, name) _registry[name] = rv - globals()[name] = rv + setattr(dtypes, name, rv) return rv def register_anonymous(dtype, name=None): try: - dtype = _np.dtype(dtype) + dtype = np.dtype(dtype) except TypeError: if isinstance(dtype, dict): # Allow dtypes such as `{'x': int, 'y': float}` for convenience - dtype = _np.dtype([(key, lookup_dtype(val).np_type) for key, val in dtype.items()]) + dtype = np.dtype( + [(key, lookup_dtype(val).np_type) for key, val in dtype.items()], align=True + ) elif isinstance(dtype, str) and "[" in dtype and dtype.endswith("]"): # Allow dtypes such as `"INT64[3, 4]"` for convenience base_dtype, shape = dtype.split("[", 1) base_dtype = lookup_dtype(base_dtype) - shape = _np.lib.format.safe_eval(f"[{shape}") - dtype = _np.dtype((base_dtype.np_type, shape)) + shape = np.lib.format.safe_eval(f"[{shape}") + dtype = np.dtype((base_dtype.np_type, shape)) else: raise if dtype in _registry: @@ -114,36 +112,36 @@ def register_anonymous(dtype, name=None): if dtype.hasobject: raise ValueError("dtype must not allow Python objects") - from .exceptions import check_status_carg + from ..exceptions import check_status_carg - gb_obj = _ffi.new("GrB_Type*") + gb_obj = ffi.new("GrB_Type*") if backend == "suitesparse": # We name this so that we can serialize and deserialize UDTs # We don't yet have C definitions np_repr = _dtype_to_string(dtype).encode() - if len(np_repr) > _lib.GxB_MAX_NAME_LEN: + if len(np_repr) > lib.GxB_MAX_NAME_LEN: msg = ( f"UDT repr is too large to serialize ({len(repr(dtype).encode())} > " - f"{_lib.GxB_MAX_NAME_LEN})." + f"{lib.GxB_MAX_NAME_LEN})." ) if name is not None: - np_repr = name.encode()[: _lib.GxB_MAX_NAME_LEN] + np_repr = name.encode()[: lib.GxB_MAX_NAME_LEN] else: - np_repr = np_repr[: _lib.GxB_MAX_NAME_LEN] - _warnings.warn( + np_repr = np_repr[: lib.GxB_MAX_NAME_LEN] + warnings.warn( f"{msg}. It will use the following name, " f"and the dtype may need to be specified when deserializing: {np_repr}", stacklevel=2, ) - status = _lib.GxB_Type_new(gb_obj, dtype.itemsize, np_repr, _NULL) + status = lib.GxB_Type_new(gb_obj, dtype.itemsize, np_repr, NULL) else: - status = _lib.GrB_Type_new(gb_obj, dtype.itemsize) + status = lib.GrB_Type_new(gb_obj, dtype.itemsize) check_status_carg(status, "Type", gb_obj[0]) # For now, let's use "opaque" unsigned bytes for the c type. if name is None: name = _default_name(dtype) - numba_type = _numba.typeof(dtype).dtype if _has_numba else None + numba_type = numba.typeof(dtype).dtype if _has_numba else None rv = DataType(name, gb_obj, None, f"uint8_t[{dtype.itemsize}]", numba_type, dtype) _registry[gb_obj] = rv _registry[dtype] = rv @@ -155,153 +153,153 @@ def register_anonymous(dtype, name=None): BOOL = DataType( "BOOL", - _lib.GrB_BOOL, + lib.GrB_BOOL, "GrB_BOOL", "_Bool", - _numba.types.bool_ if _has_numba else None, - _np.bool_, + numba.types.bool_ if _has_numba else None, + np.bool_, ) INT8 = DataType( - "INT8", _lib.GrB_INT8, "GrB_INT8", "int8_t", _numba.types.int8 if _has_numba else None, _np.int8 + "INT8", lib.GrB_INT8, "GrB_INT8", "int8_t", numba.types.int8 if _has_numba else None, np.int8 ) UINT8 = DataType( "UINT8", - _lib.GrB_UINT8, + lib.GrB_UINT8, "GrB_UINT8", "uint8_t", - _numba.types.uint8 if _has_numba else None, - _np.uint8, + numba.types.uint8 if _has_numba else None, + np.uint8, ) INT16 = DataType( "INT16", - _lib.GrB_INT16, + lib.GrB_INT16, "GrB_INT16", "int16_t", - _numba.types.int16 if _has_numba else None, - _np.int16, + numba.types.int16 if _has_numba else None, + np.int16, ) UINT16 = DataType( "UINT16", - _lib.GrB_UINT16, + lib.GrB_UINT16, "GrB_UINT16", "uint16_t", - _numba.types.uint16 if _has_numba else None, - _np.uint16, + numba.types.uint16 if _has_numba else None, + np.uint16, ) INT32 = DataType( "INT32", - _lib.GrB_INT32, + lib.GrB_INT32, "GrB_INT32", "int32_t", - _numba.types.int32 if _has_numba else None, - _np.int32, + numba.types.int32 if _has_numba else None, + np.int32, ) UINT32 = DataType( "UINT32", - _lib.GrB_UINT32, + lib.GrB_UINT32, "GrB_UINT32", "uint32_t", - _numba.types.uint32 if _has_numba else None, - _np.uint32, + numba.types.uint32 if _has_numba else None, + np.uint32, ) INT64 = DataType( "INT64", - _lib.GrB_INT64, + lib.GrB_INT64, "GrB_INT64", "int64_t", - _numba.types.int64 if _has_numba else None, - _np.int64, + numba.types.int64 if _has_numba else None, + np.int64, ) # _Index (like UINT64) is for internal use only and shouldn't be exposed to the user _INDEX = DataType( "UINT64", - _lib.GrB_UINT64, + lib.GrB_UINT64, "GrB_Index", "GrB_Index", - _numba.types.uint64 if _has_numba else None, - _np.uint64, + numba.types.uint64 if _has_numba else None, + np.uint64, ) UINT64 = DataType( "UINT64", - _lib.GrB_UINT64, + lib.GrB_UINT64, "GrB_UINT64", "uint64_t", - _numba.types.uint64 if _has_numba else None, - _np.uint64, + numba.types.uint64 if _has_numba else None, + np.uint64, ) FP32 = DataType( "FP32", - _lib.GrB_FP32, + lib.GrB_FP32, "GrB_FP32", "float", - _numba.types.float32 if _has_numba else None, - _np.float32, + numba.types.float32 if _has_numba else None, + np.float32, ) FP64 = DataType( "FP64", - _lib.GrB_FP64, + lib.GrB_FP64, "GrB_FP64", "double", - _numba.types.float64 if _has_numba else None, - _np.float64, + numba.types.float64 if _has_numba else None, + np.float64, ) -if _supports_complex and hasattr(_lib, "GxB_FC32"): +if _supports_complex and hasattr(lib, "GxB_FC32"): FC32 = DataType( "FC32", - _lib.GxB_FC32, + lib.GxB_FC32, "GxB_FC32", "float _Complex", - _numba.types.complex64 if _has_numba else None, - _np.complex64, + numba.types.complex64 if _has_numba else None, + np.complex64, ) -if _supports_complex and hasattr(_lib, "GrB_FC32"): # pragma: no cover (unused) +if _supports_complex and hasattr(lib, "GrB_FC32"): # pragma: no cover (unused) FC32 = DataType( "FC32", - _lib.GrB_FC32, + lib.GrB_FC32, "GrB_FC32", "float _Complex", - _numba.types.complex64 if _has_numba else None, - _np.complex64, + numba.types.complex64 if _has_numba else None, + np.complex64, ) -if _supports_complex and hasattr(_lib, "GxB_FC64"): +if _supports_complex and hasattr(lib, "GxB_FC64"): FC64 = DataType( "FC64", - _lib.GxB_FC64, + lib.GxB_FC64, "GxB_FC64", "double _Complex", - _numba.types.complex128 if _has_numba else None, - _np.complex128, + numba.types.complex128 if _has_numba else None, + np.complex128, ) -if _supports_complex and hasattr(_lib, "GrB_FC64"): # pragma: no cover (unused) +if _supports_complex and hasattr(lib, "GrB_FC64"): # pragma: no cover (unused) FC64 = DataType( "FC64", - _lib.GrB_FC64, + lib.GrB_FC64, "GrB_FC64", "double _Complex", - _numba.types.complex128 if _has_numba else None, - _np.complex128, + numba.types.complex128 if _has_numba else None, + np.complex128, ) # Used for testing user-defined functions _sample_values = { - INT8: _np.int8(1), - UINT8: _np.uint8(1), - INT16: _np.int16(1), - UINT16: _np.uint16(1), - INT32: _np.int32(1), - UINT32: _np.uint32(1), - INT64: _np.int64(1), - UINT64: _np.uint64(1), - FP32: _np.float32(0.5), - FP64: _np.float64(0.5), - BOOL: _np.bool_(True), + INT8: np.int8(1), + UINT8: np.uint8(1), + INT16: np.int16(1), + UINT16: np.uint16(1), + INT32: np.int32(1), + UINT32: np.uint32(1), + INT64: np.int64(1), + UINT64: np.uint64(1), + FP32: np.float32(0.5), + FP64: np.float64(0.5), + BOOL: np.bool_(True), } if _supports_complex: _sample_values.update( { - FC32: _np.complex64(complex(0, 0.5)), - FC64: _np.complex128(complex(0, 0.5)), + FC32: np.complex64(complex(0, 0.5)), + FC64: np.complex128(complex(0, 0.5)), } ) @@ -390,10 +388,10 @@ def unify(type1, type2, *, is_left_scalar=False, is_right_scalar=False): return type1 if is_left_scalar: if not is_right_scalar: - return lookup_dtype(_result_type(_np.array(0, type1.np_type), type2.np_type)) + return lookup_dtype(result_type(np.array(0, type1.np_type), type2.np_type)) elif is_right_scalar: - return lookup_dtype(_result_type(type1.np_type, _np.array(0, type2.np_type))) - return lookup_dtype(_promote_types(type1.np_type, type2.np_type)) + return lookup_dtype(result_type(type1.np_type, np.array(0, type2.np_type))) + return lookup_dtype(promote_types(type1.np_type, type2.np_type)) def _default_name(dtype): @@ -423,7 +421,7 @@ def _dtype_to_string(dtype): >>> dtype == new_dtype True """ - if isinstance(dtype, _np.dtype) and dtype not in _registry: + if isinstance(dtype, np.dtype) and dtype not in _registry: np_type = dtype else: dtype = lookup_dtype(dtype) @@ -432,11 +430,11 @@ def _dtype_to_string(dtype): np_type = dtype.np_type s = str(np_type) try: - if _np.dtype(_np.lib.format.safe_eval(s)) == np_type: # pragma: no branch (safety) + if np.dtype(np.lib.format.safe_eval(s)) == np_type: # pragma: no branch (safety) return s except Exception: pass - if _np.dtype(np_type.str) != np_type: # pragma: no cover (safety) + if np.dtype(np_type.str) != np_type: # pragma: no cover (safety) raise ValueError(f"Unable to reliably convert dtype to string and back: {dtype}") return repr(np_type.str) @@ -451,5 +449,5 @@ def _string_to_dtype(s): return lookup_dtype(s) except Exception: pass - np_type = _np.dtype(_np.lib.format.safe_eval(s)) + np_type = np.dtype(np.lib.format.safe_eval(s)) return lookup_dtype(np_type) diff --git a/graphblas/dtypes/ss.py b/graphblas/dtypes/ss.py new file mode 100644 index 000000000..e69de29bb diff --git a/graphblas/indexunary/__init__.py b/graphblas/indexunary/__init__.py index 472231597..a3cb06608 100644 --- a/graphblas/indexunary/__init__.py +++ b/graphblas/indexunary/__init__.py @@ -4,7 +4,7 @@ def __dir__(): - return globals().keys() | _delayed.keys() + return globals().keys() | _delayed.keys() | {"ss"} def __getattr__(key): @@ -13,6 +13,18 @@ def __getattr__(key): rv = func(**kwargs) globals()[key] = rv return rv + if key == "ss": + from .. import backend + + if backend != "suitesparse": + raise AttributeError( + f'module {__name__!r} only has attribute "ss" when backend is "suitesparse"' + ) + from importlib import import_module + + ss = import_module(".ss", __name__) + globals()["ss"] = ss + return ss raise AttributeError(f"module {__name__!r} has no attribute {key!r}") diff --git a/graphblas/indexunary/ss.py b/graphblas/indexunary/ss.py new file mode 100644 index 000000000..97852fc12 --- /dev/null +++ b/graphblas/indexunary/ss.py @@ -0,0 +1,5 @@ +from ..core import operator + +_delayed = {} + +del operator diff --git a/graphblas/io/_matrixmarket.py b/graphblas/io/_matrixmarket.py index 294bcfa1e..0946580c8 100644 --- a/graphblas/io/_matrixmarket.py +++ b/graphblas/io/_matrixmarket.py @@ -36,7 +36,6 @@ def mmread(source, engine="auto", *, dup_op=None, name=None, **kwargs): try: # scipy is currently needed for *all* engines from scipy.io import mmread - from scipy.sparse import isspmatrix_coo except ImportError: # pragma: no cover (import) raise ImportError("scipy is required to read Matrix Market files") from None engine = engine.lower() @@ -54,7 +53,7 @@ def mmread(source, engine="auto", *, dup_op=None, name=None, **kwargs): f'Bad engine value: {engine!r}. Must be "auto", "scipy", "fmm", or "fast_matrix_market"' ) array = mmread(source, **kwargs) - if isspmatrix_coo(array): + if array.format == "coo": nrows, ncols = array.shape return Matrix.from_coo( array.row, array.col, array.data, nrows=nrows, ncols=ncols, dup_op=dup_op, name=name diff --git a/graphblas/op/ss.py b/graphblas/op/ss.py index e45cbcda0..97852fc12 100644 --- a/graphblas/op/ss.py +++ b/graphblas/op/ss.py @@ -1,3 +1,5 @@ from ..core import operator +_delayed = {} + del operator diff --git a/graphblas/select/__init__.py b/graphblas/select/__init__.py index 72aa8d226..aaf8e12d0 100644 --- a/graphblas/select/__init__.py +++ b/graphblas/select/__init__.py @@ -8,7 +8,7 @@ def __dir__(): - return globals().keys() | _delayed.keys() + return globals().keys() | _delayed.keys() | {"ss"} def __getattr__(key): @@ -17,6 +17,18 @@ def __getattr__(key): rv = func(**kwargs) globals()[key] = rv return rv + if key == "ss": + from .. import backend + + if backend != "suitesparse": + raise AttributeError( + f'module {__name__!r} only has attribute "ss" when backend is "suitesparse"' + ) + from importlib import import_module + + ss = import_module(".ss", __name__) + globals()["ss"] = ss + return ss raise AttributeError(f"module {__name__!r} has no attribute {key!r}") diff --git a/graphblas/select/ss.py b/graphblas/select/ss.py new file mode 100644 index 000000000..97852fc12 --- /dev/null +++ b/graphblas/select/ss.py @@ -0,0 +1,5 @@ +from ..core import operator + +_delayed = {} + +del operator diff --git a/graphblas/semiring/ss.py b/graphblas/semiring/ss.py index e45cbcda0..97852fc12 100644 --- a/graphblas/semiring/ss.py +++ b/graphblas/semiring/ss.py @@ -1,3 +1,5 @@ from ..core import operator +_delayed = {} + del operator diff --git a/graphblas/ss/_core.py b/graphblas/ss/_core.py index ec5a89504..53287f1a5 100644 --- a/graphblas/ss/_core.py +++ b/graphblas/ss/_core.py @@ -2,6 +2,7 @@ from ..core import ffi, lib from ..core.base import _expect_type +from ..core.descriptor import lookup as descriptor_lookup from ..core.matrix import Matrix, TransposedMatrix from ..core.scalar import _as_scalar from ..core.ss.config import BaseConfig @@ -52,6 +53,9 @@ def diag(x, k=0, dtype=None, *, name=None, **opts): dtype = x.dtype typ = type(x) if typ is Vector: + if opts: + # Ignore opts for now + desc = descriptor_lookup(**opts) # noqa: F841 (keep desc in scope for context) size = x._size + abs(k.value) rv = Matrix(dtype, nrows=size, ncols=size, name=name) rv.ss.build_diag(x, k) @@ -120,7 +124,7 @@ class GlobalConfig(BaseConfig): memory_pool : List[int] burble : bool Enable diagnostic printing from SuiteSparse:GraphBLAS - print_1based: bool + print_1based : bool gpu_control : str, {"always", "never"} gpu_chunk : double diff --git a/graphblas/tests/conftest.py b/graphblas/tests/conftest.py index 0d1f4008a..0137ea59e 100644 --- a/graphblas/tests/conftest.py +++ b/graphblas/tests/conftest.py @@ -1,4 +1,5 @@ import atexit +import contextlib import functools import itertools import platform @@ -114,6 +115,24 @@ def ic(): # pragma: no cover (debug) return icecream.ic +@contextlib.contextmanager +def burble(): + """Show the burble diagnostics within a context.""" + # Don't keep track of previous state; always set to False when done + gb.ss.config["burble"] = True + try: + yield + finally: + gb.ss.config["burble"] = False + + +@pytest.fixture(scope="session", autouse=True) +def burble_all(): # pragma: no cover (debug) + """Show the burble diagnostics for the entire test.""" + with burble(): + yield burble + + def autocompute(func): @functools.wraps(func) def inner(*args, **kwargs): diff --git a/graphblas/tests/test_dtype.py b/graphblas/tests/test_dtype.py index 66c19cce5..233e9b65a 100644 --- a/graphblas/tests/test_dtype.py +++ b/graphblas/tests/test_dtype.py @@ -123,7 +123,7 @@ def test_dtype_bad_comparison(): def test_dtypes_match_numpy(): - for key, val in dtypes._registry.items(): + for key, val in dtypes._core._registry.items(): try: if key is int or (isinstance(key, str) and key == "int"): # For win64, numpy treats int as int32, not int64 @@ -137,7 +137,7 @@ def test_dtypes_match_numpy(): def test_pickle(): - for val in dtypes._registry.values(): + for val in dtypes._core._registry.values(): s = pickle.dumps(val) val2 = pickle.loads(s) if val._is_udt: # pragma: no cover @@ -205,7 +205,7 @@ def test_auto_register(): def test_default_names(): - from graphblas.dtypes import _default_name + from graphblas.dtypes._core import _default_name assert _default_name(np.dtype([("x", np.int32), ("y", np.float64)], align=True)) == ( "{'x': INT32, 'y': FP64}" @@ -230,9 +230,9 @@ def test_dtype_to_from_string(): except Exception: pass for dtype in types: - s = dtypes._dtype_to_string(dtype) + s = dtypes._core._dtype_to_string(dtype) try: - dtype2 = dtypes._string_to_dtype(s) + dtype2 = dtypes._core._string_to_dtype(s) except Exception: with pytest.raises(ValueError, match="Unknown dtype"): lookup_dtype(dtype) diff --git a/graphblas/tests/test_io.py b/graphblas/tests/test_io.py index 671b12bd6..bf2ca2015 100644 --- a/graphblas/tests/test_io.py +++ b/graphblas/tests/test_io.py @@ -59,18 +59,24 @@ def test_vector_to_from_numpy(): csr = gb.io.to_scipy_sparse(v, "csr") assert csr.nnz == 2 - assert ss.isspmatrix_csr(csr) + # 2023-06-25: scipy 1.11.0 added `sparray` and changed e.g. `ss.isspmatrix_csr` + assert isinstance(csr, getattr(ss, "sparray", ss.spmatrix)) + assert csr.format == "csr" np.testing.assert_array_equal(csr.toarray(), np.array([[0.0, 2.0, 4.1]])) csc = gb.io.to_scipy_sparse(v, "csc") assert csc.nnz == 2 - assert ss.isspmatrix_csc(csc) + # 2023-06-25: scipy 1.11.0 added `sparray` and changed e.g. `ss.isspmatrix_csc` + assert isinstance(csc, getattr(ss, "sparray", ss.spmatrix)) + assert csc.format == "csc" np.testing.assert_array_equal(csc.toarray(), np.array([[0.0, 2.0, 4.1]]).T) # default to csr-like coo = gb.io.to_scipy_sparse(v, "coo") assert coo.shape == csr.shape - assert ss.isspmatrix_coo(coo) + # 2023-06-25: scipy 1.11.0 added `sparray` and changed e.g. `ss.isspmatrix_coo` + assert isinstance(coo, getattr(ss, "sparray", ss.spmatrix)) + assert coo.format == "coo" assert coo.nnz == 2 np.testing.assert_array_equal(coo.toarray(), np.array([[0.0, 2.0, 4.1]])) @@ -99,7 +105,9 @@ def test_matrix_to_from_numpy(): for format in ["csr", "csc", "coo"]: sparse = gb.io.to_scipy_sparse(M, format) - assert getattr(ss, f"isspmatrix_{format}")(sparse) + # 2023-06-25: scipy 1.11.0 added `sparray` and changed e.g. `ss.isspmatrix_csr` + assert isinstance(sparse, getattr(ss, "sparray", ss.spmatrix)) + assert sparse.format == format assert sparse.nnz == 3 np.testing.assert_array_equal(sparse.toarray(), a) M2 = gb.io.from_scipy_sparse(sparse) @@ -435,6 +443,7 @@ def test_awkward_errors(): @pytest.mark.skipif("not sparse") +@pytest.mark.slow def test_vector_to_from_pydata_sparse(): coords = np.array([0, 1, 2, 3, 4], dtype="int64") data = np.array([10, 20, 30, 40, 50], dtype="int64") @@ -448,6 +457,7 @@ def test_vector_to_from_pydata_sparse(): @pytest.mark.skipif("not sparse") +@pytest.mark.slow def test_matrix_to_from_pydata_sparse(): coords = np.array([[0, 1, 2, 3, 4], [0, 1, 2, 3, 4]], dtype="int64") data = np.array([10, 20, 30, 40, 50], dtype="int64") diff --git a/graphblas/tests/test_matrix.py b/graphblas/tests/test_matrix.py index 26017f364..bc942bc49 100644 --- a/graphblas/tests/test_matrix.py +++ b/graphblas/tests/test_matrix.py @@ -4298,7 +4298,7 @@ def test_ss_descriptors(A): A(nthreads=4, axb_method="dot", sort=True) << A @ A assert A.isequal(C2) # Bad option should show list of valid options - with pytest.raises(ValueError, match="nthreads"): + with pytest.raises(ValueError, match="axb_method"): C1(bad_opt=True) << A with pytest.raises(ValueError, match="Duplicate descriptor"): (A @ A).new(nthreads=4, Nthreads=5) diff --git a/graphblas/tests/test_op.py b/graphblas/tests/test_op.py index a80012ab7..85c7663f8 100644 --- a/graphblas/tests/test_op.py +++ b/graphblas/tests/test_op.py @@ -1433,6 +1433,7 @@ def test_deprecated(): import graphblas.core.agg # noqa: F401 +@pytest.mark.slow def test_is_idempotent(): assert monoid.min.is_idempotent assert monoid.max[int].is_idempotent diff --git a/graphblas/unary/ss.py b/graphblas/unary/ss.py index e45cbcda0..97852fc12 100644 --- a/graphblas/unary/ss.py +++ b/graphblas/unary/ss.py @@ -1,3 +1,5 @@ from ..core import operator +_delayed = {} + del operator diff --git a/pyproject.toml b/pyproject.toml index 9e57b8296..ddd718ef6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -138,6 +138,7 @@ packages = [ "graphblas.core", "graphblas.core.operator", "graphblas.core.ss", + "graphblas.dtypes", "graphblas.indexunary", "graphblas.io", "graphblas.monoid", @@ -311,6 +312,7 @@ ignore = [ # "SIM401", # Use dict.get ... instead of if-else-block (Note: if-else better for coverage and sometimes clearer) "TRY004", # Prefer `TypeError` exception for invalid type (Note: good advice, but not worth the nuisance) "TRY200", # Use `raise from` to specify exception cause (Note: sometimes okay to raise original exception) + "RUF012", # Mutable class attributes should be annotated with `typing.ClassVar` (Note: no annotations yet) # Intentionally ignored "COM812", # Trailing comma missing @@ -322,6 +324,7 @@ ignore = [ "N806", # Variable ... in function should be lowercase "N807", # Function name should not start and end with `__` "N818", # Exception name ... should be named with an Error suffix (Note: good advice) + "PLR0124", # Name compared with itself, consider replacing `x == x` (Note: too strict) "PLR0911", # Too many return statements "PLR0912", # Too many branches "PLR0913", # Too many arguments to function call @@ -356,6 +359,7 @@ ignore = [ "TCH", # flake8-type-checking (Note: figure out type checking later) "ARG", # flake8-unused-arguments (Sometimes helpful, but too strict) "TD", # flake8-todos (Maybe okay to add some of these) + "FIX", # flake8-fixme (like flake8-todos) "ERA", # eradicate (We like code in comments!) "PD", # pandas-vet (Intended for scripts that use pandas, not libraries) ] diff --git a/scripts/check_versions.sh b/scripts/check_versions.sh index f0e648fd9..22f0b3cca 100755 --- a/scripts/check_versions.sh +++ b/scripts/check_versions.sh @@ -5,13 +5,13 @@ # Tip: add `--json` for more information. conda search 'numpy[channel=conda-forge]>=1.25.0' conda search 'pandas[channel=conda-forge]>=2.0.2' -conda search 'scipy[channel=conda-forge]>=1.10.1' +conda search 'scipy[channel=conda-forge]>=1.11.0' conda search 'networkx[channel=conda-forge]>=3.1' -conda search 'awkward[channel=conda-forge]>=2.2.1' +conda search 'awkward[channel=conda-forge]>=2.2.4' conda search 'sparse[channel=conda-forge]>=0.14.0' -conda search 'fast_matrix_market[channel=conda-forge]>=1.6.0' -conda search 'numba[channel=conda-forge]>=0.57.0' +conda search 'fast_matrix_market[channel=conda-forge]>=1.7.2' +conda search 'numba[channel=conda-forge]>=0.57.1' conda search 'pyyaml[channel=conda-forge]>=6.0' -conda search 'flake8-bugbear[channel=conda-forge]>=23.5.9' +conda search 'flake8-bugbear[channel=conda-forge]>=23.6.5' conda search 'flake8-simplify[channel=conda-forge]>=0.20.0' # conda search 'python[channel=conda-forge]>=3.8 *pypy*' From 145de316bd075f90ca956b8ad0e39d233ad58eae Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 27 Jun 2023 00:53:02 -0500 Subject: [PATCH 2/9] haha, oops --- graphblas/tests/conftest.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/graphblas/tests/conftest.py b/graphblas/tests/conftest.py index 0137ea59e..ce9e6488f 100644 --- a/graphblas/tests/conftest.py +++ b/graphblas/tests/conftest.py @@ -116,17 +116,20 @@ def ic(): # pragma: no cover (debug) @contextlib.contextmanager -def burble(): +def burble(): # pragma: no cover (debug) """Show the burble diagnostics within a context.""" - # Don't keep track of previous state; always set to False when done + if gb.backend != "suitesparse": + yield + return + prev = gb.ss.config["burble"] gb.ss.config["burble"] = True try: yield finally: - gb.ss.config["burble"] = False + gb.ss.config["burble"] = prev -@pytest.fixture(scope="session", autouse=True) +@pytest.fixture(scope="session") def burble_all(): # pragma: no cover (debug) """Show the burble diagnostics for the entire test.""" with burble(): From 52f0f36a79eaa4bb198af5a3c2a20cda4763f254 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 27 Jun 2023 00:54:25 -0500 Subject: [PATCH 3/9] CI: fail fast --- .github/workflows/test_and_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index 5630b8d35..4d3f9276b 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -85,7 +85,7 @@ jobs: shell: bash -l {0} strategy: # To "stress test" in CI, set `fail-fast` to `false` and perhaps add more items to `matrix.slowtask` - fail-fast: false # Every service seems super-flaky right now... + fail-fast: true # The build matrix is [os]x[slowtask] and then randomly chooses [pyver] and [sourcetype]. # This should ensure we'll have full code coverage (i.e., no chance of getting unlucky), # since we need to run all slow tests on Windows and non-Windoes OSes. From 7495d991276c2bbd85d64652b1cbda0bb6c9197c Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 27 Jun 2023 12:09:36 -0500 Subject: [PATCH 4/9] Make fast_matrix_market <1.7 and scipy >=1.11 play nicely together --- graphblas/io/_matrixmarket.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/graphblas/io/_matrixmarket.py b/graphblas/io/_matrixmarket.py index 0946580c8..750c022eb 100644 --- a/graphblas/io/_matrixmarket.py +++ b/graphblas/io/_matrixmarket.py @@ -104,13 +104,17 @@ def mmwrite( engine = engine.lower() if engine in {"auto", "fmm", "fast_matrix_market"}: try: - from fast_matrix_market import mmwrite # noqa: F811 + from fast_matrix_market import __version__, mmwrite # noqa: F811 except ImportError: # pragma: no cover (import) if engine != "auto": raise ImportError( "fast_matrix_market is required to write Matrix Market files " f'using the "{engine}" engine' ) from None + else: + import scipy as sp + + engine = "fast_matrix_market" elif engine != "scipy": raise ValueError( f'Bad engine value: {engine!r}. Must be "auto", "scipy", "fmm", or "fast_matrix_market"' @@ -119,6 +123,12 @@ def mmwrite( array = matrix.ss.export()["values"] else: array = to_scipy_sparse(matrix, format="coo") + if engine == "fast_matrix_market" and __version__ < "1.7." and sp.__version__ > "1.11.": + # 2023-06-25: scipy 1.11.0 added `sparray` and changed e.g. `ss.isspmatrix_coo`. + # fast_matrix_market updated to handle this in version 1.7.0 + # Also, it looks like fast_matrix_market has special writers for csr and csc; + # should we see if using those are faster? + array = sp.sparse.coo_matrix(array) mmwrite( target, array, From 1ef78c8e050388e6ecaec3fd440e0015d1f0ca31 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 27 Jun 2023 12:32:00 -0500 Subject: [PATCH 5/9] fix mmread to handle sparse and dense arrays --- graphblas/io/_matrixmarket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graphblas/io/_matrixmarket.py b/graphblas/io/_matrixmarket.py index 750c022eb..6773349ca 100644 --- a/graphblas/io/_matrixmarket.py +++ b/graphblas/io/_matrixmarket.py @@ -53,7 +53,7 @@ def mmread(source, engine="auto", *, dup_op=None, name=None, **kwargs): f'Bad engine value: {engine!r}. Must be "auto", "scipy", "fmm", or "fast_matrix_market"' ) array = mmread(source, **kwargs) - if array.format == "coo": + if getattr(array, "format", None) == "coo": nrows, ncols = array.shape return Matrix.from_coo( array.row, array.col, array.data, nrows=nrows, ncols=ncols, dup_op=dup_op, name=name From e8ccf3bde31bca8180ecd3177eaa74deac578962 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 27 Jun 2023 14:36:07 -0500 Subject: [PATCH 6/9] Add tests --- .github/workflows/test_and_build.yml | 3 +++ graphblas/core/descriptor.py | 2 +- graphblas/core/ss/config.py | 16 ++++++++-------- graphblas/io/_matrixmarket.py | 2 +- graphblas/monoid/__init__.py | 14 +++++++++++++- graphblas/monoid/ss.py | 5 +++++ graphblas/tests/test_dtype.py | 17 +++++++++++++++++ graphblas/tests/test_op.py | 11 +++++++++++ graphblas/tests/test_vector.py | 9 ++++++++- 9 files changed, 67 insertions(+), 12 deletions(-) create mode 100644 graphblas/monoid/ss.py diff --git a/.github/workflows/test_and_build.yml b/.github/workflows/test_and_build.yml index 4d3f9276b..209060521 100644 --- a/.github/workflows/test_and_build.yml +++ b/.github/workflows/test_and_build.yml @@ -374,6 +374,9 @@ jobs: # Tests lazy loading of lib, ffi, and NULL in gb.core echo "from graphblas.core import base" > script.py coverage run -a script.py + # Test another code pathway for loading lib + echo "from graphblas.core import lib" > script.py + coverage run -a script.py rm script.py # Tests whose coverage depend on order of tests :/ # TODO: understand why these are order-dependent and try to fix diff --git a/graphblas/core/descriptor.py b/graphblas/core/descriptor.py index 9ba3ae466..11f634afd 100644 --- a/graphblas/core/descriptor.py +++ b/graphblas/core/descriptor.py @@ -26,7 +26,7 @@ def __init__( self.mask_structure = mask_structure self.transpose_first = transpose_first self.transpose_second = transpose_second - self._context = None # Used by SuiteSparse:GraphBLAS + self._context = None # Used by SuiteSparse:GraphBLAS 8 @property def _carg(self): diff --git a/graphblas/core/ss/config.py b/graphblas/core/ss/config.py index 433716bb3..89536479d 100644 --- a/graphblas/core/ss/config.py +++ b/graphblas/core/ss/config.py @@ -65,7 +65,7 @@ def __getitem__(self, key): raise KeyError(key) key_obj, ctype = self._options[key] is_bool = ctype == "bool" - if is_context := (key in self._context_keys): + if is_context := (key in self._context_keys): # pragma: no cover (suitesparse 8) get_function_base = self._context_get_function else: get_function_base = self._get_function @@ -76,14 +76,14 @@ def __getitem__(self, key): get_function_name = f"{get_function_base}_INT64" elif ctype.startswith("double"): get_function_name = f"{get_function_base}_FP64" - elif ctype.startswith("char"): + elif ctype.startswith("char"): # pragma: no cover (suitesparse 8) get_function_name = f"{get_function_base}_CHAR" else: # pragma: no cover (sanity) raise ValueError(ctype) get_function = getattr(lib, get_function_name) is_array = "[" in ctype val_ptr = ffi.new(ctype if is_array else f"{ctype}*") - if is_context: + if is_context: # pragma: no cover (suitesparse 8) info = get_function(self._context._carg, key_obj, val_ptr) elif self._parent is None: info = get_function(key_obj, val_ptr) @@ -105,7 +105,7 @@ def __getitem__(self, key): return rv if is_bool: return bool(val_ptr[0]) - if ctype.startswith("char"): + if ctype.startswith("char"): # pragma: no cover (suitesparse 8) return ffi.string(val_ptr[0]).decode() return val_ptr[0] raise _error_code_lookup[info](f"Failed to get info for {key!r}") # pragma: no cover @@ -117,7 +117,7 @@ def __setitem__(self, key, val): if key in self._read_only: raise ValueError(f"Config option {key!r} is read-only") key_obj, ctype = self._options[key] - if is_context := (key in self._context_keys): + if is_context := (key in self._context_keys): # pragma: no cover (suitesparse 8) set_function_base = self._context_set_function else: set_function_base = self._set_function @@ -130,7 +130,7 @@ def __setitem__(self, key, val): set_function_name = f"{set_function_base}_INT64_ARRAY" elif ctype.startswith("double["): set_function_name = f"{set_function_base}_FP64_ARRAY" - elif ctype.startswith("char"): + elif ctype.startswith("char"): # pragma: no cover (suitesparse 8) set_function_name = f"{set_function_base}_CHAR" else: # pragma: no cover (sanity) raise ValueError(ctype) @@ -174,11 +174,11 @@ def __setitem__(self, key, val): f"expected {size}, got {vals.size}: {val}" ) val_obj = ffi.from_buffer(ctype, vals) - elif ctype.startswith("char"): + elif ctype.startswith("char"): # pragma: no cover (suitesparse 8) val_obj = ffi.new("char[]", val.encode()) else: val_obj = ffi.cast(ctype, val) - if is_context: + if is_context: # pragma: no cover (suitesparse 8) if self._context is None: from .context import Context diff --git a/graphblas/io/_matrixmarket.py b/graphblas/io/_matrixmarket.py index 6773349ca..558605328 100644 --- a/graphblas/io/_matrixmarket.py +++ b/graphblas/io/_matrixmarket.py @@ -128,7 +128,7 @@ def mmwrite( # fast_matrix_market updated to handle this in version 1.7.0 # Also, it looks like fast_matrix_market has special writers for csr and csc; # should we see if using those are faster? - array = sp.sparse.coo_matrix(array) + array = sp.sparse.coo_matrix(array) # FLAKY COVERAGE mmwrite( target, array, diff --git a/graphblas/monoid/__init__.py b/graphblas/monoid/__init__.py index 007aba416..ed028c5d9 100644 --- a/graphblas/monoid/__init__.py +++ b/graphblas/monoid/__init__.py @@ -4,7 +4,7 @@ def __dir__(): - return globals().keys() | _delayed.keys() + return globals().keys() | _delayed.keys() | {"ss"} def __getattr__(key): @@ -17,6 +17,18 @@ def __getattr__(key): rv = func(**kwargs) globals()[key] = rv return rv + if key == "ss": + from .. import backend + + if backend != "suitesparse": + raise AttributeError( + f'module {__name__!r} only has attribute "ss" when backend is "suitesparse"' + ) + from importlib import import_module + + ss = import_module(".ss", __name__) + globals()["ss"] = ss + return ss raise AttributeError(f"module {__name__!r} has no attribute {key!r}") diff --git a/graphblas/monoid/ss.py b/graphblas/monoid/ss.py new file mode 100644 index 000000000..97852fc12 --- /dev/null +++ b/graphblas/monoid/ss.py @@ -0,0 +1,5 @@ +from ..core import operator + +_delayed = {} + +del operator diff --git a/graphblas/tests/test_dtype.py b/graphblas/tests/test_dtype.py index 233e9b65a..47a226313 100644 --- a/graphblas/tests/test_dtype.py +++ b/graphblas/tests/test_dtype.py @@ -253,3 +253,20 @@ def test_has_complex(): from packaging.version import parse assert dtypes._supports_complex == (parse(ssgb.__version__) >= parse("7.4.3.1")) + + +def test_has_ss_attribute(): + if suitesparse: + assert dtypes.ss is not None + else: + with pytest.raises(AttributeError): + dtypes.ss + + +def test_dir(): + must_have = {"DataType", "lookup_dtype", "register_anonymous", "register_new", "ss", "unify"} + must_have.update({"FP32", "FP64", "INT8", "INT16", "INT32", "INT64"}) + must_have.update({"BOOL", "UINT8", "UINT16", "UINT32", "UINT64"}) + if dtypes._supports_complex: + must_have.update({"FC32", "FC64"}) + assert set(dir(dtypes)) & must_have == must_have diff --git a/graphblas/tests/test_op.py b/graphblas/tests/test_op.py index 85c7663f8..f508e9c1f 100644 --- a/graphblas/tests/test_op.py +++ b/graphblas/tests/test_op.py @@ -1447,3 +1447,14 @@ def test_is_idempotent(): assert not monoid.numpy.equal.is_idempotent with pytest.raises(AttributeError): binary.min.is_idempotent + + +def test_ops_have_ss(): + modules = [unary, binary, monoid, semiring, indexunary, select, op] + if suitesparse: + for mod in modules: + assert mod.ss is not None + else: + for mod in modules: + with pytest.raises(AttributeError): + mod.ss diff --git a/graphblas/tests/test_vector.py b/graphblas/tests/test_vector.py index 36ab346b8..a76182e77 100644 --- a/graphblas/tests/test_vector.py +++ b/graphblas/tests/test_vector.py @@ -1455,7 +1455,7 @@ def test_diag(v): # Extract diagonal from A if suitesparse: - w = gb.ss.diag(A, Scalar.from_value(k)) + w = gb.ss.diag(A, Scalar.from_value(k), nthreads=2) assert v.isequal(w) assert w.dtype == "INT64" @@ -1737,6 +1737,13 @@ def test_dup_expr(v): assert result.isequal(b) result = (b | b).dup(clear=True) assert result.isequal(b.dup(clear=True)) + result = v[:5].dup() + assert result.isequal(v[:5].new()) + if suitesparse: + result = v[:5].dup(nthreads=2) + assert result.isequal(v[:5].new()) + result = v[:5].dup(clear=True, nthreads=2) + assert result.isequal(Vector(v.dtype, size=5)) @pytest.mark.skipif("not suitesparse") From b2b59aee9c9ac0d23d89f3fe18f1ffc78a82b3c2 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 27 Jun 2023 14:49:22 -0500 Subject: [PATCH 7/9] Fix test --- graphblas/tests/test_op.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graphblas/tests/test_op.py b/graphblas/tests/test_op.py index f508e9c1f..b54ea76c4 100644 --- a/graphblas/tests/test_op.py +++ b/graphblas/tests/test_op.py @@ -1006,7 +1006,7 @@ def myplus(x, y): def test_create_semiring(): # stress test / sanity check - monoid_names = {x for x in dir(monoid) if not x.startswith("_")} + monoid_names = {x for x in dir(monoid) if not x.startswith("_") and x != "ss"} binary_names = {x for x in dir(binary) if not x.startswith("_") and x != "ss"} for monoid_name, binary_name in itertools.product(monoid_names, binary_names): cur_monoid = getattr(monoid, monoid_name) From 441b8ec328f7f96ec5a9b4c2740f7550a68e8626 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 27 Jun 2023 15:13:22 -0500 Subject: [PATCH 8/9] for coverage --- graphblas/tests/test_vector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graphblas/tests/test_vector.py b/graphblas/tests/test_vector.py index a76182e77..a1aabd183 100644 --- a/graphblas/tests/test_vector.py +++ b/graphblas/tests/test_vector.py @@ -1448,7 +1448,7 @@ def test_diag(v): expected = Matrix.from_coo(rows, cols, values, nrows=size, ncols=size, dtype=v.dtype) # Construct diagonal matrix A if suitesparse: - A = gb.ss.diag(v, k=k) + A = gb.ss.diag(v, k=k, nthreads=2) assert expected.isequal(A) A = v.diag(k) assert expected.isequal(A) From 823731535303b9468e81b6ece6d125f98ec8449d Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 27 Jun 2023 15:37:42 -0500 Subject: [PATCH 9/9] bump action --- .github/workflows/publish_pypi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index eca456c28..ffac645f5 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -35,7 +35,7 @@ jobs: - name: Check with twine run: python -m twine check --strict dist/* - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@v1.8.6 + uses: pypa/gh-action-pypi-publish@v1.8.7 with: user: __token__ password: ${{ secrets.PYPI_TOKEN }}