From d2ee296138efa97dcfa048d3f2d27a31a979631a Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Sun, 12 Nov 2023 17:50:14 -0700 Subject: [PATCH 01/94] MAINT: prepare 1.26.x for further development --- doc/source/release.rst | 1 + doc/source/release/1.26.3-notes.rst | 19 +++++++++++++++++++ pavement.py | 2 +- pyproject.toml | 2 +- pyproject.toml.setuppy | 2 +- 5 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 doc/source/release/1.26.3-notes.rst diff --git a/doc/source/release.rst b/doc/source/release.rst index 0e0d477ce06b..0af815c16a46 100644 --- a/doc/source/release.rst +++ b/doc/source/release.rst @@ -5,6 +5,7 @@ Release notes .. toctree:: :maxdepth: 3 + 1.26.3 1.26.2 1.26.1 1.26.0 diff --git a/doc/source/release/1.26.3-notes.rst b/doc/source/release/1.26.3-notes.rst new file mode 100644 index 000000000000..68fc303779e2 --- /dev/null +++ b/doc/source/release/1.26.3-notes.rst @@ -0,0 +1,19 @@ +.. currentmodule:: numpy + +========================== +NumPy 1.26.3 Release Notes +========================== + +NumPy 1.26.3 is a maintenance release that fixes bugs and regressions +discovered after the 1.26.2 release. The 1.26.release series is the last +planned minor release series before NumPy 2.0. The Python versions supported by +this release are 3.9-3.12. + + +Contributors +============ + + +Pull requests merged +==================== + diff --git a/pavement.py b/pavement.py index 41b04bc108fc..649e19d1fd1f 100644 --- a/pavement.py +++ b/pavement.py @@ -38,7 +38,7 @@ #----------------------------------- # Path to the release notes -RELEASE_NOTES = 'doc/source/release/1.26.2-notes.rst' +RELEASE_NOTES = 'doc/source/release/1.26.3-notes.rst' #------------------------------------------------------- diff --git a/pyproject.toml b/pyproject.toml index c8dc1a72eeba..9ba86eb13486 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ requires = [ [project] name = "numpy" -version = "1.26.2" +version = "1.26.3" # TODO: add `license-files` once PEP 639 is accepted (see meson-python#88) license = {file = "LICENSE.txt"} diff --git a/pyproject.toml.setuppy b/pyproject.toml.setuppy index 02a7d7c57de5..dd97279fa503 100644 --- a/pyproject.toml.setuppy +++ b/pyproject.toml.setuppy @@ -3,7 +3,7 @@ # to avoid building with Meson (e.g., in the Emscripten/Pyodide CI job) [project] name = "numpy" -version = "1.26.2" +version = "1.26.3" [build-system] requires = [ From fc324e360fd1eb0ab32a5ff30c0023029d70e6ea Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Sun, 19 Nov 2023 12:32:51 -0700 Subject: [PATCH 02/94] MAINT: Pin scipy-openblas version. Partial backport of #25085. --- .github/workflows/linux_musl.yml | 2 +- .github/workflows/windows.yml | 4 ++-- azure-steps-windows.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/linux_musl.yml b/.github/workflows/linux_musl.yml index 5c65a2b2e8b9..89a7e147c395 100644 --- a/.github/workflows/linux_musl.yml +++ b/.github/workflows/linux_musl.yml @@ -55,7 +55,7 @@ jobs: python -m venv test_env source test_env/bin/activate - pip install scipy-openblas64 + pip install "scipy-openblas64<=0.3.23.293.2" pip install -r build_requirements.txt -r test_requirements.txt diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 988a55b761db..eb7126a46a5b 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -51,7 +51,7 @@ jobs: env: PKG_CONFIG_PATH: ${{ github.workspace }}/.openblas run: | - python -m pip install scipy-openblas32 + python -m pip install "scipy-openblas32<=0.3.23.293.2" spin build --with-scipy-openblas=32 -j2 -- --vsenv - name: Install NumPy (Clang-cl) @@ -60,7 +60,7 @@ jobs: PKG_CONFIG_PATH: ${{ github.workspace }}/.openblas run: | "[binaries]","c = 'clang-cl'","cpp = 'clang-cl'","ar = 'llvm-lib'","c_ld = 'lld-link'","cpp_ld = 'lld-link'" | Out-File $PWD/clang-cl-build.ini -Encoding ascii - python -m pip install scipy-openblas32 + python -m pip install "scipy-openblas32<=0.3.23.293.2" spin build --with-scipy-openblas=32 -j2 -- --vsenv --native-file=$PWD/clang-cl-build.ini - name: Meson Log diff --git a/azure-steps-windows.yml b/azure-steps-windows.yml index 1cae980b4368..b518656ad26b 100644 --- a/azure-steps-windows.yml +++ b/azure-steps-windows.yml @@ -42,7 +42,7 @@ steps: python -m pip install . -v -Csetup-args="--vsenv" -Csetup-args="-Dblas=none" -Csetup-args="-Dlapack=none" -Csetup-args="-Dallow-noblas=true" } elseif ( Test-Path env:NPY_USE_BLAS_ILP64 ) { - python -m pip install scipy-openblas64 spin + python -m pip install "scipy-openblas64<=0.3.23.293.2" spin spin config-openblas --with-scipy-openblas=64 $env:PKG_CONFIG_PATH="$pwd/.openblas" python -m pip install . -v -Csetup-args="--vsenv" -Csetup-args="-Duse-ilp64=true" From 699c7f0ba8d409c3250a3098662dc2e29bfbe296 Mon Sep 17 00:00:00 2001 From: Illviljan <14371165+Illviljan@users.noreply.github.com> Date: Mon, 13 Nov 2023 23:12:40 +0100 Subject: [PATCH 03/94] TYP: add None to `__getitem__` in `numpy.array_api` (#25022) Closes gh-24982 --- numpy/array_api/_array_object.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/numpy/array_api/_array_object.py b/numpy/array_api/_array_object.py index ec465208e8b2..5aff9863d821 100644 --- a/numpy/array_api/_array_object.py +++ b/numpy/array_api/_array_object.py @@ -543,7 +543,11 @@ def __ge__(self: Array, other: Union[int, float, Array], /) -> Array: def __getitem__( self: Array, key: Union[ - int, slice, ellipsis, Tuple[Union[int, slice, ellipsis], ...], Array + int, + slice, + ellipsis, + Tuple[Union[int, slice, ellipsis, None], ...], + Array, ], /, ) -> Array: From b2550d1885d446ec3d9feea99c439ce36fc24ba0 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 5 Nov 2023 23:07:11 +0000 Subject: [PATCH 04/94] BLD,BUG: quadmath required where available [f2py] Closes gh-25000 --- numpy/f2py/_backends/meson.build.template | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/numpy/f2py/_backends/meson.build.template b/numpy/f2py/_backends/meson.build.template index 545e3995218a..99bf42e3cbbc 100644 --- a/numpy/f2py/_backends/meson.build.template +++ b/numpy/f2py/_backends/meson.build.template @@ -6,6 +6,7 @@ project('${modulename}', 'warning_level=1', 'buildtype=${buildtype}' ]) +fc = meson.get_compiler('fortran') py = import('python').find_installation(pure: false) py_dep = py.dependency() @@ -28,6 +29,8 @@ inc_f2py = include_directories(incdir_f2py) fortranobject_c = incdir_f2py / 'fortranobject.c' inc_np = include_directories(incdir_numpy, incdir_f2py) +# gh-25000 +quadmath_dep = fc.find_library('quadmath', required: false) py.extension_module('${modulename}', [ @@ -37,6 +40,7 @@ ${source_list}, include_directories: [inc_np], dependencies : [ py_dep, + quadmath_dep, ${dep_list} ], install : true) From a7ebf65aa8b7489e76eb6d656d528ee88f13eab1 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 5 Nov 2023 23:13:39 +0000 Subject: [PATCH 05/94] DOC: Add a pointer to the SciPy fortran doc --- doc/source/f2py/windows/index.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/source/f2py/windows/index.rst b/doc/source/f2py/windows/index.rst index 980346667444..ebd10a7d2962 100644 --- a/doc/source/f2py/windows/index.rst +++ b/doc/source/f2py/windows/index.rst @@ -6,8 +6,11 @@ F2PY and Windows .. warning:: - F2PY support for Windows is not at par with Linux support, and - OS specific flags can be seen via ``python -m numpy.f2py`` + F2PY support for Windows is not always at par with Linux support + +.. note:: + `ScPy's documentation`_ has some information on system-level dependencies + which are well tested for Fortran as well. Broadly speaking, there are two issues working with F2PY on Windows: @@ -19,6 +22,7 @@ extending Fortran modules for Python natively, via F2PY on Windows. Overview ======== + From a user perspective, the most UNIX compatible Windows development environment is through emulation, either via the Windows Subsystem on Linux, or facilitated by Docker. In a similar vein, traditional @@ -206,3 +210,4 @@ path using a hash. This needs to be added to the ``PATH`` variable. .. _are outdated: https://github.com/conda-forge/conda-forge.github.io/issues/1044 .. _now deprecated: https://github.com/numpy/numpy/pull/20875 .. _LLVM Flang: https://releases.llvm.org/11.0.0/tools/flang/docs/ReleaseNotes.html +.. _ScPy's documentation: http://scipy.github.io/devdocs/building/index.html#system-level-dependencies From 1996f87e3a00a8278c58c5425710d4094c1dec6d Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 5 Nov 2023 23:14:57 +0000 Subject: [PATCH 06/94] DOC: Adapt comment on supported toolchains Co-authored-by: rgommers --- doc/source/f2py/windows/index.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/source/f2py/windows/index.rst b/doc/source/f2py/windows/index.rst index ebd10a7d2962..124ad1ee66bc 100644 --- a/doc/source/f2py/windows/index.rst +++ b/doc/source/f2py/windows/index.rst @@ -18,7 +18,14 @@ Broadly speaking, there are two issues working with F2PY on Windows: - the linking issues related to the C runtime library for building Python-C extensions. The focus of this section is to establish a guideline for developing and -extending Fortran modules for Python natively, via F2PY on Windows. +extending Fortran modules for Python natively, via F2PY on Windows. + +Currently supported toolchains are: + +- Mingw-w64 C/C++/Fortran compilers +- Intel compilers +- Clang-cl + Flang +- MSVC + Flang Overview ======== From 8f1ff41cf443a69bb3655f2ea1681b54fe0fd2fc Mon Sep 17 00:00:00 2001 From: matoro Date: Mon, 6 Nov 2023 10:21:32 -0500 Subject: [PATCH 07/94] BUG: alpha doesn't use REAL(10) Same as e.g. loongarch per gh-24904. At this point seems like it should be more of an exclude list than an include one... --- numpy/f2py/crackfortran.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/crackfortran.py b/numpy/f2py/crackfortran.py index f352bbaa2720..d17b052f9654 100755 --- a/numpy/f2py/crackfortran.py +++ b/numpy/f2py/crackfortran.py @@ -2452,7 +2452,7 @@ def _selected_real_kind_func(p, r=0, radix=0): if p < 16: return 8 machine = platform.machine().lower() - if machine.startswith(('aarch64', 'arm64', 'loongarch', 'power', 'ppc', 'riscv', 's390x', 'sparc')): + if machine.startswith(('aarch64', 'alpha', 'arm64', 'loongarch', 'power', 'ppc', 'riscv', 's390x', 'sparc')): if p <= 33: return 16 else: From f54454fe1b98fce217fb7352dd6d2a115bb6f82b Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Mon, 13 Nov 2023 02:04:59 +0200 Subject: [PATCH 08/94] BUG: Fix FP overflow error in division when the divisor is scalar The bug occurred when SIMD partial load was involved, due to filling remaining lanes of the dividend vector with ones, which leads to raised overflow warnings when the divisor is denormal. This patch replaces the remaining lanes with nans rather than ones to fix this issue. --- .../src/umath/loops_arithm_fp.dispatch.c.src | 4 +++- numpy/core/tests/test_umath.py | 20 +++++++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/numpy/core/src/umath/loops_arithm_fp.dispatch.c.src b/numpy/core/src/umath/loops_arithm_fp.dispatch.c.src index 30111258d646..c32239dc12df 100644 --- a/numpy/core/src/umath/loops_arithm_fp.dispatch.c.src +++ b/numpy/core/src/umath/loops_arithm_fp.dispatch.c.src @@ -138,8 +138,10 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(@TYPE@_@kind@) npyv_store_@sfx@((@type@*)(dst + vstep), r1); } for (; len > 0; len -= hstep, src0 += vstep, dst += vstep) { - #if @is_div@ || @is_mul@ + #if @is_mul@ npyv_@sfx@ a = npyv_load_till_@sfx@((const @type@*)src0, len, 1.0@c@); + #elif @is_div@ + npyv_@sfx@ a = npyv_load_till_@sfx@((const @type@*)src0, len, NPY_NAN@C@); #else npyv_@sfx@ a = npyv_load_tillz_@sfx@((const @type@*)src0, len); #endif diff --git a/numpy/core/tests/test_umath.py b/numpy/core/tests/test_umath.py index 59c670ffed29..963e740d8dcb 100644 --- a/numpy/core/tests/test_umath.py +++ b/numpy/core/tests/test_umath.py @@ -17,7 +17,8 @@ assert_, assert_equal, assert_raises, assert_raises_regex, assert_array_equal, assert_almost_equal, assert_array_almost_equal, assert_array_max_ulp, assert_allclose, assert_no_warnings, suppress_warnings, - _gen_alignment_data, assert_array_almost_equal_nulp, IS_WASM, IS_MUSL + _gen_alignment_data, assert_array_almost_equal_nulp, IS_WASM, IS_MUSL, + IS_PYPY ) from numpy.testing._private.utils import _glibc_older_than @@ -1825,6 +1826,18 @@ def test_unary_spurious_fpexception(self, ufunc, dtype, data, escape): with assert_no_warnings(): ufunc(array) + @pytest.mark.parametrize("dtype", ('e', 'f', 'd')) + def test_divide_spurious_fpexception(self, dtype): + dt = np.dtype(dtype) + dt_info = np.finfo(dt) + subnorm = dt_info.smallest_subnormal + # Verify a bug fix caused due to filling the remaining lanes of the + # partially loaded dividend SIMD vector with ones, which leads to + # raising an overflow warning when the divisor is denormal. + # see https://github.com/numpy/numpy/issues/25097 + with assert_no_warnings(): + np.zeros(128 + 1, dtype=dt) / subnorm + class TestFPClass: @pytest.mark.parametrize("stride", [-5, -4, -3, -2, -1, 1, 2, 4, 5, 6, 7, 8, 9, 10]) @@ -4180,7 +4193,10 @@ def test_against_cmath(self): for p in points: a = complex(func(np.complex_(p))) b = cfunc(p) - assert_(abs(a - b) < atol, "%s %s: %s; cmath: %s" % (fname, p, a, b)) + assert_( + abs(a - b) < atol, + "%s %s: %s; cmath: %s" % (fname, p, a, b) + ) @pytest.mark.xfail( # manylinux2014 uses glibc2.17 From c6e29c8bfd3861dcecc181a6077972e2e1a8d61c Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Wed, 15 Nov 2023 17:44:54 +0200 Subject: [PATCH 09/94] ENH: Disable SIMD single-precision division optimization on armv7 The decision is based on the lack of native SIMD support for this operation in the armhf architecture, and the associated challenges in performance and evaluate the benefits of emulated SIMD intrinsic versus native scalar division. --- .../src/umath/loops_arithm_fp.dispatch.c.src | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/numpy/core/src/umath/loops_arithm_fp.dispatch.c.src b/numpy/core/src/umath/loops_arithm_fp.dispatch.c.src index c32239dc12df..0d0de90125f6 100644 --- a/numpy/core/src/umath/loops_arithm_fp.dispatch.c.src +++ b/numpy/core/src/umath/loops_arithm_fp.dispatch.c.src @@ -74,7 +74,29 @@ NPY_NO_EXPORT void NPY_CPU_DISPATCH_CURFX(@TYPE@_@kind@) #endif return; } -#if @VECTOR@ +#if @is_div@ && defined(NPY_HAVE_NEON) && !NPY_SIMD_F64 + /** + * The SIMD branch is disabled on armhf(armv7) due to the absence of native SIMD + * support for single-precision floating-point division. Only scalar division is + * supported natively, and without hardware for performance and accuracy comparison, + * it's challenging to evaluate the benefits of emulated SIMD intrinsic versus + * native scalar division. + * + * The `npyv_div_f32` universal intrinsic emulates the division operation using an + * approximate reciprocal combined with 3 Newton-Raphson iterations for enhanced + * precision. However, this approach has limitations: + * + * - It can cause unexpected floating-point overflows in special cases, such as when + * the divisor is subnormal (refer: https://github.com/numpy/numpy/issues/25097). + * + * - The precision may vary between the emulated SIMD and scalar division due to + * non-uniform branches (non-contiguous) in the code, leading to precision + * inconsistencies. + * + * - Considering the necessity of multiple Newton-Raphson iterations, the performance + * gain may not sufficiently offset these drawbacks. + */ +#elif @VECTOR@ if (len > npyv_nlanes_@sfx@*2 && !is_mem_overlap(src0, ssrc0, dst, sdst, len) && !is_mem_overlap(src1, ssrc1, dst, sdst, len) From 04730b1ee513ffc1e1f0ab070507c5e8a34068a4 Mon Sep 17 00:00:00 2001 From: Daniel Vanzo Date: Mon, 11 Sep 2023 13:53:12 +0200 Subject: [PATCH 10/94] BUG: Fix numpy.f2py to enable use of string optional inout argument --- numpy/f2py/cfuncs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/numpy/f2py/cfuncs.py b/numpy/f2py/cfuncs.py index f89793061bad..3251c8d50f02 100644 --- a/numpy/f2py/cfuncs.py +++ b/numpy/f2py/cfuncs.py @@ -649,6 +649,7 @@ STRINGCOPYN(buf, str, n); return 1; } + return -1; capi_fail: PRINTPYOBJERR(obj); PyErr_SetString(#modulename#_error, \"try_pyarr_from_string failed\"); From 2aff3136499a20f588e2b2a6cd4c40ea607fb917 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 19 Nov 2023 22:27:48 +0000 Subject: [PATCH 11/94] MAINT: Be explicit about failure modes [f2py-str] --- numpy/f2py/cfuncs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/numpy/f2py/cfuncs.py b/numpy/f2py/cfuncs.py index 3251c8d50f02..cc43e28eb608 100644 --- a/numpy/f2py/cfuncs.py +++ b/numpy/f2py/cfuncs.py @@ -634,6 +634,8 @@ fprintf(stderr, "try_pyarr_from_string(str='%s', len=%d, obj=%p)\\n", (char*)str,len, obj); #endif + if (!obj) return -2; /* Object missing */ + if (obj == Py_None) return -1; /* None */ if (PyArray_Check(obj)) { PyArrayObject *arr = (PyArrayObject *)obj; assert(ISCONTIGUOUS(arr)); @@ -649,7 +651,6 @@ STRINGCOPYN(buf, str, n); return 1; } - return -1; capi_fail: PRINTPYOBJERR(obj); PyErr_SetString(#modulename#_error, \"try_pyarr_from_string failed\"); From 7f435912dc83a61b5fec6b136b18abcc11d2f2b9 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 19 Nov 2023 22:51:02 +0000 Subject: [PATCH 12/94] TST: Add gh-24781 for string inout optional [f2py] --- numpy/f2py/tests/src/string/gh24662.f90 | 7 +++++++ numpy/f2py/tests/test_character.py | 9 +++++++++ 2 files changed, 16 insertions(+) create mode 100644 numpy/f2py/tests/src/string/gh24662.f90 diff --git a/numpy/f2py/tests/src/string/gh24662.f90 b/numpy/f2py/tests/src/string/gh24662.f90 new file mode 100644 index 000000000000..ca53413cc9b4 --- /dev/null +++ b/numpy/f2py/tests/src/string/gh24662.f90 @@ -0,0 +1,7 @@ +subroutine string_inout_optional(output) + implicit none + character*(32), optional, intent(inout) :: output + if (present(output)) then + output="output string" + endif +end subroutine diff --git a/numpy/f2py/tests/test_character.py b/numpy/f2py/tests/test_character.py index 373262bf96a6..93f7188625b4 100644 --- a/numpy/f2py/tests/test_character.py +++ b/numpy/f2py/tests/test_character.py @@ -595,3 +595,12 @@ class TestStringAssumedLength(util.F2PyTest): def test_gh24008(self): self.module.greet("joe", "bob") + +class TestStringOptionalInOut(util.F2PyTest): + sources = [util.getpath("tests", "src", "string", "gh24662.f90")] + + def test_gh24662(self): + self.module.string_inout_optional() + a = np.array('hi', dtype='S32') + self.module.string_inout_optional(a) + assert "output string" in a.tobytes().decode() From 6c2615ef48c675cec20b7d97dc50ace5b129ecc7 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 19 Nov 2023 23:18:03 +0000 Subject: [PATCH 13/94] TST: Add a check for non-ndarray string types --- numpy/f2py/cfuncs.py | 1 + numpy/f2py/tests/test_character.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/numpy/f2py/cfuncs.py b/numpy/f2py/cfuncs.py index cc43e28eb608..37c2f6df0879 100644 --- a/numpy/f2py/cfuncs.py +++ b/numpy/f2py/cfuncs.py @@ -636,6 +636,7 @@ #endif if (!obj) return -2; /* Object missing */ if (obj == Py_None) return -1; /* None */ + if (!PyArray_Check(obj)) goto capi_fail; /* not an ndarray */ if (PyArray_Check(obj)) { PyArrayObject *arr = (PyArrayObject *)obj; assert(ISCONTIGUOUS(arr)); diff --git a/numpy/f2py/tests/test_character.py b/numpy/f2py/tests/test_character.py index 93f7188625b4..c928e38741a1 100644 --- a/numpy/f2py/tests/test_character.py +++ b/numpy/f2py/tests/test_character.py @@ -604,3 +604,6 @@ def test_gh24662(self): a = np.array('hi', dtype='S32') self.module.string_inout_optional(a) assert "output string" in a.tobytes().decode() + with pytest.raises(Exception): + aa = "Hi" + self.module.string_inout_optional(aa) From 02ae2670a928fe9223891e1019f35b5eeec435dd Mon Sep 17 00:00:00 2001 From: Peter Hawkins Date: Mon, 13 Nov 2023 13:12:28 +0000 Subject: [PATCH 14/94] Fix -fsanitize=alignment issue in numpy/_core/src/multiarray/arraytypes.c.src OBJECT_nonzero may be called with misaligned pointers, manifesting as a -fsanitize=alignment failure. This is UB per C11 6.3.2.3 > A pointer to an object type may be converted to a pointer to a different object type. If the resulting pointer is not correctly aligned) for the referenced type, the behavior is undefined. Nevertheless, Clang only checks alignment when the unaligned pointer is accessed. https://lists.llvm.org/pipermail/llvm-dev/2016-January/094012.html Call memcpy with unaligned arguments instead to work with new Clang (https://github.com/llvm/llvm-project/pull/67766). --- numpy/core/src/multiarray/arraytypes.c.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/src/multiarray/arraytypes.c.src b/numpy/core/src/multiarray/arraytypes.c.src index bc3f743727e7..f3feee63daaf 100644 --- a/numpy/core/src/multiarray/arraytypes.c.src +++ b/numpy/core/src/multiarray/arraytypes.c.src @@ -2889,7 +2889,7 @@ OBJECT_nonzero (PyObject **ip, PyArrayObject *ap) } else { PyObject *obj; - memcpy(&obj, ip, sizeof(obj)); + memcpy(&obj, (void *)ip, sizeof(obj)); if (obj == NULL) { return NPY_FALSE; } From b446db62656419726f635e3bde3342a8bc938814 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 14 Nov 2023 10:59:32 +0100 Subject: [PATCH 15/94] TST: Build cython checks testing module only once --- numpy/core/tests/test_cython.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/numpy/core/tests/test_cython.py b/numpy/core/tests/test_cython.py index 99dd57e4c62d..0e0d00c2508b 100644 --- a/numpy/core/tests/test_cython.py +++ b/numpy/core/tests/test_cython.py @@ -28,14 +28,14 @@ pytestmark = pytest.mark.skipif(cython is None, reason="requires cython") -@pytest.fixture -def install_temp(tmp_path): +@pytest.fixture(scope='module') +def install_temp(tmpdir_factory): # Based in part on test_cython from random.tests.test_extending if IS_WASM: pytest.skip("No subprocess") srcdir = os.path.join(os.path.dirname(__file__), 'examples', 'cython') - build_dir = tmp_path / "build" + build_dir = tmpdir_factory.mktemp("cython_test") / "build" os.makedirs(build_dir, exist_ok=True) try: subprocess.check_call(["meson", "--version"]) From eaa5a6c4fc89088067fd05da473d5e969d70dbfd Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 14 Nov 2023 11:15:44 +0100 Subject: [PATCH 16/94] TST: Explicitly pass NumPy path to cython during tests This papers over gh-25135, it doesn't fix it, since we are finding the right NumPy, Cython should be finding the right pyd files also (whether this is a NumPy or a Cython issue). --- numpy/core/tests/examples/cython/meson.build | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/numpy/core/tests/examples/cython/meson.build b/numpy/core/tests/examples/cython/meson.build index 12fc640b88b4..836b74ac3866 100644 --- a/numpy/core/tests/examples/cython/meson.build +++ b/numpy/core/tests/examples/cython/meson.build @@ -14,6 +14,15 @@ npy_include_path = run_command(py, [ 'import os; os.chdir(".."); import numpy; print(os.path.abspath(numpy.get_include()))' ], check: true).stdout().strip() +npy_path = run_command(py, [ + '-c', + 'import os; os.chdir(".."); import numpy; print(os.path.dirname(numpy.__file__).removesuffix("numpy"))' + ], check: true).stdout().strip() + +# TODO: This is a hack due to gh-25135, where cython may not find the right +# __init__.pyd file. +add_project_arguments('-I', npy_path, language : 'cython') + py.extension_module( 'checks', 'checks.pyx', From fdfbbfbd3b9a6dd964b543ff2babc8569303f0f5 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Tue, 14 Nov 2023 13:08:41 +0100 Subject: [PATCH 17/94] BUG: fix issues with `newaxis` and `linalg.solve` in `numpy.array_api` [skip azp] [skip cirrus] [skip circle] --- numpy/array_api/__init__.py | 2 +- numpy/array_api/_constants.py | 1 + numpy/array_api/linalg.py | 5 +++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/numpy/array_api/__init__.py b/numpy/array_api/__init__.py index 964873faab20..77f227882e3e 100644 --- a/numpy/array_api/__init__.py +++ b/numpy/array_api/__init__.py @@ -125,7 +125,7 @@ __all__ = ["__array_api_version__"] -from ._constants import e, inf, nan, pi +from ._constants import e, inf, nan, pi, newaxis __all__ += ["e", "inf", "nan", "pi"] diff --git a/numpy/array_api/_constants.py b/numpy/array_api/_constants.py index 9541941e7c6f..15ab81d16d93 100644 --- a/numpy/array_api/_constants.py +++ b/numpy/array_api/_constants.py @@ -4,3 +4,4 @@ inf = np.inf nan = np.nan pi = np.pi +newaxis = np.newaxis diff --git a/numpy/array_api/linalg.py b/numpy/array_api/linalg.py index 58320db55ceb..09af9dfc3ae2 100644 --- a/numpy/array_api/linalg.py +++ b/numpy/array_api/linalg.py @@ -318,8 +318,9 @@ def _solve(a, b): # This does nothing currently but is left in because it will be relevant # when complex dtype support is added to the spec in 2022. signature = 'DD->D' if isComplexType(t) else 'dd->d' - extobj = get_linalg_error_extobj(_raise_linalgerror_singular) - r = gufunc(a, b, signature=signature, extobj=extobj) + with np.errstate(call=_raise_linalgerror_singular, invalid='call', + over='ignore', divide='ignore', under='ignore'): + r = gufunc(a, b, signature=signature) return wrap(r.astype(result_t, copy=False)) From d5e84c3520f4ee8a1a60094abda3651e7311215c Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sat, 18 Nov 2023 21:12:28 +0000 Subject: [PATCH 18/94] TST: Add one for gh-22819 --- numpy/f2py/tests/src/cli/gh_22819.pyf | 6 ++++++ numpy/f2py/tests/test_f2py2e.py | 28 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 numpy/f2py/tests/src/cli/gh_22819.pyf diff --git a/numpy/f2py/tests/src/cli/gh_22819.pyf b/numpy/f2py/tests/src/cli/gh_22819.pyf new file mode 100644 index 000000000000..8eb5bb106a36 --- /dev/null +++ b/numpy/f2py/tests/src/cli/gh_22819.pyf @@ -0,0 +1,6 @@ +python module test_22819 + interface + subroutine hello() + end subroutine hello + end interface +end python module test_22819 diff --git a/numpy/f2py/tests/test_f2py2e.py b/numpy/f2py/tests/test_f2py2e.py index 5f7b56a68a9d..49f0bef8aa96 100644 --- a/numpy/f2py/tests/test_f2py2e.py +++ b/numpy/f2py/tests/test_f2py2e.py @@ -71,6 +71,15 @@ def gh23598_warn(tmpdir_factory): return fn +@pytest.fixture(scope="session") +def gh22819_cli(tmpdir_factory): + """F90 file for testing disallowed CLI arguments in ghff819""" + fdat = util.getpath("tests", "src", "cli", "gh_22819.pyf").read_text() + fn = tmpdir_factory.getbasetemp() / "gh_22819.pyf" + fn.write_text(fdat, encoding="ascii") + return fn + + @pytest.fixture(scope="session") def hello_world_f77(tmpdir_factory): """Generates a single f77 file for testing""" @@ -100,6 +109,25 @@ def f2cmap_f90(tmpdir_factory): return fn +def test_gh22819_cli(capfd, gh22819_cli, monkeypatch): + """Check that module names are handled correctly + gh-22819 + Essentially, the -m name cannot be used to import the module, so the module + named in the .pyf needs to be used instead + + CLI :: -m and a .pyf file + """ + ipath = Path(gh22819_cli) + monkeypatch.setattr(sys, "argv", f"f2py -m blah {ipath}".split()) + with util.switchdir(ipath.parent): + f2pycli() + gen_paths = [item.name for item in ipath.parent.rglob("*") if item.is_file()] + assert "blahmodule.c" not in gen_paths # shouldn't be generated + assert "blah-f2pywrappers.f" not in gen_paths + assert "test_22819-f2pywrappers.f" in gen_paths + assert "test_22819module.c" in gen_paths + + def test_gh23598_warn(capfd, gh23598_warn, monkeypatch): foutl = get_io_paths(gh23598_warn, mname="test") ipath = foutl.f90inp From 69e95bd275c1fb8acd8236e2cb7f1b389bc59641 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 19 Nov 2023 01:05:08 +0000 Subject: [PATCH 19/94] BUG: Correctly disallow shadowed modulenames --- numpy/f2py/f2py2e.py | 60 ++++++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index 1cfe8cddd68c..cfc42b7433a2 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -456,6 +456,16 @@ def run_main(comline_list): f2pydir = os.path.dirname(os.path.abspath(cfuncs.__file__)) fobjhsrc = os.path.join(f2pydir, 'src', 'fortranobject.h') fobjcsrc = os.path.join(f2pydir, 'src', 'fortranobject.c') + # gh-22819 -- begin + parser = make_f2py_parser() + args, comline_list = parser.parse_known_args(comline_list) + pyf_files, _ = filter_files("", "[.]pyf([.]src|)", comline_list) + # Checks that no existing modulename is defined in a pyf file + # TODO: Remove all this when scaninputline is replaced + if "-h" not in comline_list: # Can't check what doesn't exist yet, -h creates the pyf + modname = validate_modulename(pyf_files, args.module_name) + comline_list += ['-m', modname] # needed for the rest of scaninputline + # gh-22819 -- end files, options = scaninputline(comline_list) auxfuncs.options = options capi_maps.load_f2cmap_file(options['f2cmap_file']) @@ -522,24 +532,30 @@ def get_prefix(module): p = os.path.dirname(os.path.dirname(module.__file__)) return p -def preparse_sysargv(): - # To keep backwards bug compatibility, newer flags are handled by argparse, - # and `sys.argv` is passed to the rest of `f2py` as is. +def make_f2py_parser(): parser = argparse.ArgumentParser(add_help=False) parser.add_argument("--dep", action="append", dest="dependencies") parser.add_argument("--backend", choices=['meson', 'distutils'], default='distutils') + parser.add_argument("-m", dest="module_name") + return parser + +def preparse_sysargv(): + # To keep backwards bug compatibility, newer flags are handled by argparse, + # and `sys.argv` is passed to the rest of `f2py` as is. + parser = make_f2py_parser() args, remaining_argv = parser.parse_known_args() sys.argv = [sys.argv[0]] + remaining_argv backend_key = args.backend if sys.version_info >= (3, 12) and backend_key == 'distutils': - outmess('Cannot use distutils backend with Python 3.12, using meson backend instead.') + outmess("Cannot use distutils backend with Python 3.12, using meson backend instead.\n") backend_key = 'meson' return { "dependencies": args.dependencies or [], - "backend": backend_key + "backend": backend_key, + "modulename": args.module_name, } def run_compile(): @@ -550,11 +566,11 @@ def run_compile(): # Collect dependency flags, preprocess sys.argv argy = preparse_sysargv() + modulename = argy["modulename"] dependencies = argy["dependencies"] backend_key = argy["backend"] build_backend = f2py_build_generator(backend_key) - i = sys.argv.index('-c') del sys.argv[i] @@ -634,9 +650,7 @@ def run_compile(): if '--quiet' in f2py_flags: setup_flags.append('--quiet') - modulename = 'untitled' sources = sys.argv[1:] - for optname in ['--include_paths', '--include-paths', '--f2cmap']: if optname in sys.argv: i = sys.argv.index(optname) @@ -644,20 +658,9 @@ def run_compile(): del sys.argv[i + 1], sys.argv[i] sources = sys.argv[1:] - pyf_files = [] - if '-m' in sys.argv: - i = sys.argv.index('-m') - modulename = sys.argv[i + 1] - del sys.argv[i + 1], sys.argv[i] - sources = sys.argv[1:] - else: - pyf_files, _sources = filter_files('', '[.]pyf([.]src|)', sources) - sources = pyf_files + _sources - for f in pyf_files: - modulename = auxfuncs.get_f2py_modulename(f) - if modulename: - break - + pyf_files, _sources = filter_files("", "[.]pyf([.]src|)", sources) + sources = pyf_files + _sources + modulename = validate_modulename(pyf_files, modulename) extra_objects, sources = filter_files('', '[.](o|a|so|dylib)', sources) include_dirs, sources = filter_files('-I', '', sources, remove_prefix=1) library_dirs, sources = filter_files('-L', '', sources, remove_prefix=1) @@ -704,6 +707,19 @@ def run_compile(): builder.compile() + +def validate_modulename(pyf_files, modulename='untitled'): + for f in pyf_files: + pyf_modname = auxfuncs.get_f2py_modulename(f) + if modulename != pyf_modname: + outmess( + f"Ignoring -m {modulename}.\n" + f"{f} defines {pyf_modname} to be the modulename.\n" + ) + modulename = pyf_modname + break + return modulename + def main(): if '--help-link' in sys.argv[1:]: sys.argv.remove('--help-link') From f28a90566a83b2457d95fedf88a4addb7c719197 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 19 Nov 2023 01:14:12 +0000 Subject: [PATCH 20/94] MAINT: Strengthen the validation of pyf files --- numpy/f2py/f2py2e.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index cfc42b7433a2..51fd8181930f 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -709,15 +709,16 @@ def run_compile(): def validate_modulename(pyf_files, modulename='untitled'): - for f in pyf_files: - pyf_modname = auxfuncs.get_f2py_modulename(f) - if modulename != pyf_modname: - outmess( - f"Ignoring -m {modulename}.\n" - f"{f} defines {pyf_modname} to be the modulename.\n" - ) - modulename = pyf_modname - break + if len(pyf_files) > 1: + raise ValueError("Only one .pyf file per call") + pyff = pyf_files[0] + pyf_modname = auxfuncs.get_f2py_modulename(pyff) + if modulename != pyf_modname: + outmess( + f"Ignoring -m {modulename}.\n" + f"{pyff} defines {pyf_modname} to be the modulename.\n" + ) + modulename = pyf_modname return modulename def main(): From b169db9886d06180a5d9d9ba8813665f73b33d9b Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 19 Nov 2023 01:14:27 +0000 Subject: [PATCH 21/94] TST: Add tests for CLI .pyf file work --- numpy/f2py/tests/test_f2py2e.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/numpy/f2py/tests/test_f2py2e.py b/numpy/f2py/tests/test_f2py2e.py index 49f0bef8aa96..315b4fca4504 100644 --- a/numpy/f2py/tests/test_f2py2e.py +++ b/numpy/f2py/tests/test_f2py2e.py @@ -126,6 +126,19 @@ def test_gh22819_cli(capfd, gh22819_cli, monkeypatch): assert "blah-f2pywrappers.f" not in gen_paths assert "test_22819-f2pywrappers.f" in gen_paths assert "test_22819module.c" in gen_paths + assert "Ignoring blah" + + +def test_gh22819_many_pyf(capfd, gh22819_cli, monkeypatch): + """Only one .pyf file allowed + gh-22819 + CLI :: .pyf files + """ + ipath = Path(gh22819_cli) + monkeypatch.setattr(sys, "argv", f"f2py -m blah {ipath} hello.pyf".split()) + with util.switchdir(ipath.parent): + with pytest.raises(ValueError, match="Only one .pyf file per call"): + f2pycli() def test_gh23598_warn(capfd, gh23598_warn, monkeypatch): From 989cd8cc9cac0c4334614c611b63d91ec2e1c598 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 19 Nov 2023 01:40:24 +0000 Subject: [PATCH 22/94] MAINT: Fix implementation for gh22819 [f2py] --- numpy/f2py/f2py2e.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index 51fd8181930f..01847b3dc595 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -711,14 +711,15 @@ def run_compile(): def validate_modulename(pyf_files, modulename='untitled'): if len(pyf_files) > 1: raise ValueError("Only one .pyf file per call") - pyff = pyf_files[0] - pyf_modname = auxfuncs.get_f2py_modulename(pyff) - if modulename != pyf_modname: - outmess( - f"Ignoring -m {modulename}.\n" - f"{pyff} defines {pyf_modname} to be the modulename.\n" - ) - modulename = pyf_modname + if pyf_files: + pyff = pyf_files[0] + pyf_modname = auxfuncs.get_f2py_modulename(pyff) + if modulename != pyf_modname: + outmess( + f"Ignoring -m {modulename}.\n" + f"{pyff} defines {pyf_modname} to be the modulename.\n" + ) + modulename = pyf_modname return modulename def main(): From 0715b0d3504802f894831a87bf8f16a0bf03fec5 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 19 Nov 2023 01:45:00 +0000 Subject: [PATCH 23/94] DOC: Add an entry to the changelog [f2py] --- doc/release/upcoming_changes/25181.compatibility.rst | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 doc/release/upcoming_changes/25181.compatibility.rst diff --git a/doc/release/upcoming_changes/25181.compatibility.rst b/doc/release/upcoming_changes/25181.compatibility.rst new file mode 100644 index 000000000000..aefe7cd7a437 --- /dev/null +++ b/doc/release/upcoming_changes/25181.compatibility.rst @@ -0,0 +1,4 @@ +``f2py`` will no longer accept ambiguous ``-m`` and ``.pyf`` CLI combinations. +When more than one ``.pyf`` file is passed, an error is raised. When both ``-m`` +and a ``.pyf`` is passed, a warning is emitted and the ``-m`` provided name is +ignored. From db6bdf67224572038cb4c7ea5f01a74324f9637a Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 19 Nov 2023 01:47:16 +0000 Subject: [PATCH 24/94] DOC: Rework documentation on -m [f2py] --- doc/source/f2py/usage.rst | 50 +++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/doc/source/f2py/usage.rst b/doc/source/f2py/usage.rst index 04f5a8c7d996..d812af9a2042 100644 --- a/doc/source/f2py/usage.rst +++ b/doc/source/f2py/usage.rst @@ -208,29 +208,33 @@ Other options ``-m `` Name of an extension module. Default is ``untitled``. - .. warning:: Don't use this option if a signature file (``*.pyf``) is used. - - ``--[no-]lower`` - Do [not] lower the cases in ````. By default, ``--lower`` is - assumed with ``-h`` switch, and ``--no-lower`` without the ``-h`` switch. - ``-include
`` - Writes additional headers in the C wrapper, can be passed multiple times, - generates #include
each time. Note that this is meant to be passed - in single quotes and without spaces, for example ``'-include'`` - ``--build-dir `` - All F2PY generated files are created in ````. Default is - ``tempfile.mkdtemp()``. - ``--f2cmap `` - Load Fortran-to-C ``KIND`` specifications from the given file. - ``--quiet`` - Run quietly. - ``--verbose`` - Run with extra verbosity. - ``--skip-empty-wrappers`` - Do not generate wrapper files unless required by the inputs. - This is a backwards compatibility flag to restore pre 1.22.4 behavior. - ``-v`` - Print the F2PY version and exit. +.. warning:: + Don't use this option if a signature file (``*.pyf``) is used. + + .. versionchanged:: 1.26.3 + Will ignore ``-m`` if a ``pyf`` file is provided. + +``--[no-]lower`` + Do [not] lower the cases in ````. By default, ``--lower`` is + assumed with ``-h`` switch, and ``--no-lower`` without the ``-h`` switch. +``-include
`` + Writes additional headers in the C wrapper, can be passed multiple times, + generates #include
each time. Note that this is meant to be passed + in single quotes and without spaces, for example ``'-include'`` +``--build-dir `` + All F2PY generated files are created in ````. Default is + ``tempfile.mkdtemp()``. +``--f2cmap `` + Load Fortran-to-C ``KIND`` specifications from the given file. +``--quiet`` + Run quietly. +``--verbose`` + Run with extra verbosity. +``--skip-empty-wrappers`` + Do not generate wrapper files unless required by the inputs. + This is a backwards compatibility flag to restore pre 1.22.4 behavior. +``-v`` + Print the F2PY version and exit. Execute ``f2py`` without any options to get an up-to-date list of available options. From bd22a3f7db21d2f5ac8828b1caf98bc20ae19c3d Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 19 Nov 2023 02:49:17 +0000 Subject: [PATCH 25/94] MAINT: Handle case where -m is None --- numpy/f2py/f2py2e.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index 01847b3dc595..8488929cfbc5 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -462,7 +462,7 @@ def run_main(comline_list): pyf_files, _ = filter_files("", "[.]pyf([.]src|)", comline_list) # Checks that no existing modulename is defined in a pyf file # TODO: Remove all this when scaninputline is replaced - if "-h" not in comline_list: # Can't check what doesn't exist yet, -h creates the pyf + if "-h" not in comline_list and args.module_name: # Can't check what doesn't exist yet, -h creates the pyf modname = validate_modulename(pyf_files, args.module_name) comline_list += ['-m', modname] # needed for the rest of scaninputline # gh-22819 -- end From 2b9fb88f1ee2d53597303898089f23e994c3ef7b Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 19 Nov 2023 07:03:25 +0000 Subject: [PATCH 26/94] MAINT: Fix sneaky typo [f2py] --- numpy/f2py/f2py2e.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index 8488929cfbc5..8265f30a8f8f 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -567,6 +567,8 @@ def run_compile(): # Collect dependency flags, preprocess sys.argv argy = preparse_sysargv() modulename = argy["modulename"] + if modulename is None: + modulename = 'untitled' dependencies = argy["dependencies"] backend_key = argy["backend"] build_backend = f2py_build_generator(backend_key) From 659751bab76e508d662ad927945d3bd4f7ac9dd2 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 19 Nov 2023 07:07:14 +0000 Subject: [PATCH 27/94] TST: Ensure f2py defaults are tested --- numpy/f2py/tests/test_f2py2e.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/numpy/f2py/tests/test_f2py2e.py b/numpy/f2py/tests/test_f2py2e.py index 315b4fca4504..dfea19e2bed9 100644 --- a/numpy/f2py/tests/test_f2py2e.py +++ b/numpy/f2py/tests/test_f2py2e.py @@ -1,6 +1,7 @@ import textwrap, re, sys, subprocess, shlex from pathlib import Path from collections import namedtuple +import platform import pytest @@ -197,6 +198,20 @@ def test_gen_pyf_no_overwrite(capfd, hello_world_f90, monkeypatch): assert "Use --overwrite-signature to overwrite" in err +@pytest.mark.skipif((platform.system() != 'Linux') and (sys.version_info <= (3, 12)), reason='Compiler and 3.12 required') +def test_untitled_cli(capfd, hello_world_f90, monkeypatch): + """Check that modules are named correctly + + CLI :: defaults + """ + ipath = Path(hello_world_f90) + monkeypatch.setattr(sys, "argv", f"f2py --backend meson -c {ipath}".split()) + with util.switchdir(ipath.parent): + f2pycli() + out, _ = capfd.readouterr() + assert "untitledmodule.c" in out + + @pytest.mark.xfail def test_f2py_skip(capfd, retreal_f77, monkeypatch): """Tests that functions can be skipped From 95b941d1f6dde9fa45570015cbef5b3e51282738 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Mon, 20 Nov 2023 14:15:10 -0700 Subject: [PATCH 28/94] MAINT: Fix misspelling of _backends --- numpy/f2py/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/setup.py b/numpy/f2py/setup.py index 98f1e9aaae84..05bef3000147 100644 --- a/numpy/f2py/setup.py +++ b/numpy/f2py/setup.py @@ -31,7 +31,7 @@ def configuration(parent_package='', top_path=None): config.add_data_files( 'src/fortranobject.c', 'src/fortranobject.h', - 'backends/meson.build.template', + '_backends/meson.build.template', ) config.add_data_files('*.pyi') return config From 497af3cb8bb2ba5abdb2538d96cbd60dc517477b Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 19 Nov 2023 07:36:46 +0000 Subject: [PATCH 29/94] MAINT: Don't escape newlines from f2py [f2py] --- numpy/f2py/cfuncs.py | 106 +++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/numpy/f2py/cfuncs.py b/numpy/f2py/cfuncs.py index 37c2f6df0879..23d9812a9255 100644 --- a/numpy/f2py/cfuncs.py +++ b/numpy/f2py/cfuncs.py @@ -64,7 +64,7 @@ typedefs['unsigned_short'] = 'typedef unsigned short unsigned_short;' typedefs['unsigned_long'] = 'typedef unsigned long unsigned_long;' typedefs['signed_char'] = 'typedef signed char signed_char;' -typedefs['long_long'] = """\ +typedefs['long_long'] = """ #if defined(NPY_OS_WIN32) typedef __int64 long_long; #else @@ -72,14 +72,14 @@ typedef unsigned long long unsigned_long_long; #endif """ -typedefs['unsigned_long_long'] = """\ +typedefs['unsigned_long_long'] = """ #if defined(NPY_OS_WIN32) typedef __uint64 long_long; #else typedef unsigned long long unsigned_long_long; #endif """ -typedefs['long_double'] = """\ +typedefs['long_double'] = """ #ifndef _LONG_DOUBLE typedef long double long_double; #endif @@ -93,7 +93,7 @@ ############### CPP macros #################### -cppmacros['CFUNCSMESS'] = """\ +cppmacros['CFUNCSMESS'] = """ #ifdef DEBUGCFUNCS #define CFUNCSMESS(mess) fprintf(stderr,\"debug-capi:\"mess); #define CFUNCSMESSPY(mess,obj) CFUNCSMESS(mess) \\ @@ -104,7 +104,7 @@ #define CFUNCSMESSPY(mess,obj) #endif """ -cppmacros['F_FUNC'] = """\ +cppmacros['F_FUNC'] = """ #if defined(PREPEND_FORTRAN) #if defined(NO_APPEND_FORTRAN) #if defined(UPPERCASE_FORTRAN) @@ -140,7 +140,7 @@ #define F_FUNC_US(f,F) F_FUNC(f,F) #endif """ -cppmacros['F_WRAPPEDFUNC'] = """\ +cppmacros['F_WRAPPEDFUNC'] = """ #if defined(PREPEND_FORTRAN) #if defined(NO_APPEND_FORTRAN) #if defined(UPPERCASE_FORTRAN) @@ -176,7 +176,7 @@ #define F_WRAPPEDFUNC_US(f,F) F_WRAPPEDFUNC(f,F) #endif """ -cppmacros['F_MODFUNC'] = """\ +cppmacros['F_MODFUNC'] = """ #if defined(F90MOD2CCONV1) /*E.g. Compaq Fortran */ #if defined(NO_APPEND_FORTRAN) #define F_MODFUNCNAME(m,f) $ ## m ## $ ## f @@ -210,12 +210,12 @@ #define F_MODFUNC(m,f) (*(f2pymodstruct##m##.##f)) """ -cppmacros['SWAPUNSAFE'] = """\ +cppmacros['SWAPUNSAFE'] = """ #define SWAP(a,b) (size_t)(a) = ((size_t)(a) ^ (size_t)(b));\\ (size_t)(b) = ((size_t)(a) ^ (size_t)(b));\\ (size_t)(a) = ((size_t)(a) ^ (size_t)(b)) """ -cppmacros['SWAP'] = """\ +cppmacros['SWAP'] = """ #define SWAP(a,b,t) {\\ t *c;\\ c = a;\\ @@ -224,13 +224,13 @@ """ # cppmacros['ISCONTIGUOUS']='#define ISCONTIGUOUS(m) (PyArray_FLAGS(m) & # NPY_ARRAY_C_CONTIGUOUS)' -cppmacros['PRINTPYOBJERR'] = """\ +cppmacros['PRINTPYOBJERR'] = """ #define PRINTPYOBJERR(obj)\\ fprintf(stderr,\"#modulename#.error is related to \");\\ PyObject_Print((PyObject *)obj,stderr,Py_PRINT_RAW);\\ fprintf(stderr,\"\\n\"); """ -cppmacros['MINMAX'] = """\ +cppmacros['MINMAX'] = """ #ifndef max #define max(a,b) ((a > b) ? (a) : (b)) #endif @@ -244,7 +244,7 @@ #define MIN(a,b) ((a < b) ? (a) : (b)) #endif """ -cppmacros['len..'] = """\ +cppmacros['len..'] = """ /* See fortranobject.h for definitions. The macros here are provided for BC. */ #define rank f2py_rank #define shape f2py_shape @@ -263,7 +263,7 @@ cppmacros[ 'pyobj_from_long1'] = '#define pyobj_from_long1(v) (PyLong_FromLong(v))' needs['pyobj_from_long_long1'] = ['long_long'] -cppmacros['pyobj_from_long_long1'] = """\ +cppmacros['pyobj_from_long_long1'] = """ #ifdef HAVE_LONG_LONG #define pyobj_from_long_long1(v) (PyLong_FromLongLong(v)) #else @@ -294,7 +294,7 @@ cppmacros[ 'pyobj_from_string1size'] = '#define pyobj_from_string1size(v,len) (PyUnicode_FromStringAndSize((char *)v, len))' needs['TRYPYARRAYTEMPLATE'] = ['PRINTPYOBJERR'] -cppmacros['TRYPYARRAYTEMPLATE'] = """\ +cppmacros['TRYPYARRAYTEMPLATE'] = """ /* New SciPy */ #define TRYPYARRAYTEMPLATECHAR case NPY_STRING: *(char *)(PyArray_DATA(arr))=*v; break; #define TRYPYARRAYTEMPLATELONG case NPY_LONG: *(long *)(PyArray_DATA(arr))=*v; break; @@ -331,7 +331,7 @@ """ needs['TRYCOMPLEXPYARRAYTEMPLATE'] = ['PRINTPYOBJERR'] -cppmacros['TRYCOMPLEXPYARRAYTEMPLATE'] = """\ +cppmacros['TRYCOMPLEXPYARRAYTEMPLATE'] = """ #define TRYCOMPLEXPYARRAYTEMPLATEOBJECT case NPY_OBJECT: PyArray_SETITEM(arr, PyArray_DATA(arr), pyobj_from_complex_ ## ctype ## 1((*v))); break; #define TRYCOMPLEXPYARRAYTEMPLATE(ctype,typecode)\\ PyArrayObject *arr = NULL;\\ @@ -372,7 +372,7 @@ };\\ return -1; """ -# cppmacros['NUMFROMARROBJ']="""\ +# cppmacros['NUMFROMARROBJ']=""" # define NUMFROMARROBJ(typenum,ctype) \\ # if (PyArray_Check(obj)) arr = (PyArrayObject *)obj;\\ # else arr = (PyArrayObject *)PyArray_ContiguousFromObject(obj,typenum,0,0);\\ @@ -388,7 +388,7 @@ # } # """ # XXX: Note that CNUMFROMARROBJ is identical with NUMFROMARROBJ -# cppmacros['CNUMFROMARROBJ']="""\ +# cppmacros['CNUMFROMARROBJ']=""" # define CNUMFROMARROBJ(typenum,ctype) \\ # if (PyArray_Check(obj)) arr = (PyArrayObject *)obj;\\ # else arr = (PyArrayObject *)PyArray_ContiguousFromObject(obj,typenum,0,0);\\ @@ -406,7 +406,7 @@ needs['GETSTRFROMPYTUPLE'] = ['STRINGCOPYN', 'PRINTPYOBJERR'] -cppmacros['GETSTRFROMPYTUPLE'] = """\ +cppmacros['GETSTRFROMPYTUPLE'] = """ #define GETSTRFROMPYTUPLE(tuple,index,str,len) {\\ PyObject *rv_cb_str = PyTuple_GetItem((tuple),(index));\\ if (rv_cb_str == NULL)\\ @@ -421,7 +421,7 @@ }\\ } """ -cppmacros['GETSCALARFROMPYTUPLE'] = """\ +cppmacros['GETSCALARFROMPYTUPLE'] = """ #define GETSCALARFROMPYTUPLE(tuple,index,var,ctype,mess) {\\ if ((capi_tmp = PyTuple_GetItem((tuple),(index)))==NULL) goto capi_fail;\\ if (!(ctype ## _from_pyobj((var),capi_tmp,mess)))\\ @@ -429,7 +429,7 @@ } """ -cppmacros['FAILNULL'] = """\\ +cppmacros['FAILNULL'] = """\ #define FAILNULL(p) do { \\ if ((p) == NULL) { \\ PyErr_SetString(PyExc_MemoryError, "NULL pointer found"); \\ @@ -438,11 +438,11 @@ } while (0) """ needs['MEMCOPY'] = ['string.h', 'FAILNULL'] -cppmacros['MEMCOPY'] = """\ +cppmacros['MEMCOPY'] = """ #define MEMCOPY(to,from,n)\\ do { FAILNULL(to); FAILNULL(from); (void)memcpy(to,from,n); } while (0) """ -cppmacros['STRINGMALLOC'] = """\ +cppmacros['STRINGMALLOC'] = """ #define STRINGMALLOC(str,len)\\ if ((str = (string)malloc(len+1)) == NULL) {\\ PyErr_SetString(PyExc_MemoryError, \"out of memory\");\\ @@ -451,11 +451,11 @@ (str)[len] = '\\0';\\ } """ -cppmacros['STRINGFREE'] = """\ +cppmacros['STRINGFREE'] = """ #define STRINGFREE(str) do {if (!(str == NULL)) free(str);} while (0) """ needs['STRINGPADN'] = ['string.h'] -cppmacros['STRINGPADN'] = """\ +cppmacros['STRINGPADN'] = """ /* STRINGPADN replaces null values with padding values from the right. @@ -476,7 +476,7 @@ } while (0) """ needs['STRINGCOPYN'] = ['string.h', 'FAILNULL'] -cppmacros['STRINGCOPYN'] = """\ +cppmacros['STRINGCOPYN'] = """ /* STRINGCOPYN copies N bytes. @@ -492,23 +492,23 @@ } while (0) """ needs['STRINGCOPY'] = ['string.h', 'FAILNULL'] -cppmacros['STRINGCOPY'] = """\ +cppmacros['STRINGCOPY'] = """ #define STRINGCOPY(to,from)\\ do { FAILNULL(to); FAILNULL(from); (void)strcpy(to,from); } while (0) """ -cppmacros['CHECKGENERIC'] = """\ +cppmacros['CHECKGENERIC'] = """ #define CHECKGENERIC(check,tcheck,name) \\ if (!(check)) {\\ PyErr_SetString(#modulename#_error,\"(\"tcheck\") failed for \"name);\\ /*goto capi_fail;*/\\ } else """ -cppmacros['CHECKARRAY'] = """\ +cppmacros['CHECKARRAY'] = """ #define CHECKARRAY(check,tcheck,name) \\ if (!(check)) {\\ PyErr_SetString(#modulename#_error,\"(\"tcheck\") failed for \"name);\\ /*goto capi_fail;*/\\ } else """ -cppmacros['CHECKSTRING'] = """\ +cppmacros['CHECKSTRING'] = """ #define CHECKSTRING(check,tcheck,name,show,var)\\ if (!(check)) {\\ char errstring[256];\\ @@ -516,7 +516,7 @@ PyErr_SetString(#modulename#_error, errstring);\\ /*goto capi_fail;*/\\ } else """ -cppmacros['CHECKSCALAR'] = """\ +cppmacros['CHECKSCALAR'] = """ #define CHECKSCALAR(check,tcheck,name,show,var)\\ if (!(check)) {\\ char errstring[256];\\ @@ -524,7 +524,7 @@ PyErr_SetString(#modulename#_error,errstring);\\ /*goto capi_fail;*/\\ } else """ -# cppmacros['CHECKDIMS']="""\ +# cppmacros['CHECKDIMS']=""" # define CHECKDIMS(dims,rank) \\ # for (int i=0;i<(rank);i++)\\ # if (dims[i]<0) {\\ @@ -534,12 +534,12 @@ # """ cppmacros[ 'ARRSIZE'] = '#define ARRSIZE(dims,rank) (_PyArray_multiply_list(dims,rank))' -cppmacros['OLDPYNUM'] = """\ +cppmacros['OLDPYNUM'] = """ #ifdef OLDPYNUM #error You need to install NumPy version 0.13 or higher. See https://scipy.org/install.html #endif """ -cppmacros["F2PY_THREAD_LOCAL_DECL"] = """\ +cppmacros["F2PY_THREAD_LOCAL_DECL"] = """ #ifndef F2PY_THREAD_LOCAL_DECL #if defined(_MSC_VER) #define F2PY_THREAD_LOCAL_DECL __declspec(thread) @@ -565,21 +565,21 @@ """ ################# C functions ############### -cfuncs['calcarrindex'] = """\ +cfuncs['calcarrindex'] = """ static int calcarrindex(int *i,PyArrayObject *arr) { int k,ii = i[0]; for (k=1; k < PyArray_NDIM(arr); k++) ii += (ii*(PyArray_DIM(arr,k) - 1)+i[k]); /* assuming contiguous arr */ return ii; }""" -cfuncs['calcarrindextr'] = """\ +cfuncs['calcarrindextr'] = """ static int calcarrindextr(int *i,PyArrayObject *arr) { int k,ii = i[PyArray_NDIM(arr)-1]; for (k=1; k < PyArray_NDIM(arr); k++) ii += (ii*(PyArray_DIM(arr,PyArray_NDIM(arr)-k-1) - 1)+i[PyArray_NDIM(arr)-k-1]); /* assuming contiguous arr */ return ii; }""" -cfuncs['forcomb'] = """\ +cfuncs['forcomb'] = """ static struct { int nd;npy_intp *d;int *i,*i_tr,tr; } forcombcache; static int initforcomb(npy_intp *dims,int nd,int tr) { int k; @@ -620,7 +620,7 @@ return i; }""" needs['try_pyarr_from_string'] = ['STRINGCOPYN', 'PRINTPYOBJERR', 'string'] -cfuncs['try_pyarr_from_string'] = """\ +cfuncs['try_pyarr_from_string'] = """ /* try_pyarr_from_string copies str[:len(obj)] to the data of an `ndarray`. @@ -659,7 +659,7 @@ } """ needs['string_from_pyobj'] = ['string', 'STRINGMALLOC', 'STRINGCOPYN'] -cfuncs['string_from_pyobj'] = """\ +cfuncs['string_from_pyobj'] = """ /* Create a new string buffer `str` of at most length `len` from a Python string-like object `obj`. @@ -759,7 +759,7 @@ } """ -cfuncs['character_from_pyobj'] = """\ +cfuncs['character_from_pyobj'] = """ static int character_from_pyobj(character* v, PyObject *obj, const char *errmess) { if (PyBytes_Check(obj)) { @@ -827,7 +827,7 @@ """ needs['char_from_pyobj'] = ['int_from_pyobj'] -cfuncs['char_from_pyobj'] = """\ +cfuncs['char_from_pyobj'] = """ static int char_from_pyobj(char* v, PyObject *obj, const char *errmess) { int i = 0; @@ -841,7 +841,7 @@ needs['signed_char_from_pyobj'] = ['int_from_pyobj', 'signed_char'] -cfuncs['signed_char_from_pyobj'] = """\ +cfuncs['signed_char_from_pyobj'] = """ static int signed_char_from_pyobj(signed_char* v, PyObject *obj, const char *errmess) { int i = 0; @@ -855,7 +855,7 @@ needs['short_from_pyobj'] = ['int_from_pyobj'] -cfuncs['short_from_pyobj'] = """\ +cfuncs['short_from_pyobj'] = """ static int short_from_pyobj(short* v, PyObject *obj, const char *errmess) { int i = 0; @@ -868,7 +868,7 @@ """ -cfuncs['int_from_pyobj'] = """\ +cfuncs['int_from_pyobj'] = """ static int int_from_pyobj(int* v, PyObject *obj, const char *errmess) { @@ -918,7 +918,7 @@ """ -cfuncs['long_from_pyobj'] = """\ +cfuncs['long_from_pyobj'] = """ static int long_from_pyobj(long* v, PyObject *obj, const char *errmess) { PyObject* tmp = NULL; @@ -967,7 +967,7 @@ needs['long_long_from_pyobj'] = ['long_long'] -cfuncs['long_long_from_pyobj'] = """\ +cfuncs['long_long_from_pyobj'] = """ static int long_long_from_pyobj(long_long* v, PyObject *obj, const char *errmess) { @@ -1017,7 +1017,7 @@ needs['long_double_from_pyobj'] = ['double_from_pyobj', 'long_double'] -cfuncs['long_double_from_pyobj'] = """\ +cfuncs['long_double_from_pyobj'] = """ static int long_double_from_pyobj(long_double* v, PyObject *obj, const char *errmess) { @@ -1041,7 +1041,7 @@ """ -cfuncs['double_from_pyobj'] = """\ +cfuncs['double_from_pyobj'] = """ static int double_from_pyobj(double* v, PyObject *obj, const char *errmess) { @@ -1085,7 +1085,7 @@ needs['float_from_pyobj'] = ['double_from_pyobj'] -cfuncs['float_from_pyobj'] = """\ +cfuncs['float_from_pyobj'] = """ static int float_from_pyobj(float* v, PyObject *obj, const char *errmess) { @@ -1101,7 +1101,7 @@ needs['complex_long_double_from_pyobj'] = ['complex_long_double', 'long_double', 'complex_double_from_pyobj', 'npy_math.h'] -cfuncs['complex_long_double_from_pyobj'] = """\ +cfuncs['complex_long_double_from_pyobj'] = """ static int complex_long_double_from_pyobj(complex_long_double* v, PyObject *obj, const char *errmess) { @@ -1128,7 +1128,7 @@ needs['complex_double_from_pyobj'] = ['complex_double', 'npy_math.h'] -cfuncs['complex_double_from_pyobj'] = """\ +cfuncs['complex_double_from_pyobj'] = """ static int complex_double_from_pyobj(complex_double* v, PyObject *obj, const char *errmess) { Py_complex c; @@ -1205,7 +1205,7 @@ needs['complex_float_from_pyobj'] = [ 'complex_float', 'complex_double_from_pyobj'] -cfuncs['complex_float_from_pyobj'] = """\ +cfuncs['complex_float_from_pyobj'] = """ static int complex_float_from_pyobj(complex_float* v,PyObject *obj,const char *errmess) { @@ -1220,7 +1220,7 @@ """ -cfuncs['try_pyarr_from_character'] = """\ +cfuncs['try_pyarr_from_character'] = """ static int try_pyarr_from_character(PyObject* obj, character* v) { PyArrayObject *arr = (PyArrayObject*)obj; if (!obj) return -2; @@ -1285,7 +1285,7 @@ needs['create_cb_arglist'] = ['CFUNCSMESS', 'PRINTPYOBJERR', 'MINMAX'] # create the list of arguments to be used when calling back to python -cfuncs['create_cb_arglist'] = """\ +cfuncs['create_cb_arglist'] = """ static int create_cb_arglist(PyObject* fun, PyTupleObject* xa , const int maxnofargs, const int nofoptargs, int *nofargs, PyTupleObject **args, From 5e79262d2009ab02cad1335b89dfd163b63b68f4 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 19 Nov 2023 10:15:11 +0000 Subject: [PATCH 30/94] MAINT: Create a useblocks function [f2py] --- numpy/f2py/auxfuncs.py | 10 +++++++++- numpy/f2py/common_rules.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/numpy/f2py/auxfuncs.py b/numpy/f2py/auxfuncs.py index 0c08e0a5e2cf..24e1557be3ca 100644 --- a/numpy/f2py/auxfuncs.py +++ b/numpy/f2py/auxfuncs.py @@ -50,7 +50,7 @@ 'isunsigned_long_longarray', 'isunsigned_short', 'isunsigned_shortarray', 'l_and', 'l_not', 'l_or', 'outmess', 'replace', 'show', 'stripcomma', 'throw_error', 'isattr_value', - 'deep_merge' + 'deep_merge', 'getuseblocks' ] @@ -937,3 +937,11 @@ def get_f2py_modulename(source): name = m.group('name') break return name + +def getuseblocks(pymod): + all_uses = [] + for inner in pymod['body']: + for modblock in inner['body']: + if modblock.get('use'): + all_uses.extend([x for x in modblock.get('use').keys()]) + return all_uses diff --git a/numpy/f2py/common_rules.py b/numpy/f2py/common_rules.py index 5a488bf5a5a4..1dd43afbb6de 100644 --- a/numpy/f2py/common_rules.py +++ b/numpy/f2py/common_rules.py @@ -17,7 +17,7 @@ f2py_version = __version__.version from .auxfuncs import ( - hasbody, hascommon, hasnote, isintent_hide, outmess + hasbody, hascommon, hasnote, isintent_hide, outmess, getuseblocks ) from . import capi_maps from . import func2subr From abdcb4363d34ec595a95381cdf0d20bbd7e43738 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 19 Nov 2023 10:15:35 +0000 Subject: [PATCH 31/94] BUG: Add use-block handling for common blocks Closes gh-19161 --- numpy/f2py/common_rules.py | 2 ++ numpy/f2py/f90mod_rules.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/numpy/f2py/common_rules.py b/numpy/f2py/common_rules.py index 1dd43afbb6de..5c86613c1011 100644 --- a/numpy/f2py/common_rules.py +++ b/numpy/f2py/common_rules.py @@ -78,6 +78,8 @@ def dadd(line, s=doc): outmess('\t\tConstructing COMMON block support for "%s"...\n\t\t %s\n' % ( name, ','.join(inames))) fadd('subroutine f2pyinit%s(setupfunc)' % name) + for usename in getuseblocks(m): + fadd(f'use {usename}') fadd('external setupfunc') for n in vnames: fadd(func2subr.var2fixfortran(vars, n)) diff --git a/numpy/f2py/f90mod_rules.py b/numpy/f2py/f90mod_rules.py index 1c47bee02b91..36d26968bb3c 100644 --- a/numpy/f2py/f90mod_rules.py +++ b/numpy/f2py/f90mod_rules.py @@ -99,6 +99,8 @@ def fadd(line, s=fhooks): def dadd(line, s=doc): s[0] = '%s\n%s' % (s[0], line) + + usenames = getuseblocks(pymod) for m in findf90modules(pymod): sargs, fargs, efargs, modobjs, notvars, onlyvars = [], [], [], [], [ m['name']], [] @@ -115,6 +117,9 @@ def dadd(line, s=doc): mfargs.append(n) outmess('\t\tConstructing F90 module support for "%s"...\n' % (m['name'])) + if m['name'] in usenames: + outmess(f"\t\t\tSkipping {m['name']} since it is in 'use'...\n") + continue if onlyvars: outmess('\t\t Variables: %s\n' % (' '.join(onlyvars))) chooks = [''] From 33fcf1d05bc27584e7e69428f9a00e34b56fa40d Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 19 Nov 2023 10:43:19 +0000 Subject: [PATCH 32/94] TST: Add a test for gh-19161 [f2py] --- numpy/f2py/tests/src/common/gh19161.f90 | 10 ++++++++++ numpy/f2py/tests/test_common.py | 9 +++++++++ 2 files changed, 19 insertions(+) create mode 100644 numpy/f2py/tests/src/common/gh19161.f90 diff --git a/numpy/f2py/tests/src/common/gh19161.f90 b/numpy/f2py/tests/src/common/gh19161.f90 new file mode 100644 index 000000000000..a2f40735ad66 --- /dev/null +++ b/numpy/f2py/tests/src/common/gh19161.f90 @@ -0,0 +1,10 @@ +module typedefmod + use iso_fortran_env, only: real32 +end module typedefmod + +module data + use typedefmod, only: real32 + implicit none + real(kind=real32) :: x + common/test/x +end module data diff --git a/numpy/f2py/tests/test_common.py b/numpy/f2py/tests/test_common.py index 8a4b221ef8bd..68c1b3b31c5d 100644 --- a/numpy/f2py/tests/test_common.py +++ b/numpy/f2py/tests/test_common.py @@ -16,3 +16,12 @@ def test_common_block(self): assert self.module.block.long_bn == np.array(1.0, dtype=np.float64) assert self.module.block.string_bn == np.array("2", dtype="|S1") assert self.module.block.ok == np.array(3, dtype=np.int32) + + +class TestCommonWithUse(util.F2PyTest): + sources = [util.getpath("tests", "src", "common", "gh19161.f90")] + + @pytest.mark.skipif(sys.platform == "win32", + reason="Fails with MinGW64 Gfortran (Issue #9673)") + def test_common_gh19161(self): + assert self.module.data.x == 0 From 3b9c5d6f2babb7c49c4c7c2c28618ba4ec40d16a Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 19 Nov 2023 18:29:48 +0000 Subject: [PATCH 33/94] DOC: Add a note to the release [f2py] common-kind --- doc/release/upcoming_changes/25186.improvement.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/release/upcoming_changes/25186.improvement.rst diff --git a/doc/release/upcoming_changes/25186.improvement.rst b/doc/release/upcoming_changes/25186.improvement.rst new file mode 100644 index 000000000000..6c55a2f500a1 --- /dev/null +++ b/doc/release/upcoming_changes/25186.improvement.rst @@ -0,0 +1 @@ +``f2py`` now handles ``common`` blocks which have ``kind`` specifications from modules. This further expands the usability of intrinsics like ``iso_fortran_env`` and ``iso_c_binding``. From 3ab268da385f4e38a07c47b81e43281101831766 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 20 Nov 2023 23:06:38 +0000 Subject: [PATCH 34/94] BUG: Don't try to grab callback modules Closes gh-25211 --- numpy/f2py/auxfuncs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/auxfuncs.py b/numpy/f2py/auxfuncs.py index 24e1557be3ca..af277c1a4e92 100644 --- a/numpy/f2py/auxfuncs.py +++ b/numpy/f2py/auxfuncs.py @@ -943,5 +943,5 @@ def getuseblocks(pymod): for inner in pymod['body']: for modblock in inner['body']: if modblock.get('use'): - all_uses.extend([x for x in modblock.get('use').keys()]) + all_uses.extend([x for x in modblock.get("use").keys() if "__" not in x]) return all_uses From dbd96e96561a5e346453ed868108bc904a6794e2 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Tue, 21 Nov 2023 00:06:35 +0000 Subject: [PATCH 35/94] TST: Add one for gh25211 --- numpy/f2py/tests/src/callback/gh25211.f | 10 ++++++++++ numpy/f2py/tests/src/callback/gh25211.pyf | 18 ++++++++++++++++++ numpy/f2py/tests/test_callback.py | 13 +++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 numpy/f2py/tests/src/callback/gh25211.f create mode 100644 numpy/f2py/tests/src/callback/gh25211.pyf diff --git a/numpy/f2py/tests/src/callback/gh25211.f b/numpy/f2py/tests/src/callback/gh25211.f new file mode 100644 index 000000000000..ba727a10a07e --- /dev/null +++ b/numpy/f2py/tests/src/callback/gh25211.f @@ -0,0 +1,10 @@ + SUBROUTINE FOO(FUN,R) + EXTERNAL FUN + INTEGER I + REAL*8 R, FUN +Cf2py intent(out) r + R = 0D0 + DO I=-5,5 + R = R + FUN(I) + ENDDO + END diff --git a/numpy/f2py/tests/src/callback/gh25211.pyf b/numpy/f2py/tests/src/callback/gh25211.pyf new file mode 100644 index 000000000000..f12011153370 --- /dev/null +++ b/numpy/f2py/tests/src/callback/gh25211.pyf @@ -0,0 +1,18 @@ +python module __user__routines + interface + function fun(i) result (r) + integer :: i + real*8 :: r + end function fun + end interface +end python module __user__routines + +python module callback2 + interface + subroutine foo(f,r) + use __user__routines, f=>fun + external f + real*8 intent(out) :: r + end subroutine foo + end interface +end python module callback2 diff --git a/numpy/f2py/tests/test_callback.py b/numpy/f2py/tests/test_callback.py index 018cea4fd5e3..5b6c294d33fc 100644 --- a/numpy/f2py/tests/test_callback.py +++ b/numpy/f2py/tests/test_callback.py @@ -228,3 +228,16 @@ def foo(x): r = self.module.gh18335(foo) assert r == 123 + 1 + + +class TestGH25211(util.F2PyTest): + sources = [util.getpath("tests", "src", "callback", "gh25211.f"), + util.getpath("tests", "src", "callback", "gh25211.pyf")] + module_name = "callback2" + + def test_gh18335(self): + def bar(x): + return x*x + + res = self.module.foo(bar) + assert res == 110 From 7ce7ce55e09707e5cc2fbd1c4f9c370df861f520 Mon Sep 17 00:00:00 2001 From: JM Rohwer Date: Thu, 16 Nov 2023 15:27:34 +0200 Subject: [PATCH 36/94] BUG: Fix moving compiled executable to root with f2py -c on Windows --- numpy/f2py/_backends/_meson.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/numpy/f2py/_backends/_meson.py b/numpy/f2py/_backends/_meson.py index 3176a5e08f30..59cc77146f6f 100644 --- a/numpy/f2py/_backends/_meson.py +++ b/numpy/f2py/_backends/_meson.py @@ -7,6 +7,7 @@ from ._backend import Backend from string import Template +from itertools import chain import warnings @@ -83,7 +84,10 @@ def __init__(self, *args, **kwargs): def _move_exec_to_root(self, build_dir: Path): walk_dir = Path(build_dir) / self.meson_build_dir - path_objects = walk_dir.glob(f"{self.modulename}*.so") + path_objects = chain( + walk_dir.glob(f"{self.modulename}*.so"), + walk_dir.glob(f"{self.modulename}*.pyd"), + ) for path_object in path_objects: shutil.move(path_object, Path.cwd()) From 75414cb98405feafa884fd83baa14cfae7cecc08 Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Mon, 20 Nov 2023 03:06:59 +0200 Subject: [PATCH 37/94] BUG: Fix single to half-precision conversion on PPC64/VSX3 This fix respects the lane order with regards to big/little-endians. --- numpy/core/src/common/half.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/numpy/core/src/common/half.hpp b/numpy/core/src/common/half.hpp index ff9a547766d3..484750ad84cd 100644 --- a/numpy/core/src/common/half.hpp +++ b/numpy/core/src/common/half.hpp @@ -59,7 +59,11 @@ class Half final { __vector float vf32 = vec_splats(f); __vector unsigned short vf16; __asm__ __volatile__ ("xvcvsphp %x0,%x1" : "=wa" (vf16) : "wa" (vf32)); + #ifdef __BIG_ENDIAN__ + bits_ = vec_extract(vf16, 1); + #else bits_ = vec_extract(vf16, 0); + #endif #else bits_ = half_private::FromFloatBits(BitCast(f)); #endif From 4b551412b6f44b38bef4a6d7e7c5aafeff0e6d03 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Wed, 22 Nov 2023 16:31:35 +0100 Subject: [PATCH 38/94] TST: f2py: fix issue in test skip condition This test was failing in the 32-bit wheel build job, the AND condition should be an OR one - test is Linux-only. [skip cirrus] [skip circle] --- numpy/f2py/tests/test_f2py2e.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/numpy/f2py/tests/test_f2py2e.py b/numpy/f2py/tests/test_f2py2e.py index dfea19e2bed9..b27df5fbd410 100644 --- a/numpy/f2py/tests/test_f2py2e.py +++ b/numpy/f2py/tests/test_f2py2e.py @@ -198,7 +198,8 @@ def test_gen_pyf_no_overwrite(capfd, hello_world_f90, monkeypatch): assert "Use --overwrite-signature to overwrite" in err -@pytest.mark.skipif((platform.system() != 'Linux') and (sys.version_info <= (3, 12)), reason='Compiler and 3.12 required') +@pytest.mark.skipif((platform.system() != 'Linux') or (sys.version_info <= (3, 12)), + reason='Compiler and 3.12 required') def test_untitled_cli(capfd, hello_world_f90, monkeypatch): """Check that modules are named correctly From c3bfe18e31cca426e4205207502ce3b861be26bc Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Thu, 23 Nov 2023 15:30:21 -0700 Subject: [PATCH 39/94] Revert "MAINT: Pin scipy-openblas version." This reverts commit fc324e360fd1eb0ab32a5ff30c0023029d70e6ea. --- .github/workflows/linux_musl.yml | 2 +- .github/workflows/windows.yml | 4 ++-- azure-steps-windows.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/linux_musl.yml b/.github/workflows/linux_musl.yml index 89a7e147c395..5c65a2b2e8b9 100644 --- a/.github/workflows/linux_musl.yml +++ b/.github/workflows/linux_musl.yml @@ -55,7 +55,7 @@ jobs: python -m venv test_env source test_env/bin/activate - pip install "scipy-openblas64<=0.3.23.293.2" + pip install scipy-openblas64 pip install -r build_requirements.txt -r test_requirements.txt diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index eb7126a46a5b..988a55b761db 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -51,7 +51,7 @@ jobs: env: PKG_CONFIG_PATH: ${{ github.workspace }}/.openblas run: | - python -m pip install "scipy-openblas32<=0.3.23.293.2" + python -m pip install scipy-openblas32 spin build --with-scipy-openblas=32 -j2 -- --vsenv - name: Install NumPy (Clang-cl) @@ -60,7 +60,7 @@ jobs: PKG_CONFIG_PATH: ${{ github.workspace }}/.openblas run: | "[binaries]","c = 'clang-cl'","cpp = 'clang-cl'","ar = 'llvm-lib'","c_ld = 'lld-link'","cpp_ld = 'lld-link'" | Out-File $PWD/clang-cl-build.ini -Encoding ascii - python -m pip install "scipy-openblas32<=0.3.23.293.2" + python -m pip install scipy-openblas32 spin build --with-scipy-openblas=32 -j2 -- --vsenv --native-file=$PWD/clang-cl-build.ini - name: Meson Log diff --git a/azure-steps-windows.yml b/azure-steps-windows.yml index b518656ad26b..1cae980b4368 100644 --- a/azure-steps-windows.yml +++ b/azure-steps-windows.yml @@ -42,7 +42,7 @@ steps: python -m pip install . -v -Csetup-args="--vsenv" -Csetup-args="-Dblas=none" -Csetup-args="-Dlapack=none" -Csetup-args="-Dallow-noblas=true" } elseif ( Test-Path env:NPY_USE_BLAS_ILP64 ) { - python -m pip install "scipy-openblas64<=0.3.23.293.2" spin + python -m pip install scipy-openblas64 spin spin config-openblas --with-scipy-openblas=64 $env:PKG_CONFIG_PATH="$pwd/.openblas" python -m pip install . -v -Csetup-args="--vsenv" -Csetup-args="-Duse-ilp64=true" From 8d46859094e48cbdcc949ba28c07930b4a5b77e1 Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Sat, 18 Nov 2023 16:46:52 -0500 Subject: [PATCH 40/94] MNT: do not use `long` type Removed as type at language level 3 in ed7e95912b6ed7d20e190fbf1aecb9f2a997d479 in cython --- numpy/random/_generator.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/random/_generator.pyx b/numpy/random/_generator.pyx index 1bdba3356557..ba3118d9c715 100644 --- a/numpy/random/_generator.pyx +++ b/numpy/random/_generator.pyx @@ -3757,7 +3757,7 @@ cdef class Generator: if size is None: shape = [] - elif isinstance(size, (int, long, np.integer)): + elif isinstance(size, (int, np.integer)): shape = [size] else: shape = size From f9b6c2f80ed8c7d8b43854b6496d91cc9ea85e32 Mon Sep 17 00:00:00 2001 From: mattip Date: Sun, 26 Nov 2023 17:09:22 +0200 Subject: [PATCH 41/94] BLD: sync azure buld commands with main --- azure-steps-windows.yml | 38 +++++++++----------------------------- 1 file changed, 9 insertions(+), 29 deletions(-) diff --git a/azure-steps-windows.yml b/azure-steps-windows.yml index 1cae980b4368..adc95e672c31 100644 --- a/azure-steps-windows.yml +++ b/azure-steps-windows.yml @@ -24,20 +24,8 @@ steps: displayName: 'Install utilities' - powershell: | - $ErrorActionPreference = "Stop" - mkdir C:/opt/openblas/openblas_dll - mkdir C:/opt/32/lib/pkgconfig - mkdir C:/opt/64/lib/pkgconfig - $target=$(python -c "import tools.openblas_support as obs; plat=obs.get_plat(); ilp64=obs.get_ilp64(); target=f'openblas_{plat}.zip'; obs.download_openblas(target, plat, ilp64);print(target)") - unzip -o -d c:/opt/ $target - echo "##vso[task.setvariable variable=PKG_CONFIG_PATH]c:/opt/64/lib/pkgconfig" - copy C:/opt/64/bin/*.dll C:/opt/openblas/openblas_dll - displayName: 'Download / Install OpenBLAS' - -- powershell: | - # Note: ensure the `pip install .` command remains the last one here, to - # avoid "green on failure" issues - python -c "from tools import openblas_support; openblas_support.make_init('numpy')" + # Note: ensure the `pip install .` command remains the last one here, + # to avoid "green on failure" issues If ( Test-Path env:DISABLE_BLAS ) { python -m pip install . -v -Csetup-args="--vsenv" -Csetup-args="-Dblas=none" -Csetup-args="-Dlapack=none" -Csetup-args="-Dallow-noblas=true" } @@ -45,25 +33,17 @@ steps: python -m pip install scipy-openblas64 spin spin config-openblas --with-scipy-openblas=64 $env:PKG_CONFIG_PATH="$pwd/.openblas" - python -m pip install . -v -Csetup-args="--vsenv" -Csetup-args="-Duse-ilp64=true" - } else { + # use-ilp64 is no longer needed with scipy-openblas > 0.3.24.95.0 + # python -m pip install . -v -Csetup-args="--vsenv" -Csetup-args="-Duse-ilp64=true" python -m pip install . -v -Csetup-args="--vsenv" + } else { + python -m pip install scipy-openblas32 spin + spin config-openblas --with-scipy-openblas=32 + $env:PKG_CONFIG_PATH="$pwd/.openblas" + python -m pip install . -v -Csetup-args="--vsenv" } displayName: 'Build NumPy' -- powershell: | - # copy from c:/opt/openblas/openblas_dll to numpy/../numpy.libs to ensure it can - # get loaded when numpy is imported (no RPATH on Windows) - $target = $(python -c "import sysconfig; print(sysconfig.get_path('platlib'))") - mkdir $target/numpy.libs - copy C:/opt/openblas/openblas_dll/*.dll $target/numpy.libs - displayName: 'Copy OpenBLAS DLL to site-packages' - -- script: | - python -m pip install threadpoolctl - python tools/openblas_support.py --check_version - displayName: 'Check OpenBLAS version' - - powershell: | cd tools # avoid root dir to not pick up source tree # Get a gfortran onto the path for f2py tests From 4a06d145617d4aeee7ba79e20e0043c4355f0ade Mon Sep 17 00:00:00 2001 From: mattip Date: Sun, 26 Nov 2023 21:35:29 +0200 Subject: [PATCH 42/94] TST: PyPy needs another gc.collect on latest versions --- numpy/core/tests/test_mem_policy.py | 1 + 1 file changed, 1 insertion(+) diff --git a/numpy/core/tests/test_mem_policy.py b/numpy/core/tests/test_mem_policy.py index bc3f330dc197..a381fa1d8905 100644 --- a/numpy/core/tests/test_mem_policy.py +++ b/numpy/core/tests/test_mem_policy.py @@ -440,3 +440,4 @@ def test_owner_is_base(get_module): with pytest.warns(UserWarning, match='warn_on_free'): del a gc.collect() + gc.collect() From df814cacb8c06a00ff0091e4cfa937cbc9fd4de3 Mon Sep 17 00:00:00 2001 From: DWesl <22566757+DWesl@users.noreply.github.com> Date: Thu, 30 Nov 2023 15:24:39 -0500 Subject: [PATCH 43/94] CI: Install Lapack runtime on Cygwin. Missed dependency on the new version: this should be a short-term workaround. --- .github/workflows/cygwin.yml | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/.github/workflows/cygwin.yml b/.github/workflows/cygwin.yml index 55a0bc50a628..e873d316f70b 100644 --- a/.github/workflows/cygwin.yml +++ b/.github/workflows/cygwin.yml @@ -20,7 +20,7 @@ jobs: runs-on: windows-latest if: "github.repository == 'numpy/numpy'" steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: submodules: recursive fetch-depth: 0 @@ -30,11 +30,8 @@ jobs: platform: x86_64 install-dir: 'C:\tools\cygwin' packages: >- - python39-devel python39-zipp python39-importlib-metadata - python39-cython python39-pip python39-wheel python39-cffi - python39-pytz python39-setuptools python39-pytest - python39-hypothesis liblapack-devel - gcc-fortran gcc-g++ git dash + python39-devel python39-pip python-pip-wheel python-setuptools-wheel + liblapack-devel liblapack0 gcc-fortran gcc-g++ git dash cmake ninja - name: Set Windows PATH uses: egor-tensin/cleanup-path@8469525c8ee3eddabbd3487658621a6235b3c581 # v3 with: @@ -52,10 +49,9 @@ jobs: dash -c "which python3.9; /usr/bin/python3.9 --version -V" - name: Build NumPy wheel run: | - dash -c "/usr/bin/python3.9 -m pip install 'setuptools<49.2.0' pytest pytz cffi pickle5 importlib_metadata typing_extensions" - dash -c "/usr/bin/python3.9 -m pip install -r test_requirements.txt" - dash -c "/usr/bin/python3.9 setup.py bdist_wheel" - - name: Install new NumPy + dash -c "/usr/bin/python3.9 -m pip install build pytest hypothesis pytest-xdist Cython meson" + dash -c "/usr/bin/python3.9 -m build . --wheel -Csetup-args=-Dblas=blas -Csetup-args=-Dlapack=lapack -Csetup-args=-Dcpu-dispatch=none -Csetup-args=-Dcpu-baseline=native" + - name: Install NumPy from wheel run: | bash -c "/usr/bin/python3.9 -m pip install dist/numpy-*cp39*.whl" - name: Rebase NumPy compiled extensions @@ -64,9 +60,10 @@ jobs: - name: Run NumPy test suite shell: "C:\\tools\\cygwin\\bin\\bash.exe -o igncr -eo pipefail {0}" run: | - /usr/bin/python3.9 runtests.py -n + cd tools + /usr/bin/python3.9 -m pytest --pyargs numpy -n2 -m "not slow" - name: Upload wheel if tests fail - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 if: failure() with: name: numpy-cygwin-wheel From 94c19bae9be9441dc80f3cf1388d77138ae2f6ca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Nov 2023 17:26:42 +0000 Subject: [PATCH 44/94] MAINT: Bump conda-incubator/setup-miniconda from 2.2.0 to 3.0.0 Bumps [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda) from 2.2.0 to 3.0.0. - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/3b0f2504dd76ef23b6d31f291f4913fb60ab5ff3...2defc80cc6f4028b1780c50faf08dd505d698976) --- updated-dependencies: - dependency-name: conda-incubator/setup-miniconda dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/macos.yml | 2 +- .github/workflows/wheels.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 959b51e0626b..05448a8b7447 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -51,7 +51,7 @@ jobs: ${{ github.workflow }}-${{ matrix.python-version }}-ccache-macos- - name: Setup Mambaforge - uses: conda-incubator/setup-miniconda@3b0f2504dd76ef23b6d31f291f4913fb60ab5ff3 # v2.2.0 + uses: conda-incubator/setup-miniconda@2defc80cc6f4028b1780c50faf08dd505d698976 # v3.0.0 with: python-version: ${{ matrix.python-version }} channels: conda-forge diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 76757f6f097d..ba40765f8ea8 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -129,7 +129,7 @@ jobs: name: ${{ matrix.python }}-${{ startsWith(matrix.buildplat[1], 'macosx') && 'macosx' || matrix.buildplat[1] }} path: ./wheelhouse/*.whl - - uses: conda-incubator/setup-miniconda@3b0f2504dd76ef23b6d31f291f4913fb60ab5ff3 # v2.2.0 + - uses: conda-incubator/setup-miniconda@2defc80cc6f4028b1780c50faf08dd505d698976 # v3.0.0 with: # for installation of anaconda-client, required for upload to # anaconda.org @@ -216,7 +216,7 @@ jobs: name: sdist path: ./dist/* - - uses: conda-incubator/setup-miniconda@3b0f2504dd76ef23b6d31f291f4913fb60ab5ff3 # v2.2.0 + - uses: conda-incubator/setup-miniconda@2defc80cc6f4028b1780c50faf08dd505d698976 # v3.0.0 with: # for installation of anaconda-client, required for upload to # anaconda.org From 3e280ebf12308e2744e70845470ff0dc338af62c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 30 Nov 2023 17:27:40 +0000 Subject: [PATCH 45/94] MAINT: Bump conda-incubator/setup-miniconda from 3.0.0 to 3.0.1 Bumps [conda-incubator/setup-miniconda](https://github.com/conda-incubator/setup-miniconda) from 3.0.0 to 3.0.1. - [Release notes](https://github.com/conda-incubator/setup-miniconda/releases) - [Changelog](https://github.com/conda-incubator/setup-miniconda/blob/main/CHANGELOG.md) - [Commits](https://github.com/conda-incubator/setup-miniconda/compare/2defc80cc6f4028b1780c50faf08dd505d698976...11b562958363ec5770fef326fe8ef0366f8cbf8a) --- updated-dependencies: - dependency-name: conda-incubator/setup-miniconda dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/macos.yml | 2 +- .github/workflows/wheels.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 05448a8b7447..ee9db137641d 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -51,7 +51,7 @@ jobs: ${{ github.workflow }}-${{ matrix.python-version }}-ccache-macos- - name: Setup Mambaforge - uses: conda-incubator/setup-miniconda@2defc80cc6f4028b1780c50faf08dd505d698976 # v3.0.0 + uses: conda-incubator/setup-miniconda@11b562958363ec5770fef326fe8ef0366f8cbf8a # v3.0.1 with: python-version: ${{ matrix.python-version }} channels: conda-forge diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index ba40765f8ea8..2049322173c6 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -129,7 +129,7 @@ jobs: name: ${{ matrix.python }}-${{ startsWith(matrix.buildplat[1], 'macosx') && 'macosx' || matrix.buildplat[1] }} path: ./wheelhouse/*.whl - - uses: conda-incubator/setup-miniconda@2defc80cc6f4028b1780c50faf08dd505d698976 # v3.0.0 + - uses: conda-incubator/setup-miniconda@11b562958363ec5770fef326fe8ef0366f8cbf8a # v3.0.1 with: # for installation of anaconda-client, required for upload to # anaconda.org @@ -216,7 +216,7 @@ jobs: name: sdist path: ./dist/* - - uses: conda-incubator/setup-miniconda@2defc80cc6f4028b1780c50faf08dd505d698976 # v3.0.0 + - uses: conda-incubator/setup-miniconda@11b562958363ec5770fef326fe8ef0366f8cbf8a # v3.0.1 with: # for installation of anaconda-client, required for upload to # anaconda.org From e6b2fb3aa46a3b7328b9daddd20f9a340e746fa6 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Thu, 7 Dec 2023 15:46:42 +0100 Subject: [PATCH 46/94] BLD: update vendored Meson for AIX shared library fix Closes gh-25244 --- vendored-meson/meson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendored-meson/meson b/vendored-meson/meson index ea5809096404..708d52853d74 160000 --- a/vendored-meson/meson +++ b/vendored-meson/meson @@ -1 +1 @@ -Subproject commit ea580909640457450e03d8b84d1fec9f035d7acb +Subproject commit 708d52853d744916884d66a630f20b68cf4d3674 From d31c7334b024fa04ff2b14b6ff00e141045d522d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 13 Dec 2023 12:25:37 +0100 Subject: [PATCH 47/94] Init `base` in cpu_avx512_kn Fix a false negative due to a warning that `base` could be uninitialized, e.g. on GCC 12.3.0 --- numpy/distutils/checks/cpu_avx512_knl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/distutils/checks/cpu_avx512_knl.c b/numpy/distutils/checks/cpu_avx512_knl.c index b3f4f6976514..ba32a5e50fe1 100644 --- a/numpy/distutils/checks/cpu_avx512_knl.c +++ b/numpy/distutils/checks/cpu_avx512_knl.c @@ -15,7 +15,7 @@ int main(int argc, char **argv) { - int base[128]; + int base[128]{}; __m512d ad = _mm512_loadu_pd((const __m512d*)argv[argc-1]); /* ER */ __m512i a = _mm512_castpd_si512(_mm512_exp2a23_pd(ad)); From 716e58719b0460b0864faef9ba98fd35d3e9b0ca Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 13 Dec 2023 12:34:07 +0100 Subject: [PATCH 48/94] Update C syntax --- numpy/distutils/checks/cpu_avx512_knl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/distutils/checks/cpu_avx512_knl.c b/numpy/distutils/checks/cpu_avx512_knl.c index ba32a5e50fe1..cb55e57aa220 100644 --- a/numpy/distutils/checks/cpu_avx512_knl.c +++ b/numpy/distutils/checks/cpu_avx512_knl.c @@ -15,7 +15,7 @@ int main(int argc, char **argv) { - int base[128]{}; + int base[128]={}; __m512d ad = _mm512_loadu_pd((const __m512d*)argv[argc-1]); /* ER */ __m512i a = _mm512_castpd_si512(_mm512_exp2a23_pd(ad)); From 62174a2539a30fa8d1dbe430c956e0839cff3656 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 12 Dec 2023 16:20:17 +0100 Subject: [PATCH 49/94] Fix failing test_features on SapphireRapids The /proc/cpuinfo flag for AVX512FP16 is spelled avx512_fp16 Add the underscore to the mapping to make the test pass --- numpy/core/tests/test_cpu_features.py | 1 + 1 file changed, 1 insertion(+) diff --git a/numpy/core/tests/test_cpu_features.py b/numpy/core/tests/test_cpu_features.py index 2fad4dfd98d0..48ab30a4a228 100644 --- a/numpy/core/tests/test_cpu_features.py +++ b/numpy/core/tests/test_cpu_features.py @@ -351,6 +351,7 @@ class Test_X86_Features(AbstractTest): SSE3="PNI", SSE41="SSE4_1", SSE42="SSE4_2", FMA3="FMA", AVX512VNNI="AVX512_VNNI", AVX512BITALG="AVX512_BITALG", AVX512VBMI2="AVX512_VBMI2", AVX5124FMAPS="AVX512_4FMAPS", AVX5124VNNIW="AVX512_4VNNIW", AVX512VPOPCNTDQ="AVX512_VPOPCNTDQ", + AVX512FP16="AVX512_FP16", ) def load_flags(self): self.load_flags_cpuinfo("flags") From bfde240656766867f2ae4ee3cd35016ac61a478a Mon Sep 17 00:00:00 2001 From: Sayed Adel Date: Fri, 24 Nov 2023 10:05:36 +0200 Subject: [PATCH 50/94] BUG: Fix non-contiguous memory load when ARM/Neon is enabled --- numpy/core/src/common/simd/neon/memory.h | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/numpy/core/src/common/simd/neon/memory.h b/numpy/core/src/common/simd/neon/memory.h index 2dc21e5a4305..2202013f982c 100644 --- a/numpy/core/src/common/simd/neon/memory.h +++ b/numpy/core/src/common/simd/neon/memory.h @@ -52,21 +52,12 @@ NPYV_IMPL_NEON_MEM(f64, double) ***************************/ NPY_FINLINE npyv_s32 npyv_loadn_s32(const npy_int32 *ptr, npy_intp stride) { - switch (stride) { - case 2: - return vld2q_s32((const int32_t*)ptr).val[0]; - case 3: - return vld3q_s32((const int32_t*)ptr).val[0]; - case 4: - return vld4q_s32((const int32_t*)ptr).val[0]; - default:; - int32x2_t ax = vcreate_s32(*ptr); - int32x4_t a = vcombine_s32(ax, ax); - a = vld1q_lane_s32((const int32_t*)ptr + stride, a, 1); - a = vld1q_lane_s32((const int32_t*)ptr + stride*2, a, 2); - a = vld1q_lane_s32((const int32_t*)ptr + stride*3, a, 3); - return a; - } + int32x4_t a; + a = vld1q_lane_s32((const int32_t*)ptr, a, 0); + a = vld1q_lane_s32((const int32_t*)ptr + stride, a, 1); + a = vld1q_lane_s32((const int32_t*)ptr + stride*2, a, 2); + a = vld1q_lane_s32((const int32_t*)ptr + stride*3, a, 3); + return a; } NPY_FINLINE npyv_u32 npyv_loadn_u32(const npy_uint32 *ptr, npy_intp stride) From 20fb4e493c7d1f7a9c25e9e1e0db73e19b76ab5a Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Wed, 22 Nov 2023 21:21:11 +0000 Subject: [PATCH 51/94] MAINT,BUG: Never import distutils above 3.12 [f2py] (#25123) Closes gh-24838 --- numpy/f2py/f2py2e.py | 51 ++++++++++++++++++++------------- numpy/f2py/tests/test_f2py2e.py | 32 +++++++++++++++++++++ 2 files changed, 63 insertions(+), 20 deletions(-) diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index 8265f30a8f8f..3d8325ac966c 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -38,6 +38,7 @@ # outmess=sys.stdout.write show = pprint.pprint outmess = auxfuncs.outmess +MESON_ONLY_VER = (sys.version_info >= (3, 12)) __usage__ =\ f"""Usage: @@ -548,9 +549,10 @@ def preparse_sysargv(): sys.argv = [sys.argv[0]] + remaining_argv backend_key = args.backend - if sys.version_info >= (3, 12) and backend_key == 'distutils': - outmess("Cannot use distutils backend with Python 3.12, using meson backend instead.\n") - backend_key = 'meson' + if MESON_ONLY_VER and backend_key == 'distutils': + outmess("Cannot use distutils backend with Python>=3.12," + " using meson backend instead.\n") + backend_key = "meson" return { "dependencies": args.dependencies or [], @@ -625,21 +627,27 @@ def run_compile(): for s in flib_flags: v = '--fcompiler=' if s[:len(v)] == v: - from numpy.distutils import fcompiler - fcompiler.load_all_fcompiler_classes() - allowed_keys = list(fcompiler.fcompiler_class.keys()) - nv = ov = s[len(v):].lower() - if ov not in allowed_keys: - vmap = {} # XXX - try: - nv = vmap[ov] - except KeyError: - if ov not in vmap.values(): - print('Unknown vendor: "%s"' % (s[len(v):])) - nv = ov - i = flib_flags.index(s) - flib_flags[i] = '--fcompiler=' + nv - continue + if MESON_ONLY_VER or backend_key == 'meson': + outmess( + "--fcompiler cannot be used with meson," + "set compiler with the FC environment variable\n" + ) + else: + from numpy.distutils import fcompiler + fcompiler.load_all_fcompiler_classes() + allowed_keys = list(fcompiler.fcompiler_class.keys()) + nv = ov = s[len(v):].lower() + if ov not in allowed_keys: + vmap = {} # XXX + try: + nv = vmap[ov] + except KeyError: + if ov not in vmap.values(): + print('Unknown vendor: "%s"' % (s[len(v):])) + nv = ov + i = flib_flags.index(s) + flib_flags[i] = '--fcompiler=' + nv + continue for s in del_list: i = flib_flags.index(s) del flib_flags[i] @@ -727,8 +735,11 @@ def validate_modulename(pyf_files, modulename='untitled'): def main(): if '--help-link' in sys.argv[1:]: sys.argv.remove('--help-link') - from numpy.distutils.system_info import show_all - show_all() + if MESON_ONLY_VER: + outmess("Use --dep for meson builds\n") + else: + from numpy.distutils.system_info import show_all + show_all() return # Probably outdated options that were not working before 1.16 diff --git a/numpy/f2py/tests/test_f2py2e.py b/numpy/f2py/tests/test_f2py2e.py index b27df5fbd410..920dcf2d3794 100644 --- a/numpy/f2py/tests/test_f2py2e.py +++ b/numpy/f2py/tests/test_f2py2e.py @@ -213,6 +213,38 @@ def test_untitled_cli(capfd, hello_world_f90, monkeypatch): assert "untitledmodule.c" in out +@pytest.mark.skipif((platform.system() != 'Linux') or (sys.version_info <= (3, 12)), reason='Compiler and 3.12 required') +def test_no_py312_distutils_fcompiler(capfd, hello_world_f90, monkeypatch): + """Check that no distutils imports are performed on 3.12 + CLI :: --fcompiler --help-link --backend distutils + """ + MNAME = "hi" + foutl = get_io_paths(hello_world_f90, mname=MNAME) + ipath = foutl.f90inp + monkeypatch.setattr( + sys, "argv", f"f2py {ipath} -c --fcompiler=gfortran -m {MNAME}".split() + ) + with util.switchdir(ipath.parent): + f2pycli() + out, _ = capfd.readouterr() + assert "--fcompiler cannot be used with meson" in out + monkeypatch.setattr( + sys, "argv", f"f2py --help-link".split() + ) + with util.switchdir(ipath.parent): + f2pycli() + out, _ = capfd.readouterr() + assert "Use --dep for meson builds" in out + MNAME = "hi2" # Needs to be different for a new -c + monkeypatch.setattr( + sys, "argv", f"f2py {ipath} -c -m {MNAME} --backend distutils".split() + ) + with util.switchdir(ipath.parent): + f2pycli() + out, _ = capfd.readouterr() + assert "Cannot use distutils backend with Python>=3.12" in out + + @pytest.mark.xfail def test_f2py_skip(capfd, retreal_f77, monkeypatch): """Tests that functions can be skipped From b532bbd63246898e824612bf08557745b529af81 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Thu, 21 Dec 2023 21:52:02 +0100 Subject: [PATCH 52/94] MAINT: make the import-time check for old Accelerate more specific This avoids triggering a hard import error on any warning; something that may be due to other reasons than using old Accelerate (e.g., gh-25433 is for a specific combination of Clang, macOS and OpenBLAS). Closes gh-25433 --- numpy/__init__.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/numpy/__init__.py b/numpy/__init__.py index 47703b7d492d..91da496a9527 100644 --- a/numpy/__init__.py +++ b/numpy/__init__.py @@ -388,20 +388,25 @@ def _mac_os_check(): pass if sys.platform == "darwin": + from . import exceptions with warnings.catch_warnings(record=True) as w: _mac_os_check() # Throw runtime error, if the test failed Check for warning and error_message - error_message = "" if len(w) > 0: - error_message = "{}: {}".format(w[-1].category.__name__, str(w[-1].message)) - msg = ( - "Polyfit sanity test emitted a warning, most likely due " - "to using a buggy Accelerate backend." - "\nIf you compiled yourself, more information is available at:" - "\nhttps://numpy.org/doc/stable/user/building.html#accelerated-blas-lapack-libraries" - "\nOtherwise report this to the vendor " - "that provided NumPy.\n{}\n".format(error_message)) - raise RuntimeError(msg) + for _wn in w: + if _wn.category is exceptions.RankWarning: + # Ignore other warnings, they may not be relevant (see gh-25433). + error_message = f"{_wn.category.__name__}: {str(_wn.message)}" + msg = ( + "Polyfit sanity test emitted a warning, most likely due " + "to using a buggy Accelerate backend." + "\nIf you compiled yourself, more information is available at:" + "\nhttps://numpy.org/devdocs/building/index.html" + "\nOtherwise report this to the vendor " + "that provided NumPy.\n\n{}\n".format(error_message)) + raise RuntimeError(msg) + del _wn + del w del _mac_os_check # We usually use madvise hugepages support, but on some old kernels it From 9e3059ad817fb329fea9c533bacd192c4d716a54 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Fri, 22 Dec 2023 11:04:53 +0100 Subject: [PATCH 53/94] BUG: fix macOS version checks for Accelerate support This fixes the build for macOS 10.5/10.6, which is still relevant for MacPorts. It also makes version comparisons more robust, so that things won't break if there is ever a macOS 13.10 release. Closes gh-25406 --- vendored-meson/meson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendored-meson/meson b/vendored-meson/meson index 708d52853d74..e6e1bf974b25 160000 --- a/vendored-meson/meson +++ b/vendored-meson/meson @@ -1 +1 @@ -Subproject commit 708d52853d744916884d66a630f20b68cf4d3674 +Subproject commit e6e1bf974b2557974c06254c2447e862a59aae7f From 460b179839b2c5c49359bc635d4bb8c536e8c5ae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 17:54:55 +0000 Subject: [PATCH 54/94] MAINT: Bump actions/setup-node from 3.8.1 to 4.0.1 Bumps [actions/setup-node](https://github.com/actions/setup-node) from 3.8.1 to 4.0.1. - [Release notes](https://github.com/actions/setup-node/releases) - [Commits](https://github.com/actions/setup-node/compare/5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d...b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8) --- updated-dependencies: - dependency-name: actions/setup-node dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/emscripten.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml index 8e23c1bc4836..94e4809074c1 100644 --- a/.github/workflows/emscripten.yml +++ b/.github/workflows/emscripten.yml @@ -62,7 +62,7 @@ jobs: CFLAGS=-g2 LDFLAGS=-g2 pyodide build - name: set up node - uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0 + uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4.0.1 with: node-version: ${{ env.NODE_VERSION }} From 610917e399587a5356bbbdaa9bbe86443a79bb2f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 17:54:57 +0000 Subject: [PATCH 55/94] MAINT: Bump larsoner/circleci-artifacts-redirector-action Bumps [larsoner/circleci-artifacts-redirector-action](https://github.com/larsoner/circleci-artifacts-redirector-action) from 7e80b26b4c58b5d3272effc05491c18296b1ad6e to 328d16f501600fcb4535e1024a538077cd333ea8. - [Release notes](https://github.com/larsoner/circleci-artifacts-redirector-action/releases) - [Commits](https://github.com/larsoner/circleci-artifacts-redirector-action/compare/7e80b26b4c58b5d3272effc05491c18296b1ad6e...328d16f501600fcb4535e1024a538077cd333ea8) --- updated-dependencies: - dependency-name: larsoner/circleci-artifacts-redirector-action dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- .github/workflows/circleci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/circleci.yml b/.github/workflows/circleci.yml index c3d1c0ccf1b0..1347ba3a0f7c 100644 --- a/.github/workflows/circleci.yml +++ b/.github/workflows/circleci.yml @@ -17,7 +17,7 @@ jobs: statuses: write steps: - name: GitHub Action step - uses: larsoner/circleci-artifacts-redirector-action@7e80b26b4c58b5d3272effc05491c18296b1ad6e # master + uses: larsoner/circleci-artifacts-redirector-action@328d16f501600fcb4535e1024a538077cd333ea8 # master with: repo-token: ${{ secrets.GITHUB_TOKEN }} api-token: ${{ secrets.CIRCLE_TOKEN }} From fa79acf90ba12ba764e810f1469cf5b76e6ec12d Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 18 Dec 2023 16:44:47 -0700 Subject: [PATCH 56/94] BUG: avoid seg fault from OOB access in RandomState.set_state() --- numpy/random/mtrand.pyx | 19 ++++++++++--------- numpy/random/tests/test_random.py | 5 +++++ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/numpy/random/mtrand.pyx b/numpy/random/mtrand.pyx index 9ffaa572d87e..690dea1485c9 100644 --- a/numpy/random/mtrand.pyx +++ b/numpy/random/mtrand.pyx @@ -367,15 +367,16 @@ cdef class RandomState: else: if not isinstance(state, (tuple, list)): raise TypeError('state must be a dict or a tuple.') - if state[0] != 'MT19937': - raise ValueError('set_state can only be used with legacy MT19937' - 'state instances.') - st = {'bit_generator': state[0], - 'state': {'key': state[1], 'pos': state[2]}} - if len(state) > 3: - st['has_gauss'] = state[3] - st['gauss'] = state[4] - value = st + with cython.boundscheck(True): + if state[0] != 'MT19937': + raise ValueError('set_state can only be used with legacy ' + 'MT19937 state instances.') + st = {'bit_generator': state[0], + 'state': {'key': state[1], 'pos': state[2]}} + if len(state) > 3: + st['has_gauss'] = state[3] + st['gauss'] = state[4] + value = st self._aug_state.gauss = st.get('gauss', 0.0) self._aug_state.has_gauss = st.get('has_gauss', 0) diff --git a/numpy/random/tests/test_random.py b/numpy/random/tests/test_random.py index e64ace711953..3d081fe1dbd1 100644 --- a/numpy/random/tests/test_random.py +++ b/numpy/random/tests/test_random.py @@ -147,6 +147,11 @@ def test_negative_binomial(self): # arguments without truncation. self.prng.negative_binomial(0.5, 0.5) + def test_set_invalid_state(self): + # gh-25402 + with pytest.raises(IndexError): + self.prng.set_state(()) + class TestRandint: From 9d27974b4db9f524f4787f586390fb58a3012053 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 10 Dec 2023 01:18:37 +0000 Subject: [PATCH 57/94] TST: Add one for gh-25337 Co-authored-by: andrea-bia --- numpy/f2py/tests/src/regression/gh25337/data.f90 | 8 ++++++++ numpy/f2py/tests/src/regression/gh25337/use_data.f90 | 6 ++++++ numpy/f2py/tests/test_regression.py | 11 +++++++++++ 3 files changed, 25 insertions(+) create mode 100644 numpy/f2py/tests/src/regression/gh25337/data.f90 create mode 100644 numpy/f2py/tests/src/regression/gh25337/use_data.f90 diff --git a/numpy/f2py/tests/src/regression/gh25337/data.f90 b/numpy/f2py/tests/src/regression/gh25337/data.f90 new file mode 100644 index 000000000000..483d13ceb95c --- /dev/null +++ b/numpy/f2py/tests/src/regression/gh25337/data.f90 @@ -0,0 +1,8 @@ +module data + real(8) :: shift +contains + subroutine set_shift(in_shift) + real(8), intent(in) :: in_shift + shift = in_shift + end subroutine set_shift +end module data diff --git a/numpy/f2py/tests/src/regression/gh25337/use_data.f90 b/numpy/f2py/tests/src/regression/gh25337/use_data.f90 new file mode 100644 index 000000000000..b3fae8b875d0 --- /dev/null +++ b/numpy/f2py/tests/src/regression/gh25337/use_data.f90 @@ -0,0 +1,6 @@ +subroutine shift_a(dim_a, a) + use data, only: shift + integer, intent(in) :: dim_a + real(8), intent(inout), dimension(dim_a) :: a + a = a + shift +end subroutine shift_a diff --git a/numpy/f2py/tests/test_regression.py b/numpy/f2py/tests/test_regression.py index 044f952f2268..1c1097830943 100644 --- a/numpy/f2py/tests/test_regression.py +++ b/numpy/f2py/tests/test_regression.py @@ -64,3 +64,14 @@ def test_include_path(): fnames_in_dir = os.listdir(incdir) for fname in ("fortranobject.c", "fortranobject.h"): assert fname in fnames_in_dir + + +class TestModuleAndSubroutine(util.F2PyTest): + module_name = "example" + sources = [util.getpath("tests", "src", "regression", "gh25337", "data.f90"), + util.getpath("tests", "src", "regression", "gh25337", "use_data.f90")] + + @pytest.mark.slow + def test_gh25337(self): + self.module.data.set_shift(3) + assert "data" in dir(self.module) From 657263a3fc3e813a4fca16e36760f1751a528f57 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 10 Dec 2023 01:20:12 +0000 Subject: [PATCH 58/94] BUG: Fix gh-25337 [f2py] --- numpy/f2py/f90mod_rules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/f2py/f90mod_rules.py b/numpy/f2py/f90mod_rules.py index 36d26968bb3c..9d4876122da6 100644 --- a/numpy/f2py/f90mod_rules.py +++ b/numpy/f2py/f90mod_rules.py @@ -117,7 +117,7 @@ def dadd(line, s=doc): mfargs.append(n) outmess('\t\tConstructing F90 module support for "%s"...\n' % (m['name'])) - if m['name'] in usenames: + if m['name'] in usenames and not onlyvars: outmess(f"\t\t\tSkipping {m['name']} since it is in 'use'...\n") continue if onlyvars: From b400a5258d0fbf0dd38a317e91a77be5007e3d57 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Mon, 18 Dec 2023 22:58:43 +0100 Subject: [PATCH 59/94] BUG: Fix two errors related to not checking for failed allocations Tracking down issues in the 32bit debug run, it turned out to be a memory leak and the crashes were just memory errors that were not propagated correctly. --- numpy/core/src/multiarray/nditer_constr.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/numpy/core/src/multiarray/nditer_constr.c b/numpy/core/src/multiarray/nditer_constr.c index dfa84c51cda5..427dd3d876bc 100644 --- a/numpy/core/src/multiarray/nditer_constr.c +++ b/numpy/core/src/multiarray/nditer_constr.c @@ -198,6 +198,9 @@ NpyIter_AdvancedNew(int nop, PyArrayObject **op_in, npy_uint32 flags, /* Allocate memory for the iterator */ iter = (NpyIter*) PyObject_Malloc(NIT_SIZEOF_ITERATOR(itflags, ndim, nop)); + if (iter == NULL) { + return NULL; + } NPY_IT_TIME_POINT(c_malloc); From 712014715f9cd48670deec75fbf40932355bc0a6 Mon Sep 17 00:00:00 2001 From: Raghuveer Devulapalli Date: Fri, 22 Dec 2023 14:07:17 -0800 Subject: [PATCH 60/94] CI: enables tests on spr and tgl using Intel SDE --- .github/workflows/linux_simd.yml | 72 +++++++++++++++++++++++++------- 1 file changed, 58 insertions(+), 14 deletions(-) diff --git a/.github/workflows/linux_simd.yml b/.github/workflows/linux_simd.yml index f8b694124d41..0f13dadad456 100644 --- a/.github/workflows/linux_simd.yml +++ b/.github/workflows/linux_simd.yml @@ -148,7 +148,7 @@ jobs: - uses: ./.github/meson_actions name: Build/Test - intel_sde: + intel_sde_avx512: needs: [baseline_only] runs-on: ubuntu-latest steps: @@ -162,7 +162,7 @@ jobs: - name: Install Intel SDE run: | - curl -o /tmp/sde.tar.xz https://downloadmirror.intel.com/784319/sde-external-9.24.0-2023-07-13-lin.tar.xz + curl -o /tmp/sde.tar.xz https://downloadmirror.intel.com/788820/sde-external-9.27.0-2023-09-13-lin.tar.xz mkdir /tmp/sde && tar -xvf /tmp/sde.tar.xz -C /tmp/sde/ sudo mv /tmp/sde/* /opt/sde && sudo ln -s /opt/sde/sde64 /usr/bin/sde @@ -190,6 +190,52 @@ jobs: sde -knm -- python -c "import numpy; numpy.show_config()" && sde -knm -- python -m pytest $NUMPY_SITE/numpy/core/tests/test_simd* + - name: linalg/ufunc/umath tests (TGL) + run: | + export NUMPY_SITE=$(realpath build-install/usr/lib/python*/site-packages/) + export PYTHONPATH="$PYTHONPATH:$NUMPY_SITE" + cd build-install && + sde -tgl -- python -c "import numpy; numpy.show_config()" && + sde -tgl -- python -m pytest $NUMPY_SITE/numpy/core/tests/test_umath* \ + $NUMPY_SITE/numpy/core/tests/test_ufunc.py \ + $NUMPY_SITE/numpy/core/tests/test_multiarray.py \ + $NUMPY_SITE/numpy/linalg/tests/test_* + + + intel_sde_spr: + needs: [baseline_only] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + submodules: recursive + fetch-depth: 0 + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 + with: + python-version: '3.11' + + - name: Install Intel SDE + run: | + curl -o /tmp/sde.tar.xz https://downloadmirror.intel.com/788820/sde-external-9.27.0-2023-09-13-lin.tar.xz + mkdir /tmp/sde && tar -xvf /tmp/sde.tar.xz -C /tmp/sde/ + sudo mv /tmp/sde/* /opt/sde && sudo ln -s /opt/sde/sde64 /usr/bin/sde + + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y g++-13 + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 1 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-13 1 + python -m pip install -r build_requirements.txt + python -m pip install pytest pytest-xdist hypothesis typing_extensions + + - name: Build + run: spin build -- -Dallow-noblas=true -Dcpu-baseline=avx512_spr + + - name: Meson Log + if: always() + run: cat build/meson-logs/meson-log.txt + - name: SIMD tests (SPR) run: | export NUMPY_SITE=$(realpath build-install/usr/lib/python*/site-packages/) @@ -198,15 +244,13 @@ jobs: sde -spr -- python -c "import numpy; numpy.show_config()" && sde -spr -- python -m pytest $NUMPY_SITE/numpy/core/tests/test_simd* - # Can't run on SDE just yet: see https://github.com/numpy/numpy/issues/23545#issuecomment-1659047365 - # - #- name: linalg/ufunc/umath tests (SPR) - # run: | - # export NUMPY_SITE=$(realpath build-install/usr/lib/python*/site-packages/) - # export PYTHONPATH="$PYTHONPATH:$NUMPY_SITE" - # cd build-install && - # sde -spr -- python -c "import numpy; numpy.show_config()" && - # sde -spr -- python -m pytest $NUMPY_SITE/numpy/core/tests/test_umath* \ - # $NUMPY_SITE/numpy/core/tests/test_ufunc.py \ - # $NUMPY_SITE/numpy/linalg/tests/test_* - + - name: linalg/ufunc/umath tests on Intel SPR + run: | + export NUMPY_SITE=$(realpath build-install/usr/lib/python*/site-packages/) + export PYTHONPATH="$PYTHONPATH:$NUMPY_SITE" + cd build-install && + sde -spr -- python -c "import numpy; numpy.show_config()" && + sde -spr -- python -m pytest $NUMPY_SITE/numpy/core/tests/test_umath* \ + $NUMPY_SITE/numpy/core/tests/test_ufunc.py \ + $NUMPY_SITE/numpy/core/tests/test_multiarray.py \ + $NUMPY_SITE/numpy/linalg/tests/test_* From 8c9254c0a9d9af394744cc657f315dc4ac369764 Mon Sep 17 00:00:00 2001 From: Raghuveer Devulapalli Date: Fri, 22 Dec 2023 15:55:36 -0800 Subject: [PATCH 61/94] Add inline keywork to avx512_qsort _Float16 --- numpy/core/src/npysort/x86-simd-sort | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/src/npysort/x86-simd-sort b/numpy/core/src/npysort/x86-simd-sort index 85fbe7d1abca..0631a88763a4 160000 --- a/numpy/core/src/npysort/x86-simd-sort +++ b/numpy/core/src/npysort/x86-simd-sort @@ -1 +1 @@ -Subproject commit 85fbe7d1abca3b9a224ba1c62d52afe9a180f8ef +Subproject commit 0631a88763a4a0a4c9e84d5eeb0ec5d36053730b From a8f153ba11ceb84cfbc0cb90f0d43f4fb35be17a Mon Sep 17 00:00:00 2001 From: Raghuveer Devulapalli Date: Fri, 22 Dec 2023 16:01:55 -0800 Subject: [PATCH 62/94] Include 16-bit qsort for both dispatches --- numpy/core/src/npysort/simd_qsort_16bit.dispatch.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/numpy/core/src/npysort/simd_qsort_16bit.dispatch.cpp b/numpy/core/src/npysort/simd_qsort_16bit.dispatch.cpp index 3f5099758c5f..a75f882ff4a1 100644 --- a/numpy/core/src/npysort/simd_qsort_16bit.dispatch.cpp +++ b/numpy/core/src/npysort/simd_qsort_16bit.dispatch.cpp @@ -9,6 +9,7 @@ #if defined(NPY_HAVE_AVX512_SPR) && !defined(_MSC_VER) #include "x86-simd-sort/src/avx512fp16-16bit-qsort.hpp" + #include "x86-simd-sort/src/avx512-16bit-qsort.hpp" #elif defined(NPY_HAVE_AVX512_ICL) && !defined(_MSC_VER) #include "x86-simd-sort/src/avx512-16bit-qsort.hpp" #endif From 2e0200a20a86aa8f8ebe09ac8ce5173c69c2f769 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Fri, 22 Dec 2023 15:48:52 +0100 Subject: [PATCH 63/94] BLD: fix uninitialized variable warnings from simd/neon/memory.h The warning was this, repeated many times: ``` In file included from ../numpy/_core/src/common/simd/neon/neon.h:76: ../numpy/_core/src/common/simd/neon/memory.h:56:56: warning: variable 'a' is uninitialized when used here [-Wuninitialized] a = vld1q_lane_s32((const int32_t*)ptr, a, 0); ^ ``` [skip cirrus] [skip azp] --- numpy/core/src/common/simd/neon/memory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/src/common/simd/neon/memory.h b/numpy/core/src/common/simd/neon/memory.h index 2202013f982c..e7503b822e03 100644 --- a/numpy/core/src/common/simd/neon/memory.h +++ b/numpy/core/src/common/simd/neon/memory.h @@ -52,7 +52,7 @@ NPYV_IMPL_NEON_MEM(f64, double) ***************************/ NPY_FINLINE npyv_s32 npyv_loadn_s32(const npy_int32 *ptr, npy_intp stride) { - int32x4_t a; + int32x4_t a = vdupq_n_s32(0); a = vld1q_lane_s32((const int32_t*)ptr, a, 0); a = vld1q_lane_s32((const int32_t*)ptr + stride, a, 1); a = vld1q_lane_s32((const int32_t*)ptr + stride*2, a, 2); From 3e6606e4dcf3da887927718d3a9aec6bfb54fd8e Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Wed, 22 Nov 2023 19:32:43 +0000 Subject: [PATCH 64/94] TST: Add one for gh-25207 --- numpy/f2py/tests/src/isocintrin/isoCtests.f90 | 8 +++++++- numpy/f2py/tests/test_isoc.py | 6 ++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/numpy/f2py/tests/src/isocintrin/isoCtests.f90 b/numpy/f2py/tests/src/isocintrin/isoCtests.f90 index 42db6cccc14d..fb66ec407c26 100644 --- a/numpy/f2py/tests/src/isocintrin/isoCtests.f90 +++ b/numpy/f2py/tests/src/isocintrin/isoCtests.f90 @@ -1,5 +1,5 @@ module coddity - use iso_c_binding, only: c_double, c_int + use iso_c_binding, only: c_double, c_int, c_int64_t implicit none contains subroutine c_add(a, b, c) bind(c, name="c_add") @@ -14,4 +14,10 @@ function wat(x, y) result(z) bind(c) z = x + 7 end function wat + ! gh-25207 + subroutine c_add_int64(a, b, c) bind(c) + integer(c_int64_t), intent(in) :: a, b + integer(c_int64_t), intent(out) :: c + c = a + b + end subroutine c_add_int64 end module coddity diff --git a/numpy/f2py/tests/test_isoc.py b/numpy/f2py/tests/test_isoc.py index 7e189bd7b830..c6bbd7e03923 100644 --- a/numpy/f2py/tests/test_isoc.py +++ b/numpy/f2py/tests/test_isoc.py @@ -17,3 +17,9 @@ def test_bindc_function(self): out = self.module.coddity.wat(1, 20) exp_out = 8 assert out == exp_out + + # gh-25207 + def test_bindc_kinds(self): + out = self.module.coddity.c_add_int64(1, 20) + exp_out = 21 + assert out == exp_out From d7e78b60c9f76df7c62f878984957d9915c23058 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Wed, 22 Nov 2023 19:37:51 +0000 Subject: [PATCH 65/94] BUG: Work with the ISOC_BINDINGS more consisently Closes gh-25207 --- numpy/f2py/auxfuncs.py | 59 ++++++++++++++++++++++++++++++++++- numpy/f2py/capi_maps.py | 28 +++-------------- numpy/f2py/tests/test_isoc.py | 15 +++++++++ 3 files changed, 78 insertions(+), 24 deletions(-) diff --git a/numpy/f2py/auxfuncs.py b/numpy/f2py/auxfuncs.py index af277c1a4e92..865ce5a3be32 100644 --- a/numpy/f2py/auxfuncs.py +++ b/numpy/f2py/auxfuncs.py @@ -50,7 +50,7 @@ 'isunsigned_long_longarray', 'isunsigned_short', 'isunsigned_shortarray', 'l_and', 'l_not', 'l_or', 'outmess', 'replace', 'show', 'stripcomma', 'throw_error', 'isattr_value', - 'deep_merge', 'getuseblocks' + 'deep_merge', 'getuseblocks', 'process_f2cmap_dict' ] @@ -945,3 +945,60 @@ def getuseblocks(pymod): if modblock.get('use'): all_uses.extend([x for x in modblock.get("use").keys() if "__" not in x]) return all_uses + +def process_f2cmap_dict(f2cmap_all, new_map, c2py_map): + """ + Update the Fortran-to-C type mapping dictionary with new mappings. + + This function integrates a new mapping dictionary into an existing + Fortran-to-C type mapping dictionary. It ensures that all keys are in + lowercase and validates new entries against a given C-to-Python mapping + dictionary. Redefinitions and invalid entries are reported with a warning. + + Parameters + ---------- + f2cmap_all : dict + The existing Fortran-to-C type mapping dictionary that will be updated. + It should be a dictionary of dictionaries where the main keys represent + Fortran types and the nested dictionaries map Fortran type specifiers + to corresponding C types. + + new_map : dict + A dictionary containing new type mappings to be added to `f2cmap_all`. + The structure should be similar to `f2cmap_all`, with keys representing + Fortran types and values being dictionaries of type specifiers and their + C type equivalents. + + c2py_map : dict + A dictionary used for validating the C types in `new_map`. It maps C + types to corresponding Python types and is used to ensure that the C + types specified in `new_map` are valid. + + Returns + ------- + dict + The updated Fortran-to-C type mapping dictionary. + """ + for k, d1 in new_map.items(): + for k1 in list(d1.keys()): + d1[k1.lower()] = d1.pop(k1) + k_lower = k.lower() + + if k_lower not in f2cmap_all: + f2cmap_all[k_lower] = {} + for k1, v1 in d1.items(): + if v1 in c2py_map: + if k1 in f2cmap_all[k_lower]: + outmess( + "\tWarning: redefinition of {'%s':{'%s':'%s'->'%s'}}\n" + % (k_lower, k1, f2cmap_all[k_lower][k1], v1) + ) + f2cmap_all[k_lower][k1] = v1 + outmess('\tMapping "%s(kind=%s)" to "%s"\n' % (k_lower, k1, v1)) + else: + errmess( + "\tIgnoring map {'%s':{'%s':'%s'}}: '%s' must be in %s\n" + % (k_lower, k1, v1, v1, list(c2py_map.keys())) + ) + + return f2cmap_all diff --git a/numpy/f2py/capi_maps.py b/numpy/f2py/capi_maps.py index 32b6db5c5935..93c001c75f6a 100644 --- a/numpy/f2py/capi_maps.py +++ b/numpy/f2py/capi_maps.py @@ -29,7 +29,7 @@ __all__ = [ 'getctype', 'getstrlength', 'getarrdims', 'getpydocsign', 'getarrdocsign', 'getinit', 'sign2map', 'routsign2map', 'modsign2map', - 'cb_sign2map', 'cb_routsign2map', 'common_sign2map' + 'cb_sign2map', 'cb_routsign2map', 'common_sign2map', 'process_f2cmap_dict' ] @@ -131,7 +131,7 @@ 'byte': {'': 'char'}, } -f2cmap_all = deep_merge(f2cmap_all, iso_c_binding_map) +f2cmap_all = process_f2cmap_dict(f2cmap_all, iso_c_binding_map, c2py_map) f2cmap_default = copy.deepcopy(f2cmap_all) f2cmap_mapped = [] @@ -156,29 +156,11 @@ def load_f2cmap_file(f2cmap_file): outmess('Reading f2cmap from {!r} ...\n'.format(f2cmap_file)) with open(f2cmap_file) as f: d = eval(f.read().lower(), {}, {}) - for k, d1 in d.items(): - for k1 in d1.keys(): - d1[k1.lower()] = d1[k1] - d[k.lower()] = d[k] - for k in d.keys(): - if k not in f2cmap_all: - f2cmap_all[k] = {} - for k1 in d[k].keys(): - if d[k][k1] in c2py_map: - if k1 in f2cmap_all[k]: - outmess( - "\tWarning: redefinition of {'%s':{'%s':'%s'->'%s'}}\n" % (k, k1, f2cmap_all[k][k1], d[k][k1])) - f2cmap_all[k][k1] = d[k][k1] - outmess('\tMapping "%s(kind=%s)" to "%s"\n' % - (k, k1, d[k][k1])) - f2cmap_mapped.append(d[k][k1]) - else: - errmess("\tIgnoring map {'%s':{'%s':'%s'}}: '%s' must be in %s\n" % ( - k, k1, d[k][k1], d[k][k1], list(c2py_map.keys()))) + f2cmap_all = process_f2cmap_dict(f2cmap_all, d, c2py_map) outmess('Successfully applied user defined f2cmap changes\n') except Exception as msg: - errmess( - 'Failed to apply user defined f2cmap changes: %s. Skipping.\n' % (msg)) + errmess('Failed to apply user defined f2cmap changes: %s. Skipping.\n' % (msg)) + cformat_map = {'double': '%g', 'float': '%g', diff --git a/numpy/f2py/tests/test_isoc.py b/numpy/f2py/tests/test_isoc.py index c6bbd7e03923..84c7e4355ef3 100644 --- a/numpy/f2py/tests/test_isoc.py +++ b/numpy/f2py/tests/test_isoc.py @@ -23,3 +23,18 @@ def test_bindc_kinds(self): out = self.module.coddity.c_add_int64(1, 20) exp_out = 21 assert out == exp_out + + +def test_process_f2cmap_dict(): + from numpy.f2py.auxfuncs import process_f2cmap_dict + f2cmap_all = {'integer': {'8': 'long_long'}} + new_map = {'INTEGER': {'4': 'int'}} + c2py_map = {'int': 'int', 'long_long': 'long'} + + expected_result = {'integer': {'8': 'long_long', '4': 'int'}} + + # Call the function + result = process_f2cmap_dict(f2cmap_all, new_map, c2py_map) + + # Assert the result is as expected + assert result == expected_result From 47dfd656d4665a481f02fa4dc43a6224da24225f Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Wed, 22 Nov 2023 19:38:53 +0000 Subject: [PATCH 66/94] MAINT: Kill now unused deep-merge [f2py] --- numpy/f2py/auxfuncs.py | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/numpy/f2py/auxfuncs.py b/numpy/f2py/auxfuncs.py index 865ce5a3be32..d99ae0d6a697 100644 --- a/numpy/f2py/auxfuncs.py +++ b/numpy/f2py/auxfuncs.py @@ -50,7 +50,7 @@ 'isunsigned_long_longarray', 'isunsigned_short', 'isunsigned_shortarray', 'l_and', 'l_not', 'l_or', 'outmess', 'replace', 'show', 'stripcomma', 'throw_error', 'isattr_value', - 'deep_merge', 'getuseblocks', 'process_f2cmap_dict' + 'getuseblocks', 'process_f2cmap_dict' ] @@ -899,28 +899,6 @@ def applyrules(rules, d, var={}): del ret[k] return ret -def deep_merge(dict1, dict2): - """Recursively merge two dictionaries into a new dictionary. - - Parameters: - - dict1: The base dictionary. - - dict2: The dictionary to merge into a copy of dict1. - If a key exists in both, the dict2 value will take precedence. - - Returns: - - A new merged dictionary. - """ - merged_dict = deepcopy(dict1) - for key, value in dict2.items(): - if key in merged_dict: - if isinstance(merged_dict[key], dict) and isinstance(value, dict): - merged_dict[key] = deep_merge(merged_dict[key], value) - else: - merged_dict[key] = value - else: - merged_dict[key] = value - return merged_dict - _f2py_module_name_match = re.compile(r'\s*python\s*module\s*(?P[\w_]+)', re.I).match _f2py_user_module_name_match = re.compile(r'\s*python\s*module\s*(?P[\w_]*?' From 05d161ff7c6d88f160df2b072dbd45a1d228effe Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Wed, 22 Nov 2023 19:49:25 +0000 Subject: [PATCH 67/94] ENH: Populate all maps for iso_c bindings --- numpy/f2py/_isocbind.py | 64 +++++++++++++++++++++++++++++++++++++++++ numpy/f2py/capi_maps.py | 12 +++++--- 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/numpy/f2py/_isocbind.py b/numpy/f2py/_isocbind.py index 81f52fb4dece..76b0a4b6a9e9 100644 --- a/numpy/f2py/_isocbind.py +++ b/numpy/f2py/_isocbind.py @@ -40,6 +40,70 @@ } } +isoc_c2pycode_map = { + 'int': 'i', # int + 'short int': 'h', # short int + 'long': 'l', # long int + 'long long': 'q', # long long int + 'signed char': 'b', # signed char + 'size_t': 'I', # size_t (approx unsigned int) + 'int8_t': 'b', # int8_t + 'int16_t': 'h', # int16_t + 'int32_t': 'i', # int32_t + 'int64_t': 'q', # int64_t + 'int_least8_t': 'b', # int_least8_t + 'int_least16_t': 'h', # int_least16_t + 'int_least32_t': 'i', # int_least32_t + 'int_least64_t': 'q', # int_least64_t + 'int_fast8_t': 'b', # int_fast8_t + 'int_fast16_t': 'h', # int_fast16_t + 'int_fast32_t': 'i', # int_fast32_t + 'int_fast64_t': 'q', # int_fast64_t + 'intmax_t': 'q', # intmax_t (approx long long) + 'intptr_t': 'q', # intptr_t (approx long long) + 'ptrdiff_t': 'q', # intptr_t (approx long long) + 'float': 'f', # float + 'double': 'd', # double + 'long double': 'g', # long double + 'float _Complex': 'F', # float _Complex + 'double _Complex': 'D', # double _Complex + 'long double _Complex': 'G', # long double _Complex + '_Bool': '?', # Bool + 'char': 'c', # char +} + +iso_c2py_map = { + 'int': 'int', + 'short int': 'int', # forced casting + 'long': 'int', + 'long long': 'long', + 'signed char': 'int', # forced casting + 'size_t': 'int', # approx Python int + 'int8_t': 'int', # forced casting + 'int16_t': 'int', # forced casting + 'int32_t': 'int', + 'int64_t': 'long', + 'int_least8_t': 'int', # forced casting + 'int_least16_t': 'int', # forced casting + 'int_least32_t': 'int', + 'int_least64_t': 'long', + 'int_fast8_t': 'int', # forced casting + 'int_fast16_t': 'int', # forced casting + 'int_fast32_t': 'int', + 'int_fast64_t': 'long', + 'intmax_t': 'long', + 'intptr_t': 'long', + 'ptrdiff_t': 'long', + 'float': 'float', + 'double': 'float', # forced casting + 'long double': 'float', # forced casting + 'float _Complex': 'complex', # forced casting + 'double _Complex': 'complex', + 'long double _Complex': 'complex', # forced casting + '_Bool': 'bool', + 'char': 'bytes', # approx Python bytes +} + isoc_kindmap = {} for fortran_type, c_type_dict in iso_c_binding_map.items(): for c_type in c_type_dict.keys(): diff --git a/numpy/f2py/capi_maps.py b/numpy/f2py/capi_maps.py index 93c001c75f6a..1dc63ac43ad9 100644 --- a/numpy/f2py/capi_maps.py +++ b/numpy/f2py/capi_maps.py @@ -19,7 +19,7 @@ import os from .crackfortran import markoutercomma from . import cb_rules -from ._isocbind import iso_c_binding_map +from ._isocbind import iso_c_binding_map, isoc_c2pycode_map, iso_c2py_map # The environment provided by auxfuncs.py is needed for some calls to eval. # As the needed functions cannot be determined by static inspection of the @@ -131,13 +131,17 @@ 'byte': {'': 'char'}, } -f2cmap_all = process_f2cmap_dict(f2cmap_all, iso_c_binding_map, c2py_map) +# Add ISO_C handling +c2pycode_map.update(isoc_c2pycode_map) +c2py_map.update(iso_c2py_map) +f2cmap_all, _ = process_f2cmap_dict(f2cmap_all, iso_c_binding_map, c2py_map) +# End ISO_C handling f2cmap_default = copy.deepcopy(f2cmap_all) f2cmap_mapped = [] def load_f2cmap_file(f2cmap_file): - global f2cmap_all + global f2cmap_all, f2cmap_mapped f2cmap_all = copy.deepcopy(f2cmap_default) @@ -156,7 +160,7 @@ def load_f2cmap_file(f2cmap_file): outmess('Reading f2cmap from {!r} ...\n'.format(f2cmap_file)) with open(f2cmap_file) as f: d = eval(f.read().lower(), {}, {}) - f2cmap_all = process_f2cmap_dict(f2cmap_all, d, c2py_map) + f2cmap_all, f2cmap_mapped = process_f2cmap_dict(f2cmap_all, d, c2py_map) outmess('Successfully applied user defined f2cmap changes\n') except Exception as msg: errmess('Failed to apply user defined f2cmap changes: %s. Skipping.\n' % (msg)) From 1e127aa898ae8aff21a11a0ba6fcde04a10d1fa2 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Wed, 22 Nov 2023 20:13:06 +0000 Subject: [PATCH 68/94] MAINT: Correctly return mapped types [f2py] --- numpy/f2py/auxfuncs.py | 35 +++++++++++++++++++++-------------- numpy/f2py/tests/test_isoc.py | 14 ++++++++------ 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/numpy/f2py/auxfuncs.py b/numpy/f2py/auxfuncs.py index d99ae0d6a697..33fc96306383 100644 --- a/numpy/f2py/auxfuncs.py +++ b/numpy/f2py/auxfuncs.py @@ -926,7 +926,8 @@ def getuseblocks(pymod): def process_f2cmap_dict(f2cmap_all, new_map, c2py_map): """ - Update the Fortran-to-C type mapping dictionary with new mappings. + Update the Fortran-to-C type mapping dictionary with new mappings and + return a list of successfully mapped C types. This function integrates a new mapping dictionary into an existing Fortran-to-C type mapping dictionary. It ensures that all keys are in @@ -954,29 +955,35 @@ def process_f2cmap_dict(f2cmap_all, new_map, c2py_map): Returns ------- - dict - The updated Fortran-to-C type mapping dictionary. + tuple of (dict, list) + The updated Fortran-to-C type mapping dictionary and a list of + successfully mapped C types. """ + f2cmap_mapped = [] + + new_map_lower = {} for k, d1 in new_map.items(): - for k1 in list(d1.keys()): - d1[k1.lower()] = d1.pop(k1) - k_lower = k.lower() + d1_lower = {k1.lower(): v1 for k1, v1 in d1.items()} + new_map_lower[k.lower()] = d1_lower + + for k, d1 in new_map_lower.items(): + if k not in f2cmap_all: + f2cmap_all[k] = {} - if k_lower not in f2cmap_all: - f2cmap_all[k_lower] = {} for k1, v1 in d1.items(): if v1 in c2py_map: - if k1 in f2cmap_all[k_lower]: + if k1 in f2cmap_all[k]: outmess( "\tWarning: redefinition of {'%s':{'%s':'%s'->'%s'}}\n" - % (k_lower, k1, f2cmap_all[k_lower][k1], v1) + % (k, k1, f2cmap_all[k][k1], v1) ) - f2cmap_all[k_lower][k1] = v1 - outmess('\tMapping "%s(kind=%s)" to "%s"\n' % (k_lower, k1, v1)) + f2cmap_all[k][k1] = v1 + outmess('\tMapping "%s(kind=%s)" to "%s"\n' % (k, k1, v1)) + f2cmap_mapped.append(v1) else: errmess( "\tIgnoring map {'%s':{'%s':'%s'}}: '%s' must be in %s\n" - % (k_lower, k1, v1, v1, list(c2py_map.keys())) + % (k, k1, v1, v1, list(c2py_map.keys())) ) - return f2cmap_all + return f2cmap_all, f2cmap_mapped diff --git a/numpy/f2py/tests/test_isoc.py b/numpy/f2py/tests/test_isoc.py index 84c7e4355ef3..a6950daaf54f 100644 --- a/numpy/f2py/tests/test_isoc.py +++ b/numpy/f2py/tests/test_isoc.py @@ -27,14 +27,16 @@ def test_bindc_kinds(self): def test_process_f2cmap_dict(): from numpy.f2py.auxfuncs import process_f2cmap_dict - f2cmap_all = {'integer': {'8': 'long_long'}} - new_map = {'INTEGER': {'4': 'int'}} - c2py_map = {'int': 'int', 'long_long': 'long'} - expected_result = {'integer': {'8': 'long_long', '4': 'int'}} + f2cmap_all = {"integer": {"8": "rubbish_type"}} + new_map = {"INTEGER": {"4": "int"}} + c2py_map = {"int": "int", "rubbish_type": "long"} + + exp_map, exp_maptyp = ({"integer": {"8": "rubbish_type", "4": "int"}}, ["int"]) # Call the function - result = process_f2cmap_dict(f2cmap_all, new_map, c2py_map) + res_map, res_maptyp = process_f2cmap_dict(f2cmap_all, new_map, c2py_map) # Assert the result is as expected - assert result == expected_result + assert res_map == exp_map + assert res_maptyp == exp_maptyp From 62676de04402bf9f8b8a4c6cab697f7c96c3c86b Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Wed, 22 Nov 2023 20:22:17 +0000 Subject: [PATCH 69/94] DOC: Add a note on the iso_c2py_map implementation --- numpy/f2py/_isocbind.py | 67 ++++++++++++++++++++++------------------- numpy/f2py/cfuncs.py | 2 ++ 2 files changed, 38 insertions(+), 31 deletions(-) diff --git a/numpy/f2py/_isocbind.py b/numpy/f2py/_isocbind.py index 76b0a4b6a9e9..15ca245692ba 100644 --- a/numpy/f2py/_isocbind.py +++ b/numpy/f2py/_isocbind.py @@ -72,37 +72,42 @@ 'char': 'c', # char } -iso_c2py_map = { - 'int': 'int', - 'short int': 'int', # forced casting - 'long': 'int', - 'long long': 'long', - 'signed char': 'int', # forced casting - 'size_t': 'int', # approx Python int - 'int8_t': 'int', # forced casting - 'int16_t': 'int', # forced casting - 'int32_t': 'int', - 'int64_t': 'long', - 'int_least8_t': 'int', # forced casting - 'int_least16_t': 'int', # forced casting - 'int_least32_t': 'int', - 'int_least64_t': 'long', - 'int_fast8_t': 'int', # forced casting - 'int_fast16_t': 'int', # forced casting - 'int_fast32_t': 'int', - 'int_fast64_t': 'long', - 'intmax_t': 'long', - 'intptr_t': 'long', - 'ptrdiff_t': 'long', - 'float': 'float', - 'double': 'float', # forced casting - 'long double': 'float', # forced casting - 'float _Complex': 'complex', # forced casting - 'double _Complex': 'complex', - 'long double _Complex': 'complex', # forced casting - '_Bool': 'bool', - 'char': 'bytes', # approx Python bytes -} +# TODO: At some point these should be included, but then they'd need special +# handling in cfuncs.py e.g. needs[int64_t_from_pyobj] These are not very hard +# to add, since they all derive from the base `int_from_pyobj`, e.g. the way +# `short_from_pyobj` and others do +iso_c2py_map = {} +# iso_c2py_map = { +# 'int': 'int', +# 'short int': 'int', # forced casting +# 'long': 'int', +# 'long long': 'long', +# 'signed char': 'int', # forced casting +# 'size_t': 'int', # approx Python int +# 'int8_t': 'int', # forced casting +# 'int16_t': 'int', # forced casting +# 'int32_t': 'int', +# 'int64_t': 'long', +# 'int_least8_t': 'int', # forced casting +# 'int_least16_t': 'int', # forced casting +# 'int_least32_t': 'int', +# 'int_least64_t': 'long', +# 'int_fast8_t': 'int', # forced casting +# 'int_fast16_t': 'int', # forced casting +# 'int_fast32_t': 'int', +# 'int_fast64_t': 'long', +# 'intmax_t': 'long', +# 'intptr_t': 'long', +# 'ptrdiff_t': 'long', +# 'float': 'float', +# 'double': 'float', # forced casting +# 'long double': 'float', # forced casting +# 'float _Complex': 'complex', # forced casting +# 'double _Complex': 'complex', +# 'long double _Complex': 'complex', # forced casting +# '_Bool': 'bool', +# 'char': 'bytes', # approx Python bytes +# } isoc_kindmap = {} for fortran_type, c_type_dict in iso_c_binding_map.items(): diff --git a/numpy/f2py/cfuncs.py b/numpy/f2py/cfuncs.py index 23d9812a9255..67ec4d06f7af 100644 --- a/numpy/f2py/cfuncs.py +++ b/numpy/f2py/cfuncs.py @@ -826,6 +826,8 @@ } """ +# TODO: These should be dynamically generated, too many mapped to int things, +# see note in _isocbind.py needs['char_from_pyobj'] = ['int_from_pyobj'] cfuncs['char_from_pyobj'] = """ static int From 2368305bdc8148bfc55076a8af87a9da380dedf5 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Wed, 22 Nov 2023 20:44:31 +0000 Subject: [PATCH 70/94] MAINT: Reduce chatter to pass tests --- numpy/f2py/_isocbind.py | 4 ++-- numpy/f2py/auxfuncs.py | 17 +++++++++++------ numpy/f2py/capi_maps.py | 2 +- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/numpy/f2py/_isocbind.py b/numpy/f2py/_isocbind.py index 15ca245692ba..102f32493fbb 100644 --- a/numpy/f2py/_isocbind.py +++ b/numpy/f2py/_isocbind.py @@ -67,8 +67,8 @@ 'long double': 'g', # long double 'float _Complex': 'F', # float _Complex 'double _Complex': 'D', # double _Complex - 'long double _Complex': 'G', # long double _Complex - '_Bool': '?', # Bool + 'long double _Complex': 'D', # very approximate complex + '_Bool': 'i', # Bool but not really 'char': 'c', # char } diff --git a/numpy/f2py/auxfuncs.py b/numpy/f2py/auxfuncs.py index 33fc96306383..f5cda876e3be 100644 --- a/numpy/f2py/auxfuncs.py +++ b/numpy/f2py/auxfuncs.py @@ -924,7 +924,7 @@ def getuseblocks(pymod): all_uses.extend([x for x in modblock.get("use").keys() if "__" not in x]) return all_uses -def process_f2cmap_dict(f2cmap_all, new_map, c2py_map): +def process_f2cmap_dict(f2cmap_all, new_map, c2py_map, verbose = False): """ Update the Fortran-to-C type mapping dictionary with new mappings and return a list of successfully mapped C types. @@ -953,6 +953,9 @@ def process_f2cmap_dict(f2cmap_all, new_map, c2py_map): types to corresponding Python types and is used to ensure that the C types specified in `new_map` are valid. + verbose : boolean + A flag used to provide information about the types mapped + Returns ------- tuple of (dict, list) @@ -978,12 +981,14 @@ def process_f2cmap_dict(f2cmap_all, new_map, c2py_map): % (k, k1, f2cmap_all[k][k1], v1) ) f2cmap_all[k][k1] = v1 - outmess('\tMapping "%s(kind=%s)" to "%s"\n' % (k, k1, v1)) + if verbose: + outmess('\tMapping "%s(kind=%s)" to "%s"\n' % (k, k1, v1)) f2cmap_mapped.append(v1) else: - errmess( - "\tIgnoring map {'%s':{'%s':'%s'}}: '%s' must be in %s\n" - % (k, k1, v1, v1, list(c2py_map.keys())) - ) + if verbose: + errmess( + "\tIgnoring map {'%s':{'%s':'%s'}}: '%s' must be in %s\n" + % (k, k1, v1, v1, list(c2py_map.keys())) + ) return f2cmap_all, f2cmap_mapped diff --git a/numpy/f2py/capi_maps.py b/numpy/f2py/capi_maps.py index 1dc63ac43ad9..17776af08c6e 100644 --- a/numpy/f2py/capi_maps.py +++ b/numpy/f2py/capi_maps.py @@ -160,7 +160,7 @@ def load_f2cmap_file(f2cmap_file): outmess('Reading f2cmap from {!r} ...\n'.format(f2cmap_file)) with open(f2cmap_file) as f: d = eval(f.read().lower(), {}, {}) - f2cmap_all, f2cmap_mapped = process_f2cmap_dict(f2cmap_all, d, c2py_map) + f2cmap_all, f2cmap_mapped = process_f2cmap_dict(f2cmap_all, d, c2py_map, True) outmess('Successfully applied user defined f2cmap changes\n') except Exception as msg: errmess('Failed to apply user defined f2cmap changes: %s. Skipping.\n' % (msg)) From 599e4270857491662b1b4b06f9a521c4b3c22ac8 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 26 Nov 2023 20:48:02 +0000 Subject: [PATCH 71/94] BUG: Map to types with bindings [f2py] Fixes gh-25226 MAINT: Cleanup and scope fix better [f2py] --- numpy/f2py/_isocbind.py | 131 ++++++++++++---------------------------- 1 file changed, 39 insertions(+), 92 deletions(-) diff --git a/numpy/f2py/_isocbind.py b/numpy/f2py/_isocbind.py index 102f32493fbb..3043c5d9163f 100644 --- a/numpy/f2py/_isocbind.py +++ b/numpy/f2py/_isocbind.py @@ -1,113 +1,60 @@ +""" +ISO_C_BINDING maps for f2py2e. +Only required declarations/macros/functions will be used. + +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. +Permission to use, modify, and distribute this software is given under the +terms of the NumPy License. + +NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. +""" +# These map to keys in c2py_map, via forced casting for now, see gh-25229 iso_c_binding_map = { 'integer': { 'c_int': 'int', - 'c_short': 'short int', - 'c_long': 'long int', - 'c_long_long': 'long long int', - 'c_signed_char': 'signed char', - 'c_size_t': 'size_t', - 'c_int8_t': 'int8_t', - 'c_int16_t': 'int16_t', - 'c_int32_t': 'int32_t', - 'c_int64_t': 'int64_t', - 'c_int_least8_t': 'int_least8_t', - 'c_int_least16_t': 'int_least16_t', - 'c_int_least32_t': 'int_least32_t', - 'c_int_least64_t': 'int_least64_t', - 'c_int_fast8_t': 'int_fast8_t', - 'c_int_fast16_t': 'int_fast16_t', - 'c_int_fast32_t': 'int_fast32_t', - 'c_int_fast64_t': 'int_fast64_t', - 'c_intmax_t': 'intmax_t', - 'c_intptr_t': 'intptr_t', - 'c_ptrdiff_t': 'intptr_t', + 'c_short': 'short', # 'short' <=> 'int' for now + 'c_long': 'long', # 'long' <=> 'int' for now + 'c_long_long': 'long_long', + 'c_signed_char': 'signed_char', + 'c_size_t': 'unsigned', # size_t <=> 'unsigned' for now + 'c_int8_t': 'signed_char', # int8_t <=> 'signed_char' for now + 'c_int16_t': 'short', # int16_t <=> 'short' for now + 'c_int32_t': 'int', # int32_t <=> 'int' for now + 'c_int64_t': 'long_long', + 'c_int_least8_t': 'signed_char', # int_least8_t <=> 'signed_char' for now + 'c_int_least16_t': 'short', # int_least16_t <=> 'short' for now + 'c_int_least32_t': 'int', # int_least32_t <=> 'int' for now + 'c_int_least64_t': 'long_long', + 'c_int_fast8_t': 'signed_char', # int_fast8_t <=> 'signed_char' for now + 'c_int_fast16_t': 'short', # int_fast16_t <=> 'short' for now + 'c_int_fast32_t': 'int', # int_fast32_t <=> 'int' for now + 'c_int_fast64_t': 'long_long', + 'c_intmax_t': 'long_long', # intmax_t <=> 'long_long' for now + 'c_intptr_t': 'long', # intptr_t <=> 'long' for now + 'c_ptrdiff_t': 'long', # ptrdiff_t <=> 'long' for now }, 'real': { 'c_float': 'float', 'c_double': 'double', - 'c_long_double': 'long double' + 'c_long_double': 'long_double' }, 'complex': { - 'c_float_complex': 'float _Complex', - 'c_double_complex': 'double _Complex', - 'c_long_double_complex': 'long double _Complex' + 'c_float_complex': 'complex_float', + 'c_double_complex': 'complex_double', + 'c_long_double_complex': 'complex_long_double' }, 'logical': { - 'c_bool': '_Bool' + 'c_bool': 'unsigned_char' # _Bool <=> 'unsigned_char' for now }, 'character': { 'c_char': 'char' } } -isoc_c2pycode_map = { - 'int': 'i', # int - 'short int': 'h', # short int - 'long': 'l', # long int - 'long long': 'q', # long long int - 'signed char': 'b', # signed char - 'size_t': 'I', # size_t (approx unsigned int) - 'int8_t': 'b', # int8_t - 'int16_t': 'h', # int16_t - 'int32_t': 'i', # int32_t - 'int64_t': 'q', # int64_t - 'int_least8_t': 'b', # int_least8_t - 'int_least16_t': 'h', # int_least16_t - 'int_least32_t': 'i', # int_least32_t - 'int_least64_t': 'q', # int_least64_t - 'int_fast8_t': 'b', # int_fast8_t - 'int_fast16_t': 'h', # int_fast16_t - 'int_fast32_t': 'i', # int_fast32_t - 'int_fast64_t': 'q', # int_fast64_t - 'intmax_t': 'q', # intmax_t (approx long long) - 'intptr_t': 'q', # intptr_t (approx long long) - 'ptrdiff_t': 'q', # intptr_t (approx long long) - 'float': 'f', # float - 'double': 'd', # double - 'long double': 'g', # long double - 'float _Complex': 'F', # float _Complex - 'double _Complex': 'D', # double _Complex - 'long double _Complex': 'D', # very approximate complex - '_Bool': 'i', # Bool but not really - 'char': 'c', # char -} - -# TODO: At some point these should be included, but then they'd need special -# handling in cfuncs.py e.g. needs[int64_t_from_pyobj] These are not very hard -# to add, since they all derive from the base `int_from_pyobj`, e.g. the way -# `short_from_pyobj` and others do +# TODO: See gh-25229 +isoc_c2pycode_map = {} iso_c2py_map = {} -# iso_c2py_map = { -# 'int': 'int', -# 'short int': 'int', # forced casting -# 'long': 'int', -# 'long long': 'long', -# 'signed char': 'int', # forced casting -# 'size_t': 'int', # approx Python int -# 'int8_t': 'int', # forced casting -# 'int16_t': 'int', # forced casting -# 'int32_t': 'int', -# 'int64_t': 'long', -# 'int_least8_t': 'int', # forced casting -# 'int_least16_t': 'int', # forced casting -# 'int_least32_t': 'int', -# 'int_least64_t': 'long', -# 'int_fast8_t': 'int', # forced casting -# 'int_fast16_t': 'int', # forced casting -# 'int_fast32_t': 'int', -# 'int_fast64_t': 'long', -# 'intmax_t': 'long', -# 'intptr_t': 'long', -# 'ptrdiff_t': 'long', -# 'float': 'float', -# 'double': 'float', # forced casting -# 'long double': 'float', # forced casting -# 'float _Complex': 'complex', # forced casting -# 'double _Complex': 'complex', -# 'long double _Complex': 'complex', # forced casting -# '_Bool': 'bool', -# 'char': 'bytes', # approx Python bytes -# } isoc_kindmap = {} for fortran_type, c_type_dict in iso_c_binding_map.items(): From da7713d32cf8406bf473bdcf86fa378740006a54 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 26 Nov 2023 20:05:46 +0000 Subject: [PATCH 72/94] TST: Add a another test for gh-25207 --- numpy/f2py/tests/src/isocintrin/isoCtests.f90 | 11 +++++++++++ numpy/f2py/tests/test_isoc.py | 10 ++++++++++ 2 files changed, 21 insertions(+) diff --git a/numpy/f2py/tests/src/isocintrin/isoCtests.f90 b/numpy/f2py/tests/src/isocintrin/isoCtests.f90 index fb66ec407c26..765f7c1ce660 100644 --- a/numpy/f2py/tests/src/isocintrin/isoCtests.f90 +++ b/numpy/f2py/tests/src/isocintrin/isoCtests.f90 @@ -20,4 +20,15 @@ subroutine c_add_int64(a, b, c) bind(c) integer(c_int64_t), intent(out) :: c c = a + b end subroutine c_add_int64 + ! gh-25207 + subroutine add_arr(A, B, C) + integer(c_int64_t), intent(in) :: A(3) + integer(c_int64_t), intent(in) :: B(3) + integer(c_int64_t), intent(out) :: C(3) + integer :: j + + do j = 1, 3 + C(j) = A(j)+B(j) + end do + end subroutine end module coddity diff --git a/numpy/f2py/tests/test_isoc.py b/numpy/f2py/tests/test_isoc.py index a6950daaf54f..594bd7caea76 100644 --- a/numpy/f2py/tests/test_isoc.py +++ b/numpy/f2py/tests/test_isoc.py @@ -1,5 +1,7 @@ from . import util import numpy as np +import pytest +from numpy.testing import assert_allclose class TestISOC(util.F2PyTest): sources = [ @@ -24,6 +26,14 @@ def test_bindc_kinds(self): exp_out = 21 assert out == exp_out + # gh-25207 + def test_bindc_add_arr(self): + a = np.array([1,2,3]) + b = np.array([1,2,3]) + out = self.module.coddity.add_arr(a, b) + exp_out = a*2 + assert_allclose(out, exp_out) + def test_process_f2cmap_dict(): from numpy.f2py.auxfuncs import process_f2cmap_dict From b56c7d53f4b6cbd1840d939950d6cd2395141621 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Thu, 30 Nov 2023 00:59:53 +0000 Subject: [PATCH 73/94] BUG: Fix module name bug in signature files [urgent] [f2py] (#25267) * TST: Add one for gh-25263 Co-authored-by: jmrohwer * BUG: Handle modules correctly for F77 Closes gh-25263 --------- Co-authored-by: jmrohwer --- numpy/f2py/f2py2e.py | 14 +++++++++++--- numpy/f2py/tests/test_f2py2e.py | 16 ++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index 3d8325ac966c..f1b88c592c59 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -463,9 +463,17 @@ def run_main(comline_list): pyf_files, _ = filter_files("", "[.]pyf([.]src|)", comline_list) # Checks that no existing modulename is defined in a pyf file # TODO: Remove all this when scaninputline is replaced - if "-h" not in comline_list and args.module_name: # Can't check what doesn't exist yet, -h creates the pyf - modname = validate_modulename(pyf_files, args.module_name) - comline_list += ['-m', modname] # needed for the rest of scaninputline + modname = "untitled" # Default + if args.module_name: + if "-h" in comline_list: + modname = ( + args.module_name + ) # Directly use from args when -h is present + else: + modname = validate_modulename( + pyf_files, args.module_name + ) # Validate modname when -h is not present + comline_list += ['-m', modname] # needed for the rest of scaninputline # gh-22819 -- end files, options = scaninputline(comline_list) auxfuncs.options = options diff --git a/numpy/f2py/tests/test_f2py2e.py b/numpy/f2py/tests/test_f2py2e.py index 920dcf2d3794..659e0e96bd09 100644 --- a/numpy/f2py/tests/test_f2py2e.py +++ b/numpy/f2py/tests/test_f2py2e.py @@ -339,6 +339,22 @@ def test_mod_gen_f77(capfd, hello_world_f90, monkeypatch): assert Path.exists(foutl.wrap77) +def test_mod_gen_gh25263(capfd, hello_world_f77, monkeypatch): + """Check that pyf files are correctly generated with module structure + CLI :: -m -h pyf_file + BUG: numpy-gh #20520 + """ + MNAME = "hi" + foutl = get_io_paths(hello_world_f77, mname=MNAME) + ipath = foutl.finp + monkeypatch.setattr(sys, "argv", f'f2py {ipath} -m {MNAME} -h hi.pyf'.split()) + with util.switchdir(ipath.parent): + f2pycli() + with Path('hi.pyf').open() as hipyf: + pyfdat = hipyf.read() + assert "python module hi" in pyfdat + + def test_lower_cmod(capfd, hello_world_f77, monkeypatch): """Lowers cases by flag or when -h is present From b6df150d070d14a7c1c639d46a91d8f2ab7e117a Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Thu, 30 Nov 2023 23:17:46 +0000 Subject: [PATCH 74/94] BUG: Handle .pyf.src again Closes gh-25286 --- numpy/f2py/f2py2e.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index 3d8325ac966c..6d9c22f13591 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -463,9 +463,16 @@ def run_main(comline_list): pyf_files, _ = filter_files("", "[.]pyf([.]src|)", comline_list) # Checks that no existing modulename is defined in a pyf file # TODO: Remove all this when scaninputline is replaced - if "-h" not in comline_list and args.module_name: # Can't check what doesn't exist yet, -h creates the pyf + modname = None + if "-h" not in comline_list: modname = validate_modulename(pyf_files, args.module_name) - comline_list += ['-m', modname] # needed for the rest of scaninputline + # If -h is present or no valid modname found, use the module name from args + if modname is None and args.module_name: + modname = args.module_name + # If no module name is specified in args or found in pyf_files, use default + if modname is None: + modname = "untitled" + comline_list += ["-m", modname] # needed for the rest of scaninputline # gh-22819 -- end files, options = scaninputline(comline_list) auxfuncs.options = options From 285668af838885b85306975cd7c8c6064ecf2375 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 1 Dec 2023 07:37:11 +0000 Subject: [PATCH 75/94] MAINT: Migrate pyf.src from distutils [skip-ci] --- numpy/f2py/_src_pyf.py | 239 +++++++++++++++++++++++++++++++ numpy/f2py/tests/test_pyf_src.py | 44 ++++++ 2 files changed, 283 insertions(+) create mode 100644 numpy/f2py/_src_pyf.py create mode 100644 numpy/f2py/tests/test_pyf_src.py diff --git a/numpy/f2py/_src_pyf.py b/numpy/f2py/_src_pyf.py new file mode 100644 index 000000000000..6247b95bfe46 --- /dev/null +++ b/numpy/f2py/_src_pyf.py @@ -0,0 +1,239 @@ +import re + +# START OF CODE VENDORED FROM `numpy.distutils.from_template` +############################################################# +""" +process_file(filename) + + takes templated file .xxx.src and produces .xxx file where .xxx + is .pyf .f90 or .f using the following template rules: + + '<..>' denotes a template. + + All function and subroutine blocks in a source file with names that + contain '<..>' will be replicated according to the rules in '<..>'. + + The number of comma-separated words in '<..>' will determine the number of + replicates. + + '<..>' may have two different forms, named and short. For example, + + named: + where anywhere inside a block '

' will be replaced with + 'd', 's', 'z', and 'c' for each replicate of the block. + + <_c> is already defined: <_c=s,d,c,z> + <_t> is already defined: <_t=real,double precision,complex,double complex> + + short: + , a short form of the named, useful when no

appears inside + a block. + + In general, '<..>' contains a comma separated list of arbitrary + expressions. If these expression must contain a comma|leftarrow|rightarrow, + then prepend the comma|leftarrow|rightarrow with a backslash. + + If an expression matches '\\' then it will be replaced + by -th expression. + + Note that all '<..>' forms in a block must have the same number of + comma-separated entries. + + Predefined named template rules: + + + + + +""" + +routine_start_re = re.compile(r'(\n|\A)(( (\$|\*))|)\s*(subroutine|function)\b', re.I) +routine_end_re = re.compile(r'\n\s*end\s*(subroutine|function)\b.*(\n|\Z)', re.I) +function_start_re = re.compile(r'\n (\$|\*)\s*function\b', re.I) + +def parse_structure(astr): + """ Return a list of tuples for each function or subroutine each + tuple is the start and end of a subroutine or function to be + expanded. + """ + + spanlist = [] + ind = 0 + while True: + m = routine_start_re.search(astr, ind) + if m is None: + break + start = m.start() + if function_start_re.match(astr, start, m.end()): + while True: + i = astr.rfind('\n', ind, start) + if i==-1: + break + start = i + if astr[i:i+7]!='\n $': + break + start += 1 + m = routine_end_re.search(astr, m.end()) + ind = end = m and m.end()-1 or len(astr) + spanlist.append((start, end)) + return spanlist + +template_re = re.compile(r"<\s*(\w[\w\d]*)\s*>") +named_re = re.compile(r"<\s*(\w[\w\d]*)\s*=\s*(.*?)\s*>") +list_re = re.compile(r"<\s*((.*?))\s*>") + +def find_repl_patterns(astr): + reps = named_re.findall(astr) + names = {} + for rep in reps: + name = rep[0].strip() or unique_key(names) + repl = rep[1].replace(r'\,', '@comma@') + thelist = conv(repl) + names[name] = thelist + return names + +def find_and_remove_repl_patterns(astr): + names = find_repl_patterns(astr) + astr = re.subn(named_re, '', astr)[0] + return astr, names + +item_re = re.compile(r"\A\\(?P\d+)\Z") +def conv(astr): + b = astr.split(',') + l = [x.strip() for x in b] + for i in range(len(l)): + m = item_re.match(l[i]) + if m: + j = int(m.group('index')) + l[i] = l[j] + return ','.join(l) + +def unique_key(adict): + """ Obtain a unique key given a dictionary.""" + allkeys = list(adict.keys()) + done = False + n = 1 + while not done: + newkey = '__l%s' % (n) + if newkey in allkeys: + n += 1 + else: + done = True + return newkey + + +template_name_re = re.compile(r'\A\s*(\w[\w\d]*)\s*\Z') +def expand_sub(substr, names): + substr = substr.replace(r'\>', '@rightarrow@') + substr = substr.replace(r'\<', '@leftarrow@') + lnames = find_repl_patterns(substr) + substr = named_re.sub(r"<\1>", substr) # get rid of definition templates + + def listrepl(mobj): + thelist = conv(mobj.group(1).replace(r'\,', '@comma@')) + if template_name_re.match(thelist): + return "<%s>" % (thelist) + name = None + for key in lnames.keys(): # see if list is already in dictionary + if lnames[key] == thelist: + name = key + if name is None: # this list is not in the dictionary yet + name = unique_key(lnames) + lnames[name] = thelist + return "<%s>" % name + + substr = list_re.sub(listrepl, substr) # convert all lists to named templates + # newnames are constructed as needed + + numsubs = None + base_rule = None + rules = {} + for r in template_re.findall(substr): + if r not in rules: + thelist = lnames.get(r, names.get(r, None)) + if thelist is None: + raise ValueError('No replicates found for <%s>' % (r)) + if r not in names and not thelist.startswith('_'): + names[r] = thelist + rule = [i.replace('@comma@', ',') for i in thelist.split(',')] + num = len(rule) + + if numsubs is None: + numsubs = num + rules[r] = rule + base_rule = r + elif num == numsubs: + rules[r] = rule + else: + print("Mismatch in number of replacements (base <{}={}>) " + "for <{}={}>. Ignoring.".format(base_rule, ','.join(rules[base_rule]), r, thelist)) + if not rules: + return substr + + def namerepl(mobj): + name = mobj.group(1) + return rules.get(name, (k+1)*[name])[k] + + newstr = '' + for k in range(numsubs): + newstr += template_re.sub(namerepl, substr) + '\n\n' + + newstr = newstr.replace('@rightarrow@', '>') + newstr = newstr.replace('@leftarrow@', '<') + return newstr + +def process_str(allstr): + newstr = allstr + writestr = '' + + struct = parse_structure(newstr) + + oldend = 0 + names = {} + names.update(_special_names) + for sub in struct: + cleanedstr, defs = find_and_remove_repl_patterns(newstr[oldend:sub[0]]) + writestr += cleanedstr + names.update(defs) + writestr += expand_sub(newstr[sub[0]:sub[1]], names) + oldend = sub[1] + writestr += newstr[oldend:] + + return writestr + +include_src_re = re.compile(r"(\n|\A)\s*include\s*['\"](?P[\w\d./\\]+\.src)['\"]", re.I) + +def resolve_includes(source): + d = os.path.dirname(source) + with open(source) as fid: + lines = [] + for line in fid: + m = include_src_re.match(line) + if m: + fn = m.group('name') + if not os.path.isabs(fn): + fn = os.path.join(d, fn) + if os.path.isfile(fn): + lines.extend(resolve_includes(fn)) + else: + lines.append(line) + else: + lines.append(line) + return lines + +def process_file(source): + lines = resolve_includes(source) + return process_str(''.join(lines)) + +_special_names = find_repl_patterns(''' +<_c=s,d,c,z> +<_t=real,double precision,complex,double complex> + + + + + +''') + +# END OF CODE VENDORED FROM `numpy.distutils.from_template` +########################################################### diff --git a/numpy/f2py/tests/test_pyf_src.py b/numpy/f2py/tests/test_pyf_src.py new file mode 100644 index 000000000000..f77ded2f31d4 --- /dev/null +++ b/numpy/f2py/tests/test_pyf_src.py @@ -0,0 +1,44 @@ +# This test is ported from numpy.distutils +from numpy.f2py._src_pyf import process_str +from numpy.testing import assert_equal + + +pyf_src = """ +python module foo + <_rd=real,double precision> + interface + subroutine foosub(tol) + <_rd>, intent(in,out) :: tol + end subroutine foosub + end interface +end python module foo +""" + +expected_pyf = """ +python module foo + interface + subroutine sfoosub(tol) + real, intent(in,out) :: tol + end subroutine sfoosub + subroutine dfoosub(tol) + double precision, intent(in,out) :: tol + end subroutine dfoosub + end interface +end python module foo +""" + + +def normalize_whitespace(s): + """ + Remove leading and trailing whitespace, and convert internal + stretches of whitespace to a single space. + """ + return ' '.join(s.split()) + + +def test_from_template(): + """Regression test for gh-10712.""" + pyf = process_str(pyf_src) + normalized_pyf = normalize_whitespace(pyf) + normalized_expected_pyf = normalize_whitespace(expected_pyf) + assert_equal(normalized_pyf, normalized_expected_pyf) From 23f0de74652555dd00f65c6026d22932d736666c Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 1 Dec 2023 15:20:30 +0000 Subject: [PATCH 76/94] Revert "BUG: Handle .pyf.src again" This reverts commit 9240ebacb7fab1cbafbe94c2d60d26d0cde46bca. --- numpy/f2py/f2py2e.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index 6d9c22f13591..f1b88c592c59 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -463,16 +463,17 @@ def run_main(comline_list): pyf_files, _ = filter_files("", "[.]pyf([.]src|)", comline_list) # Checks that no existing modulename is defined in a pyf file # TODO: Remove all this when scaninputline is replaced - modname = None - if "-h" not in comline_list: - modname = validate_modulename(pyf_files, args.module_name) - # If -h is present or no valid modname found, use the module name from args - if modname is None and args.module_name: - modname = args.module_name - # If no module name is specified in args or found in pyf_files, use default - if modname is None: - modname = "untitled" - comline_list += ["-m", modname] # needed for the rest of scaninputline + modname = "untitled" # Default + if args.module_name: + if "-h" in comline_list: + modname = ( + args.module_name + ) # Directly use from args when -h is present + else: + modname = validate_modulename( + pyf_files, args.module_name + ) # Validate modname when -h is not present + comline_list += ['-m', modname] # needed for the rest of scaninputline # gh-22819 -- end files, options = scaninputline(comline_list) auxfuncs.options = options From b625eb109b399932b3fdc1baa4f0f471c9cd8829 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 1 Dec 2023 16:23:57 +0000 Subject: [PATCH 77/94] BUG: Ensure defines are on newlines [f2py] --- numpy/f2py/cfuncs.py | 51 ++++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/numpy/f2py/cfuncs.py b/numpy/f2py/cfuncs.py index 23d9812a9255..f974c3083feb 100644 --- a/numpy/f2py/cfuncs.py +++ b/numpy/f2py/cfuncs.py @@ -254,14 +254,19 @@ #define slen f2py_slen #define size f2py_size """ -cppmacros[ - 'pyobj_from_char1'] = '#define pyobj_from_char1(v) (PyLong_FromLong(v))' -cppmacros[ - 'pyobj_from_short1'] = '#define pyobj_from_short1(v) (PyLong_FromLong(v))' +cppmacros['pyobj_from_char1'] = r""" +#define pyobj_from_char1(v) (PyLong_FromLong(v)) +""" +cppmacros['pyobj_from_short1'] = r""" +#define pyobj_from_short1(v) (PyLong_FromLong(v)) +""" needs['pyobj_from_int1'] = ['signed_char'] -cppmacros['pyobj_from_int1'] = '#define pyobj_from_int1(v) (PyLong_FromLong(v))' -cppmacros[ - 'pyobj_from_long1'] = '#define pyobj_from_long1(v) (PyLong_FromLong(v))' +cppmacros['pyobj_from_int1'] = r""" +#define pyobj_from_int1(v) (PyLong_FromLong(v)) +""" +cppmacros['pyobj_from_long1'] = r""" +#define pyobj_from_long1(v) (PyLong_FromLong(v)) +""" needs['pyobj_from_long_long1'] = ['long_long'] cppmacros['pyobj_from_long_long1'] = """ #ifdef HAVE_LONG_LONG @@ -272,27 +277,27 @@ #endif """ needs['pyobj_from_long_double1'] = ['long_double'] -cppmacros[ - 'pyobj_from_long_double1'] = '#define pyobj_from_long_double1(v) (PyFloat_FromDouble(v))' -cppmacros[ - 'pyobj_from_double1'] = '#define pyobj_from_double1(v) (PyFloat_FromDouble(v))' -cppmacros[ - 'pyobj_from_float1'] = '#define pyobj_from_float1(v) (PyFloat_FromDouble(v))' +cppmacros['pyobj_from_long_double1'] = """ +#define pyobj_from_long_double1(v) (PyFloat_FromDouble(v))""" +cppmacros['pyobj_from_double1'] = """ +#define pyobj_from_double1(v) (PyFloat_FromDouble(v))""" +cppmacros['pyobj_from_float1'] = """ +#define pyobj_from_float1(v) (PyFloat_FromDouble(v))""" needs['pyobj_from_complex_long_double1'] = ['complex_long_double'] -cppmacros[ - 'pyobj_from_complex_long_double1'] = '#define pyobj_from_complex_long_double1(v) (PyComplex_FromDoubles(v.r,v.i))' +cppmacros['pyobj_from_complex_long_double1'] = """ +#define pyobj_from_complex_long_double1(v) (PyComplex_FromDoubles(v.r,v.i))""" needs['pyobj_from_complex_double1'] = ['complex_double'] -cppmacros[ - 'pyobj_from_complex_double1'] = '#define pyobj_from_complex_double1(v) (PyComplex_FromDoubles(v.r,v.i))' +cppmacros['pyobj_from_complex_double1'] = """ +#define pyobj_from_complex_double1(v) (PyComplex_FromDoubles(v.r,v.i))""" needs['pyobj_from_complex_float1'] = ['complex_float'] -cppmacros[ - 'pyobj_from_complex_float1'] = '#define pyobj_from_complex_float1(v) (PyComplex_FromDoubles(v.r,v.i))' +cppmacros['pyobj_from_complex_float1'] = """ +#define pyobj_from_complex_float1(v) (PyComplex_FromDoubles(v.r,v.i))""" needs['pyobj_from_string1'] = ['string'] -cppmacros[ - 'pyobj_from_string1'] = '#define pyobj_from_string1(v) (PyUnicode_FromString((char *)v))' +cppmacros['pyobj_from_string1'] = """ +#define pyobj_from_string1(v) (PyUnicode_FromString((char *)v))""" needs['pyobj_from_string1size'] = ['string'] -cppmacros[ - 'pyobj_from_string1size'] = '#define pyobj_from_string1size(v,len) (PyUnicode_FromStringAndSize((char *)v, len))' +cppmacros['pyobj_from_string1size'] = """ +#define pyobj_from_string1size(v,len) (PyUnicode_FromStringAndSize((char *)v, len))""" needs['TRYPYARRAYTEMPLATE'] = ['PRINTPYOBJERR'] cppmacros['TRYPYARRAYTEMPLATE'] = """ /* New SciPy */ From 1f5b05ee5f05310127e5358430bb20583f4ed9dc Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 1 Dec 2023 21:48:12 +0000 Subject: [PATCH 78/94] BUG: Don't lower .pyf files --- numpy/f2py/f2py2e.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index f1b88c592c59..cea22b61d14f 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -696,12 +696,12 @@ def run_compile(): # Construct wrappers / signatures / things if backend_key == 'meson': - outmess('Using meson backend\nWill pass --lower to f2py\nSee https://numpy.org/doc/stable/f2py/buildtools/meson.html') - f2py_flags.append('--lower') - if pyf_files: - run_main(f" {' '.join(f2py_flags)} {' '.join(pyf_files)}".split()) - else: + if not pyf_files: + outmess('Using meson backend\nWill pass --lower to f2py\nSee https://numpy.org/doc/stable/f2py/buildtools/meson.html\n') + f2py_flags.append('--lower') run_main(f" {' '.join(f2py_flags)} -m {modulename} {' '.join(sources)}".split()) + else: + run_main(f" {' '.join(f2py_flags)} {' '.join(pyf_files)}".split()) # Now use the builder builder = build_backend( From c3551a1c6c028e0c79822ca3820196d144afb3d5 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 1 Dec 2023 22:25:18 +0000 Subject: [PATCH 79/94] TST: Add a test mimicing some flapack checks --- numpy/f2py/tests/src/string/gh25286.f90 | 14 +++++++++++ numpy/f2py/tests/src/string/gh25286.pyf | 12 ++++++++++ numpy/f2py/tests/src/string/gh25286_bc.pyf | 12 ++++++++++ numpy/f2py/tests/test_character.py | 27 ++++++++++++++++++++++ 4 files changed, 65 insertions(+) create mode 100644 numpy/f2py/tests/src/string/gh25286.f90 create mode 100644 numpy/f2py/tests/src/string/gh25286.pyf create mode 100644 numpy/f2py/tests/src/string/gh25286_bc.pyf diff --git a/numpy/f2py/tests/src/string/gh25286.f90 b/numpy/f2py/tests/src/string/gh25286.f90 new file mode 100644 index 000000000000..db1c7100d2ab --- /dev/null +++ b/numpy/f2py/tests/src/string/gh25286.f90 @@ -0,0 +1,14 @@ +subroutine charint(trans, info) + character, intent(in) :: trans + integer, intent(out) :: info + if (trans == 'N') then + info = 1 + else if (trans == 'T') then + info = 2 + else if (trans == 'C') then + info = 3 + else + info = -1 + end if + +end subroutine charint diff --git a/numpy/f2py/tests/src/string/gh25286.pyf b/numpy/f2py/tests/src/string/gh25286.pyf new file mode 100644 index 000000000000..7b9609071bce --- /dev/null +++ b/numpy/f2py/tests/src/string/gh25286.pyf @@ -0,0 +1,12 @@ +python module _char_handling_test + interface + subroutine charint(trans, info) + callstatement (*f2py_func)(&trans, &info) + callprotoargument char*, int* + + character, intent(in), check(trans=='N'||trans=='T'||trans=='C') :: trans = 'N' + integer intent(out) :: info + + end subroutine charint + end interface +end python module _char_handling_test diff --git a/numpy/f2py/tests/src/string/gh25286_bc.pyf b/numpy/f2py/tests/src/string/gh25286_bc.pyf new file mode 100644 index 000000000000..e7b10fa9215e --- /dev/null +++ b/numpy/f2py/tests/src/string/gh25286_bc.pyf @@ -0,0 +1,12 @@ +python module _char_handling_test + interface + subroutine charint(trans, info) + callstatement (*f2py_func)(&trans, &info) + callprotoargument char*, int* + + character, intent(in), check(*trans=='N'||*trans=='T'||*trans=='C') :: trans = 'N' + integer intent(out) :: info + + end subroutine charint + end interface +end python module _char_handling_test diff --git a/numpy/f2py/tests/test_character.py b/numpy/f2py/tests/test_character.py index c928e38741a1..e55b1b6b233f 100644 --- a/numpy/f2py/tests/test_character.py +++ b/numpy/f2py/tests/test_character.py @@ -607,3 +607,30 @@ def test_gh24662(self): with pytest.raises(Exception): aa = "Hi" self.module.string_inout_optional(aa) + + +@pytest.mark.slow +class TestNewCharHandling(util.F2PyTest): + # from v1.24 onwards, gh-19388 + sources = [ + util.getpath("tests", "src", "string", "gh25286.pyf"), + util.getpath("tests", "src", "string", "gh25286.f90") + ] + module_name = "_char_handling_test" + + def test_gh25286(self): + info = self.module.charint('T') + assert info == 2 + +@pytest.mark.slow +class TestBCCharHandling(util.F2PyTest): + # SciPy style, "incorrect" bindings with a hook + sources = [ + util.getpath("tests", "src", "string", "gh25286_bc.pyf"), + util.getpath("tests", "src", "string", "gh25286.f90") + ] + module_name = "_char_handling_test" + + def test_gh25286(self): + info = self.module.charint('T') + assert info == 2 From a12802f74c99b21bdeb23ac6837b517ea84b32b0 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 1 Dec 2023 22:49:38 +0000 Subject: [PATCH 80/94] BUG: Don't autogenerate untitled modules [f2py] --- numpy/f2py/f2py2e.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index cea22b61d14f..d9bb61ada7ac 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -463,7 +463,6 @@ def run_main(comline_list): pyf_files, _ = filter_files("", "[.]pyf([.]src|)", comline_list) # Checks that no existing modulename is defined in a pyf file # TODO: Remove all this when scaninputline is replaced - modname = "untitled" # Default if args.module_name: if "-h" in comline_list: modname = ( @@ -473,7 +472,7 @@ def run_main(comline_list): modname = validate_modulename( pyf_files, args.module_name ) # Validate modname when -h is not present - comline_list += ['-m', modname] # needed for the rest of scaninputline + comline_list += ['-m', modname] # needed for the rest of scaninputline # gh-22819 -- end files, options = scaninputline(comline_list) auxfuncs.options = options From 37ec06b4196fe70da10953559ad1dda00876c342 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Fri, 1 Dec 2023 22:55:14 +0000 Subject: [PATCH 81/94] MAINT: Overwrite compiled extensions with meson Address https://github.com/numpy/numpy/issues/24874#issuecomment-1835632293 --- numpy/f2py/_backends/_meson.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/numpy/f2py/_backends/_meson.py b/numpy/f2py/_backends/_meson.py index 59cc77146f6f..4841bb3d5933 100644 --- a/numpy/f2py/_backends/_meson.py +++ b/numpy/f2py/_backends/_meson.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import errno import shutil import subprocess @@ -88,8 +89,14 @@ def _move_exec_to_root(self, build_dir: Path): walk_dir.glob(f"{self.modulename}*.so"), walk_dir.glob(f"{self.modulename}*.pyd"), ) + # Same behavior as distutils + # https://github.com/numpy/numpy/issues/24874#issuecomment-1835632293 for path_object in path_objects: - shutil.move(path_object, Path.cwd()) + dest_path = Path.cwd() / path_object.name + if dest_path.exists(): + dest_path.unlink() + shutil.copy2(path_object, dest_path) + os.remove(path_object) def _get_build_command(self): return [ From 70a8efeb7c05e8de9730e4c7c4a32c006f0558b2 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 20 Nov 2023 01:32:12 +0000 Subject: [PATCH 82/94] DOC: Rework the build tools page [f2py] --- doc/source/f2py/buildtools/index.rst | 29 ++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/doc/source/f2py/buildtools/index.rst b/doc/source/f2py/buildtools/index.rst index d3bfd93a5324..0696df1345ab 100644 --- a/doc/source/f2py/buildtools/index.rst +++ b/doc/source/f2py/buildtools/index.rst @@ -7,15 +7,18 @@ F2PY and Build Systems In this section we will cover the various popular build systems and their usage with ``f2py``. -.. note:: - **As of November 2021** +.. versionchanged:: NumPy 1.26.x + + The default build system for ``f2py`` has traditionally been through the + enhanced ``numpy.distutils`` module. This module is based on ``distutils`` + which was removed in ``Python 3.12.0`` in **October 2023**. Like the rest of + NumPy and SciPy, ``f2py`` uses ``meson`` now, see + :ref:`distutils-status-migration` for some more details. - The default build system for ``F2PY`` has traditionally been through the - enhanced ``numpy.distutils`` module. This module is based on ``distutils`` which - will be removed in ``Python 3.12.0`` in **October 2023**; ``setuptools`` does not - have support for Fortran or ``F2PY`` and it is unclear if it will be supported - in the future. Alternative methods are thus increasingly more important. +.. note:: + All changes to ``f2py`` are tested on SciPy, so their `CI configuration`_ is + always supported. Basic Concepts =============== @@ -78,12 +81,12 @@ Signature files their contents; which shifts the burden of checking for generated files onto the build system. -.. note:: +.. versionchanged:: NumPy ``1.22.4`` - From NumPy ``1.22.4`` onwards, ``f2py`` will deterministically generate - wrapper files based on the input file Fortran standard (F77 or greater). - ``--skip-empty-wrappers`` can be passed to ``f2py`` to restore the previous - behaviour of only generating wrappers when needed by the input . + ``f2py`` will deterministically generate wrapper files based on the input + file Fortran standard (F77 or greater). ``--skip-empty-wrappers`` can be + passed to ``f2py`` to restore the previous behaviour of only generating + wrappers when needed by the input . In theory keeping the above requirements in hand, any build system can be @@ -106,3 +109,5 @@ Build Systems skbuild .. _`issue 20385`: https://github.com/numpy/numpy/issues/20385 + +.. _`CI configuration`: https://docs.scipy.org/doc/scipy/dev/toolchain.html#official-builds From 203fa05bf6d4d4bae56ee5c0082c8060d66951f6 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 20 Nov 2023 01:33:00 +0000 Subject: [PATCH 83/94] DOC: Cleanup and add some examples --- doc/source/f2py/f2py-examples.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/doc/source/f2py/f2py-examples.rst b/doc/source/f2py/f2py-examples.rst index 8dcdec075a44..a43b65bfdaaf 100644 --- a/doc/source/f2py/f2py-examples.rst +++ b/doc/source/f2py/f2py-examples.rst @@ -31,11 +31,6 @@ Python just like any other extension module. Creating a compiled extension module ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. note:: - - This usage depends heavily on ``numpy.distutils``, see :ref:`f2py-bldsys` - for more details. - You can also get f2py to both compile :file:`add.f` along with the produced extension module leaving only a shared-library extension file that can be imported from Python:: From 414212370b892217ac731bf47a5b7040ffde66ca Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 20 Nov 2023 01:36:32 +0000 Subject: [PATCH 84/94] DOC: Update example list and sources [f2py] --- doc/source/f2py/f2py-examples.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/source/f2py/f2py-examples.rst b/doc/source/f2py/f2py-examples.rst index a43b65bfdaaf..7396d950b697 100644 --- a/doc/source/f2py/f2py-examples.rst +++ b/doc/source/f2py/f2py-examples.rst @@ -6,6 +6,11 @@ F2PY examples Below are some examples of F2PY usage. This list is not comprehensive, but can be used as a starting point when wrapping your own code. +.. note:: + + The best place to look for examples is the `NumPy issue tracker`_, or the test + cases for ``f2py`` + F2PY walkthrough: a basic extension module ------------------------------------------ @@ -238,3 +243,6 @@ Read more * `F2py example: Interactive System for Ice sheet Simulation `_ * `"Interfacing With Other Languages" section on the SciPy Cookbook. `_ + +.. _`NumPy issue tracker`: https://github.com/numpy/numpy/issues?q=is%3Aissue+label%3A%22component%3A+numpy.f2py%22+is%3Aclosed +.. _`test cases`: https://github.com/numpy/numpy/tree/main/doc/source/f2py/code From 0c214d01663ab1cf6f7d1d3371346fa921aed734 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 20 Nov 2023 01:50:22 +0000 Subject: [PATCH 85/94] DOC: Cleanup F2Py doc with reduced distutils info --- doc/source/f2py/f2py.getting-started.rst | 21 +++++++++++++-------- doc/source/f2py/index.rst | 20 ++++++++++++++++---- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/doc/source/f2py/f2py.getting-started.rst b/doc/source/f2py/f2py.getting-started.rst index e9656481415d..51d48da29e02 100644 --- a/doc/source/f2py/f2py.getting-started.rst +++ b/doc/source/f2py/f2py.getting-started.rst @@ -22,25 +22,28 @@ following steps: * F2PY compiles all sources and builds an extension module containing the wrappers. - * In building the extension modules, F2PY uses ``numpy_distutils`` which - supports a number of Fortran 77/90/95 compilers, including Gnu, Intel, Sun - Fortran, SGI MIPSpro, Absoft, NAG, Compaq etc. For different build systems, - see :ref:`f2py-bldsys`. + * In building the extension modules, F2PY uses ``meson`` and used to use + ``numpy.distutils`` For different build systems, see :ref:`f2py-bldsys`. * Depending on your operating system, you may need to install the Python development headers (which provide the file ``Python.h``) separately. In Linux Debian-based distributions this package should be called ``python3-dev``, in Fedora-based distributions it is ``python3-devel``. For macOS, depending how Python was installed, your mileage may vary. In Windows, the headers are - typically installed already. + typically installed already, see :ref:`f2py-windows`. + +.. note:: + + F2PY supports all the operating systems SciPy is tested on so their + `system dependencies panel`_ is a good reference. Depending on the situation, these steps can be carried out in a single composite command or step-by-step; in which case some steps can be omitted or combined with others. -Below, we describe three typical approaches of using F2PY. These can be read in -order of increasing effort, but also cater to different access levels depending -on whether the Fortran code can be freely modified. +Below, we describe three typical approaches of using F2PY with Fortran 77. These +can be read in order of increasing effort, but also cater to different access +levels depending on whether the Fortran code can be freely modified. The following example Fortran 77 code will be used for illustration, save it as ``fib1.f``: @@ -298,3 +301,5 @@ the previous case:: >>> print(fib3.fib(8)) [ 0. 1. 1. 2. 3. 5. 8. 13.] + +.. _`system dependencies panel`: http://scipy.github.io/devdocs/building/index.html#system-level-dependencies diff --git a/doc/source/f2py/index.rst b/doc/source/f2py/index.rst index dedfe3f644dd..37a4d7fba788 100644 --- a/doc/source/f2py/index.rst +++ b/doc/source/f2py/index.rst @@ -5,11 +5,13 @@ F2PY user guide and reference manual ===================================== The purpose of the ``F2PY`` --*Fortran to Python interface generator*-- utility -is to provide a connection between Python and Fortran. F2PY is a part of NumPy_ -(``numpy.f2py``) and also available as a standalone command line tool. +is to provide a connection between Python and Fortran. F2PY distributed as part +of NumPy_ (``numpy.f2py``) and once installed is also available as a standalone +command line tool. Originally created by Pearu Peterson, and older changelogs +are in the `historical reference`_. -F2PY facilitates creating/building Python C/API extension modules that make it -possible +F2PY facilitates creating/building native `Python C/API extension modules`_ that +make it possible * to call Fortran 77/90/95 external subroutines and Fortran 90/95 module subroutines as well as C functions; @@ -18,6 +20,14 @@ possible from Python. + +.. note:: + + Fortran 77 is essentially feature complete, and an increasing amount of + Modern Fortran is supported within F2PY. Most ``iso_c_binding`` interfaces + can be compiled to native extension modules automatically with ``f2py``. + Bug reports welcome! + F2PY can be used either as a command line tool ``f2py`` or as a Python module ``numpy.f2py``. While we try to provide the command line tool as part of the numpy setup, some platforms like Windows make it difficult to @@ -46,3 +56,5 @@ replace all calls to ``f2py`` mentioned in this guide with the longer version. .. _Python: https://www.python.org/ .. _NumPy: https://www.numpy.org/ +.. _`historical reference`: https://web.archive.org/web/20140822061353/http://cens.ioc.ee/projects/f2py2e +.. _Python C/API extension modules: https://docs.python.org/3/extending/extending.html#extending-python-with-c-or-c From 1b7216e012b78f11d4dc6379e7985ccc62d6dfac Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 20 Nov 2023 02:00:41 +0000 Subject: [PATCH 86/94] DOC: More directives and reworked [f2py] text --- doc/source/f2py/buildtools/distutils.rst | 8 ++- doc/source/f2py/buildtools/meson.rst | 19 +++--- doc/source/f2py/python-usage.rst | 8 +-- doc/source/f2py/signature-file.rst | 10 ++-- doc/source/f2py/usage.rst | 74 +++++++++++++++++++++++- doc/source/f2py/windows/index.rst | 4 +- 6 files changed, 103 insertions(+), 20 deletions(-) diff --git a/doc/source/f2py/buildtools/distutils.rst b/doc/source/f2py/buildtools/distutils.rst index 9abeee8b84a7..87e17a811cd0 100644 --- a/doc/source/f2py/buildtools/distutils.rst +++ b/doc/source/f2py/buildtools/distutils.rst @@ -4,6 +4,12 @@ Using via `numpy.distutils` ============================= +.. legacy:: + + ``distutils`` has been removed in favor of ``meson`` see + :ref:`distutils-status-migration`. + + .. currentmodule:: numpy.distutils.core :mod:`numpy.distutils` is part of NumPy, and extends the standard Python @@ -75,4 +81,4 @@ Extensions to ``distutils`` .. code-block:: bash - python -m numpy.f2py -c --help-fcompiler + python -m numpy.f2py -c --backend distutils --help-fcompiler diff --git a/doc/source/f2py/buildtools/meson.rst b/doc/source/f2py/buildtools/meson.rst index e8bfe0ea8e41..c17c5d2ddc87 100644 --- a/doc/source/f2py/buildtools/meson.rst +++ b/doc/source/f2py/buildtools/meson.rst @@ -4,20 +4,25 @@ Using via ``meson`` =================== +.. note:: + + Much of this document is now obsoleted, one can run ``f2py`` with + ``--build-dir`` to get a skeleton ``meson`` project with basic dependencies + setup. + +.. versionchanged:: 1.26.x + + The default build system for ``f2py`` is now ``meson``, see + :ref:`distutils-status-migration` for some more details.. + The key advantage gained by leveraging ``meson`` over the techniques described in :ref:`f2py-distutils` is that this feeds into existing systems and larger projects with ease. ``meson`` has a rather pythonic syntax which makes it more comfortable and amenable to extension for ``python`` users. -.. note:: - - Meson needs to be at-least ``0.46.0`` in order to resolve the ``python`` include directories. - - -Fibonacci Walkthrough (F77) +Fibonacci walkthrough (F77) =========================== - We will need the generated ``C`` wrapper before we can use a general purpose build system like ``meson``. We will acquire this by: diff --git a/doc/source/f2py/python-usage.rst b/doc/source/f2py/python-usage.rst index db1ee1ec8c3b..54f74f02b6bf 100644 --- a/doc/source/f2py/python-usage.rst +++ b/doc/source/f2py/python-usage.rst @@ -9,10 +9,10 @@ and use cases, see :ref:`f2py-examples`. Fortran type objects ==================== -All wrappers for Fortran/C routines, common blocks, or for Fortran 90 module -data generated by F2PY are exposed to Python as ``fortran`` type objects. -Routine wrappers are callable ``fortran`` type objects while wrappers to Fortran -data have attributes referring to data objects. +All wrappers for Fortran/C routines, common blocks, or Fortran 90 module data +generated by F2PY are exposed to Python as ``fortran`` type objects. Routine +wrappers are callable ``fortran`` type objects while wrappers to Fortran data +have attributes referring to data objects. All ``fortran`` type objects have an attribute ``_cpointer`` that contains a :c:type:`PyCapsule` referring to the C pointer of the corresponding Fortran/C function diff --git a/doc/source/f2py/signature-file.rst b/doc/source/f2py/signature-file.rst index bf358528f558..ba370d73582b 100644 --- a/doc/source/f2py/signature-file.rst +++ b/doc/source/f2py/signature-file.rst @@ -5,7 +5,7 @@ The interface definition file (.pyf) is how you can fine-tune the interface between Python and Fortran. The syntax specification for signature files (``.pyf`` files) is modeled on the Fortran 90/95 language specification. Almost -all Fortran 90/95 standard constructs are understood, both in free and fixed +all Fortran standard constructs are understood, both in free and fixed format (recall that Fortran 77 is a subset of Fortran 90/95). F2PY introduces some extensions to the Fortran 90/95 language specification that help in the design of the Fortran to Python interface, making it more "Pythonic". @@ -18,10 +18,10 @@ library is built. .. note:: - Currently, F2PY may fail with valid Fortran constructs, such as intrinsic - modules. If this happens, you can check the - `NumPy GitHub issue tracker `_ for - possible workarounds or work-in-progress ideas. + Currently, F2PY may fail with some valid Fortran constructs. If this happens, + you can check the `NumPy GitHub issue tracker + `_ for possible workarounds or + work-in-progress ideas. In general, the contents of the signature files are case-sensitive. When scanning Fortran codes to generate a signature file, F2PY lowers all cases diff --git a/doc/source/f2py/usage.rst b/doc/source/f2py/usage.rst index d812af9a2042..0e8497efc9d4 100644 --- a/doc/source/f2py/usage.rst +++ b/doc/source/f2py/usage.rst @@ -160,7 +160,79 @@ following options can be used in this mode: libraries (vecLib on MacOSX, ATLAS elsewhere), use ``--link-lapack_opt``. See also ``--help-link`` switch. -.. note:: +.. warning:: + From Python 3.12 onwards, ``distutils`` has been removed. Use environment + variables or native files to interact with ``meson`` instead. See its `FAQ + `__ for more information. + +Among other options (see below) and options described for previous modes, the following can be used. + +.. note:: + + .. versionchanged:: 1.26.0 + There are now two separate build backends which can be used, ``distutils`` + and ``meson``. Users are **strongly** recommended to switch to ``meson`` + since it is the default above Python ``3.12``. + +Common build flags: + +``--backend `` + Specify the build backend for the compilation process. The supported backends + are ``meson`` and ``distutils``. If not specified, defaults to ``distutils``. + On Python 3.12 or higher, the default is ``meson``. +``--f77flags=`` + Specify F77 compiler flags +``--f90flags=`` + Specify F90 compiler flags +``--debug`` + Compile with debugging information +``-l`` + Use the library ```` when linking. +``-D[=]`` + Define macro ```` as ````. +``-U`` + Define macro ```` +``-I

`` + Append directory ```` to the list of directories searched for include + files. +``-L`` + Add directory ```` to the list of directories to be searched for + ``-l``. + +The ``meson`` specific flags are: + +``--dep `` **meson only** + Specify a meson dependency for the module. This may be passed multiple times + for multiple dependencies. Dependencies are stored in a list for further + processing. Example: ``--dep lapack --dep scalapack`` This will identify + "lapack" and "scalapack" as dependencies and remove them from argv, leaving a + dependencies list containing ["lapack", "scalapack"]. + +The older ``distutils`` flags are: + +``--help-fcompiler`` **no meson** + List the available Fortran compilers. +``--fcompiler=`` **no meson** + Specify a Fortran compiler type by vendor. +``--f77exec=`` **no meson** + Specify the path to a F77 compiler +``--f90exec=`` **no meson** + Specify the path to a F90 compiler +``--opt=`` **no meson** + Specify optimization flags +``--arch=`` **no meson** + Specify architecture specific optimization flags +``--noopt`` **no meson** + Compile without optimization flags +``--noarch`` **no meson** + Compile without arch-dependent optimization flags +``link-`` **no meson** + Link the extension module with as defined by + ``numpy_distutils/system_info.py``. E.g. to link with optimized LAPACK + libraries (vecLib on MacOSX, ATLAS elsewhere), use ``--link-lapack_opt``. + See also ``--help-link`` switch. + +.. note:: The ``f2py -c`` option must be applied either to an existing ``.pyf`` file (plus the source/object/library files) or one must specify the diff --git a/doc/source/f2py/windows/index.rst b/doc/source/f2py/windows/index.rst index 124ad1ee66bc..5ab66cb77d23 100644 --- a/doc/source/f2py/windows/index.rst +++ b/doc/source/f2py/windows/index.rst @@ -6,7 +6,7 @@ F2PY and Windows .. warning:: - F2PY support for Windows is not always at par with Linux support + F2PY support for Windows is not always at par with Linux support .. note:: `ScPy's documentation`_ has some information on system-level dependencies @@ -14,7 +14,7 @@ F2PY and Windows Broadly speaking, there are two issues working with F2PY on Windows: -- the lack of actively developed FOSS Fortran compilers, and, +- the lack of actively developed FOSS Fortran compilers, and, - the linking issues related to the C runtime library for building Python-C extensions. The focus of this section is to establish a guideline for developing and From 39d2b90bfcc5440810a73eb280a927732e7b53b5 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 20 Nov 2023 04:10:00 +0000 Subject: [PATCH 87/94] DOC: Begin disutils to meson cheatsheet --- .../f2py/buildtools/distutils-to-meson.rst | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 doc/source/f2py/buildtools/distutils-to-meson.rst diff --git a/doc/source/f2py/buildtools/distutils-to-meson.rst b/doc/source/f2py/buildtools/distutils-to-meson.rst new file mode 100644 index 000000000000..338413928eda --- /dev/null +++ b/doc/source/f2py/buildtools/distutils-to-meson.rst @@ -0,0 +1,190 @@ +.. _f2py-meson-distutils: + + +1 Migrating to ``meson`` +------------------------ + +As per the timeline laid out in :ref:`distutils-status-migration`, +``distutils`` has ceased to be the default build backend for ``f2py``. This page +collects common workflows in both formats. + +.. note:: + + This is a ****living**** document, `pull requests `_ are very welcome! + +1.1 Baseline +~~~~~~~~~~~~ + +We will start out with a slightly modern variation of the classic Fibonnaci +series generator. + +.. code:: fortran + + ! fib.f90 + subroutine fib(a, n) + use iso_c_binding + integer(c_int), intent(in) :: n + integer(c_int), intent(out) :: a(n) + do i = 1, n + if (i .eq. 1) then + a(i) = 0.0d0 + elseif (i .eq. 2) then + a(i) = 1.0d0 + else + a(i) = a(i - 1) + a(i - 2) + end if + end do + end + +This will not win any awards, but can be a reasonable starting point. + +1.2 Compilation options +~~~~~~~~~~~~~~~~~~~~~~~ + +1.2.1 Basic Usage +^^^^^^^^^^^^^^^^^ + +This is unchanged: + +.. code:: bash + + python -m numpy.f2py -c fib.f90 -m fib + ❯ python -c "import fib; print(fib.fib(30))" + [ 0 1 1 2 3 5 8 13 21 34 + 55 89 144 233 377 610 987 1597 2584 4181 + 6765 10946 17711 28657 46368 75025 121393 196418 317811 514229] + +1.2.2 Specify the backend +^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. tab-set:: + + .. tab-item:: Distutils + :sync: distutils + + .. code-block:: bash + + python -m numpy.f2py -c fib.f90 -m fib --backend distutils + + This is the default for Python versions before 3.12. + + .. tab-item:: Meson + :sync: meson + + .. code-block:: bash + + python -m numpy.f2py -c fib.f90 -m fib --backend meson + + This is the only option for Python versions after 3.12. + +1.2.3 Pass a compiler name +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. tab-set:: + + .. tab-item:: Distutils + :sync: distutils + + .. code-block:: bash + + python -m numpy.f2py -c fib.f90 -m fib --backend distutils --fcompiler=gfortran + + .. tab-item:: Meson + :sync: meson + + .. code-block:: bash + + FC="gfortran" python -m numpy.f2py -c fib.f90 -m fib --backend meson + + Native files can also be used. + +Similarly, ``CC`` can be used in both cases to set the ``C`` compiler. Since the +environment variables are generally pretty common across both, so a small +sample is included below. + +.. table:: + + +------------------------------------+-------------------------------+ + | **Name** | **What** | + +------------------------------------+-------------------------------+ + | FC | Fortran compiler | + +------------------------------------+-------------------------------+ + | CC | C compiler | + +------------------------------------+-------------------------------+ + | CFLAGS | C compiler options | + +------------------------------------+-------------------------------+ + | FFLAGS | Fortran compiler options | + +------------------------------------+-------------------------------+ + | LDFLAGS | Linker options | + +------------------------------------+-------------------------------+ + | LD\ :sub:`LIBRARY`\ \ :sub:`PATH`\ | Library file locations (Unix) | + +------------------------------------+-------------------------------+ + | LIBS | Libraries to link against | + +------------------------------------+-------------------------------+ + | PATH | Search path for executables | + +------------------------------------+-------------------------------+ + | LDFLAGS | Linker flags | + +------------------------------------+-------------------------------+ + | CXX | C++ compiler | + +------------------------------------+-------------------------------+ + | CXXFLAGS | C++ compiler options | + +------------------------------------+-------------------------------+ + + +.. note:: + + For Windows, these may not work very reliably, so `native files `_ are likely the + best bet, or by direct `1.3 Customizing builds`_. + +1.2.4 Dependencies +^^^^^^^^^^^^^^^^^^ + +Here, ``meson`` can actually be used to set dependencies more robustly. + +.. tab-set:: + + .. tab-item:: Distutils + :sync: distutils + + .. code-block:: bash + + python -m numpy.f2py -c fib.f90 -m fib --backend distutils -llapack + + Note that this approach in practice is error prone. + + .. tab-item:: Meson + :sync: meson + + .. code-block:: bash + + python -m numpy.f2py -c fib.f90 -m fib --backend meson --dep lapack + + This maps to ``dependency("lapack")`` and so can be used for a wide variety + of dependencies. They can be `customized further `_ + to use CMake or other systems to resolve dependencies. + +1.3 Customizing builds +~~~~~~~~~~~~~~~~~~~~~~ + +.. tab-set:: + + .. tab-item:: Distutils + :sync: distutils + + .. code-block:: bash + + python -m numpy.f2py -c fib.f90 -m fib --backend distutils --build-dir blah + + This can be technically integrated with other codes, see :ref:`f2py-distutils`. + + .. tab-item:: Meson + :sync: meson + + .. code-block:: bash + + python -m numpy.f2py -c fib.f90 -m fib --backend meson --build-dir blah + + The resulting build can be customized via the + `Meson Build How-To Guide `_. + In fact, the resulting set of files can even be commited directly and used + as a meson subproject in a separate codebase. From 4c194e2f8efce9513dde87ac94311468c248db24 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Mon, 20 Nov 2023 04:10:22 +0000 Subject: [PATCH 88/94] DOC: Aggressively link to new document --- doc/source/f2py/buildtools/index.rst | 9 +++++++-- doc/source/f2py/f2py.getting-started.rst | 6 ++++++ doc/source/f2py/index.rst | 1 + 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/doc/source/f2py/buildtools/index.rst b/doc/source/f2py/buildtools/index.rst index 0696df1345ab..7c4c1d8072bf 100644 --- a/doc/source/f2py/buildtools/index.rst +++ b/doc/source/f2py/buildtools/index.rst @@ -15,10 +15,14 @@ with ``f2py``. NumPy and SciPy, ``f2py`` uses ``meson`` now, see :ref:`distutils-status-migration` for some more details. + All changes to ``f2py`` are tested on SciPy, so their `CI configuration`_ is + always supported. + + .. note:: - All changes to ``f2py`` are tested on SciPy, so their `CI configuration`_ is - always supported. + See :ref:`f2py-meson-distutils` for migration information. + Basic Concepts =============== @@ -107,6 +111,7 @@ Build Systems meson cmake skbuild + distutils-to-meson .. _`issue 20385`: https://github.com/numpy/numpy/issues/20385 diff --git a/doc/source/f2py/f2py.getting-started.rst b/doc/source/f2py/f2py.getting-started.rst index 51d48da29e02..45e1b3dcb2b6 100644 --- a/doc/source/f2py/f2py.getting-started.rst +++ b/doc/source/f2py/f2py.getting-started.rst @@ -25,6 +25,12 @@ following steps: * In building the extension modules, F2PY uses ``meson`` and used to use ``numpy.distutils`` For different build systems, see :ref:`f2py-bldsys`. + +.. note:: + + See :ref:`f2py-meson-distutils` for migration information. + + * Depending on your operating system, you may need to install the Python development headers (which provide the file ``Python.h``) separately. In Linux Debian-based distributions this package should be called ``python3-dev``, diff --git a/doc/source/f2py/index.rst b/doc/source/f2py/index.rst index 37a4d7fba788..b2a9067c657e 100644 --- a/doc/source/f2py/index.rst +++ b/doc/source/f2py/index.rst @@ -53,6 +53,7 @@ replace all calls to ``f2py`` mentioned in this guide with the longer version. buildtools/index advanced windows/index + buildtools/distutils-to-meson .. _Python: https://www.python.org/ .. _NumPy: https://www.numpy.org/ From 75d9e85652ada36137bd74a031ee2e92f918033c Mon Sep 17 00:00:00 2001 From: Andrea Date: Sat, 2 Dec 2023 16:31:37 +0100 Subject: [PATCH 89/94] BUG: Add external library handling for meson [f2py] --- .../f2py/buildtools/distutils-to-meson.rst | 21 +++++++++++++ numpy/f2py/_backends/_meson.py | 30 +++++++++++++++++++ numpy/f2py/_backends/meson.build.template | 5 ++++ 3 files changed, 56 insertions(+) diff --git a/doc/source/f2py/buildtools/distutils-to-meson.rst b/doc/source/f2py/buildtools/distutils-to-meson.rst index 338413928eda..236436c03d33 100644 --- a/doc/source/f2py/buildtools/distutils-to-meson.rst +++ b/doc/source/f2py/buildtools/distutils-to-meson.rst @@ -163,6 +163,27 @@ Here, ``meson`` can actually be used to set dependencies more robustly. of dependencies. They can be `customized further `_ to use CMake or other systems to resolve dependencies. +1.2.5 Libraries +^^^^^^^^^^^^^^^ + +Both ``meson`` and ``distutils`` are capable of linking against libraries. + +.. tab-set:: + + .. tab-item:: Distutils + :sync: distutils + + .. code-block:: bash + + python -m numpy.f2py -c fib.f90 -m fib --backend distutils -lmylib -L/path/to/mylib + + .. tab-item:: Meson + :sync: meson + + .. code-block:: bash + + python -m numpy.f2py -c fib.f90 -m fib --backend meson -lmylib -L/path/to/mylib + 1.3 Customizing builds ~~~~~~~~~~~~~~~~~~~~~~ diff --git a/numpy/f2py/_backends/_meson.py b/numpy/f2py/_backends/_meson.py index 4841bb3d5933..04a3ab0b2031 100644 --- a/numpy/f2py/_backends/_meson.py +++ b/numpy/f2py/_backends/_meson.py @@ -21,6 +21,8 @@ def __init__( modulename: str, sources: list[Path], deps: list[str], + libraries: list[str], + library_dirs: list[Path], object_files: list[Path], linker_args: list[str], c_args: list[str], @@ -32,12 +34,15 @@ def __init__( ) self.sources = sources self.deps = deps + self.libraries = libraries + self.library_dirs = library_dirs self.substitutions = {} self.objects = object_files self.pipeline = [ self.initialize_template, self.sources_substitution, self.deps_substitution, + self.libraries_substitution, ] self.build_type = build_type @@ -67,6 +72,29 @@ def deps_substitution(self) -> None: [f"dependency('{dep}')" for dep in self.deps] ) + def libraries_substitution(self) -> None: + self.substitutions["lib_dir_declarations"] = "\n".join( + [ + f"lib_dir_{i} = declare_dependency(link_args : ['-L{lib_dir}'])" + for i, lib_dir in enumerate(self.library_dirs) + ] + ) + + self.substitutions["lib_declarations"] = "\n".join( + [ + f"{lib} = declare_dependency(link_args : ['-l{lib}'])" + for lib in self.libraries + ] + ) + + indent = " " * 21 + self.substitutions["lib_list"] = f"\n{indent}".join( + [f"{lib}," for lib in self.libraries] + ) + self.substitutions["lib_dir_list"] = f"\n{indent}".join( + [f"lib_dir_{i}," for i in range(len(self.library_dirs))] + ) + def generate_meson_build(self): for node in self.pipeline: node() @@ -111,6 +139,8 @@ def write_meson_build(self, build_dir: Path) -> None: self.modulename, self.sources, self.dependencies, + self.libraries, + self.library_dirs, self.extra_objects, self.flib_flags, self.fc_flags, diff --git a/numpy/f2py/_backends/meson.build.template b/numpy/f2py/_backends/meson.build.template index 99bf42e3cbbc..65e3ecb19c30 100644 --- a/numpy/f2py/_backends/meson.build.template +++ b/numpy/f2py/_backends/meson.build.template @@ -32,6 +32,9 @@ inc_np = include_directories(incdir_numpy, incdir_f2py) # gh-25000 quadmath_dep = fc.find_library('quadmath', required: false) +${lib_declarations} +${lib_dir_declarations} + py.extension_module('${modulename}', [ ${source_list}, @@ -42,5 +45,7 @@ ${source_list}, py_dep, quadmath_dep, ${dep_list} +${lib_list} +${lib_dir_list} ], install : true) From 6158be3e99c45ed67edf2ac0a448d38485926a09 Mon Sep 17 00:00:00 2001 From: Stefano Rivera Date: Wed, 6 Dec 2023 08:41:44 -0400 Subject: [PATCH 90/94] Run f2py's meson backend with the same python that ran f2py On Debian, we sometimes have multiple Python 3 versions in the archive. When using f2py3.X, I'd expect meson to run with the same python, not the default python. Not sure how far up the stack this should come from, I kept it simple for this PR, if someone has better ideas, they can direct me. --- numpy/f2py/_backends/_meson.py | 5 +++++ numpy/f2py/_backends/meson.build.template | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/numpy/f2py/_backends/_meson.py b/numpy/f2py/_backends/_meson.py index 4841bb3d5933..10f5256d3eff 100644 --- a/numpy/f2py/_backends/_meson.py +++ b/numpy/f2py/_backends/_meson.py @@ -4,6 +4,7 @@ import errno import shutil import subprocess +import sys from pathlib import Path from ._backend import Backend @@ -25,6 +26,7 @@ def __init__( linker_args: list[str], c_args: list[str], build_type: str, + python_exe: str, ): self.modulename = modulename self.build_template_path = ( @@ -40,6 +42,7 @@ def __init__( self.deps_substitution, ] self.build_type = build_type + self.python_exe = python_exe def meson_build_template(self) -> str: if not self.build_template_path.is_file(): @@ -54,6 +57,7 @@ def meson_build_template(self) -> str: def initialize_template(self) -> None: self.substitutions["modulename"] = self.modulename self.substitutions["buildtype"] = self.build_type + self.substitutions["python"] = self.python_exe def sources_substitution(self) -> None: indent = " " * 21 @@ -115,6 +119,7 @@ def write_meson_build(self, build_dir: Path) -> None: self.flib_flags, self.fc_flags, self.build_type, + sys.executable, ) src = meson_template.generate_meson_build() Path(build_dir).mkdir(parents=True, exist_ok=True) diff --git a/numpy/f2py/_backends/meson.build.template b/numpy/f2py/_backends/meson.build.template index 99bf42e3cbbc..463f9cc2238f 100644 --- a/numpy/f2py/_backends/meson.build.template +++ b/numpy/f2py/_backends/meson.build.template @@ -8,7 +8,7 @@ project('${modulename}', ]) fc = meson.get_compiler('fortran') -py = import('python').find_installation(pure: false) +py = import('python').find_installation('${python}', pure: false) py_dep = py.dependency() incdir_numpy = run_command(py, From bf7265ccb7cfac2a8c1ea681d1cccb4f13b3ee34 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Sun, 24 Dec 2023 11:51:19 -0700 Subject: [PATCH 91/94] MAINT: Update ``numpy/f2py/_backends`` from main. --- numpy/f2py/_backends/_distutils.py | 2 +- numpy/f2py/_backends/_meson.py | 50 ++++++++++++----------- numpy/f2py/_backends/meson.build.template | 5 ++- 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/numpy/f2py/_backends/_distutils.py b/numpy/f2py/_backends/_distutils.py index e548fc543010..e9b22a3921a5 100644 --- a/numpy/f2py/_backends/_distutils.py +++ b/numpy/f2py/_backends/_distutils.py @@ -13,7 +13,7 @@ class DistutilsBackend(Backend): def __init__(sef, *args, **kwargs): warnings.warn( - "distutils has been deprecated since NumPy 1.26." + "distutils has been deprecated since NumPy 1.26.x" "Use the Meson backend instead, or generate wrappers" "without -c and use a custom build script", VisibleDeprecationWarning, diff --git a/numpy/f2py/_backends/_meson.py b/numpy/f2py/_backends/_meson.py index 71760e18607d..f324e0f595fb 100644 --- a/numpy/f2py/_backends/_meson.py +++ b/numpy/f2py/_backends/_meson.py @@ -24,6 +24,7 @@ def __init__( deps: list[str], libraries: list[str], library_dirs: list[Path], + include_dirs: list[Path], object_files: list[Path], linker_args: list[str], c_args: list[str], @@ -38,12 +39,17 @@ def __init__( self.deps = deps self.libraries = libraries self.library_dirs = library_dirs + if include_dirs is not None: + self.include_dirs = include_dirs + else: + self.include_dirs = [] self.substitutions = {} self.objects = object_files self.pipeline = [ self.initialize_template, self.sources_substitution, self.deps_substitution, + self.include_substitution, self.libraries_substitution, ] self.build_type = build_type @@ -67,13 +73,13 @@ def initialize_template(self) -> None: def sources_substitution(self) -> None: indent = " " * 21 self.substitutions["source_list"] = f",\n{indent}".join( - [f"'{source}'" for source in self.sources] + [f"{indent}'{source}'" for source in self.sources] ) def deps_substitution(self) -> None: indent = " " * 21 self.substitutions["dep_list"] = f",\n{indent}".join( - [f"dependency('{dep}')" for dep in self.deps] + [f"{indent}dependency('{dep}')" for dep in self.deps] ) def libraries_substitution(self) -> None: @@ -93,10 +99,16 @@ def libraries_substitution(self) -> None: indent = " " * 21 self.substitutions["lib_list"] = f"\n{indent}".join( - [f"{lib}," for lib in self.libraries] + [f"{indent}{lib}," for lib in self.libraries] ) self.substitutions["lib_dir_list"] = f"\n{indent}".join( - [f"lib_dir_{i}," for i in range(len(self.library_dirs))] + [f"{indent}lib_dir_{i}," for i in range(len(self.library_dirs))] + ) + + def include_substitution(self) -> None: + indent = " " * 21 + self.substitutions["inc_list"] = f",\n{indent}".join( + [f"{indent}'{inc}'" for inc in self.include_dirs] ) def generate_meson_build(self): @@ -130,13 +142,6 @@ def _move_exec_to_root(self, build_dir: Path): shutil.copy2(path_object, dest_path) os.remove(path_object) - def _get_build_command(self): - return [ - "meson", - "setup", - self.meson_build_dir, - ] - def write_meson_build(self, build_dir: Path) -> None: """Writes the meson build file at specified location""" meson_template = MesonTemplate( @@ -145,6 +150,7 @@ def write_meson_build(self, build_dir: Path) -> None: self.dependencies, self.libraries, self.library_dirs, + self.include_dirs, self.extra_objects, self.flib_flags, self.fc_flags, @@ -157,19 +163,14 @@ def write_meson_build(self, build_dir: Path) -> None: meson_build_file.write_text(src) return meson_build_file + def _run_subprocess_command(self, command, cwd): + subprocess.run(command, cwd=cwd, check=True) + def run_meson(self, build_dir: Path): - completed_process = subprocess.run(self._get_build_command(), cwd=build_dir) - if completed_process.returncode != 0: - raise subprocess.CalledProcessError( - completed_process.returncode, completed_process.args - ) - completed_process = subprocess.run( - ["meson", "compile", "-C", self.meson_build_dir], cwd=build_dir - ) - if completed_process.returncode != 0: - raise subprocess.CalledProcessError( - completed_process.returncode, completed_process.args - ) + setup_command = ["meson", "setup", self.meson_build_dir] + self._run_subprocess_command(setup_command, build_dir) + compile_command = ["meson", "compile", "-C", self.meson_build_dir] + self._run_subprocess_command(compile_command, build_dir) def compile(self) -> None: self.sources = _prepare_sources(self.modulename, self.sources, self.build_dir) @@ -183,7 +184,8 @@ def _prepare_sources(mname, sources, bdir): Path(bdir).mkdir(parents=True, exist_ok=True) # Copy sources for source in sources: - shutil.copy(source, bdir) + if Path(source).exists() and Path(source).is_file(): + shutil.copy(source, bdir) generated_sources = [ Path(f"{mname}module.c"), Path(f"{mname}-f2pywrappers2.f90"), diff --git a/numpy/f2py/_backends/meson.build.template b/numpy/f2py/_backends/meson.build.template index 52e34634493a..8e34fdc8d4d6 100644 --- a/numpy/f2py/_backends/meson.build.template +++ b/numpy/f2py/_backends/meson.build.template @@ -40,7 +40,10 @@ py.extension_module('${modulename}', ${source_list}, fortranobject_c ], - include_directories: [inc_np], + include_directories: [ + inc_np, +${inc_list} + ], dependencies : [ py_dep, quadmath_dep, From 44467f2930704028a43e9fbbf5172289618aa789 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Sun, 24 Dec 2023 12:36:02 -0700 Subject: [PATCH 92/94] MAINT: Easy updates of ``f2py/*.py`` from main. These mostly update the copyright notices and a few includes. The trickier updates of ``f2py2e.py`` and ``crackfortran.py`` are left to a later PR. --- numpy/f2py/__init__.py | 8 ++++++++ numpy/f2py/auxfuncs.py | 10 ++-------- numpy/f2py/capi_maps.py | 9 ++------- numpy/f2py/cb_rules.py | 9 ++------- numpy/f2py/cfuncs.py | 8 ++------ numpy/f2py/common_rules.py | 9 ++------- numpy/f2py/crackfortran.py | 6 ++---- numpy/f2py/f2py2e.py | 14 ++++++-------- numpy/f2py/f90mod_rules.py | 9 ++------- numpy/f2py/func2subr.py | 8 ++------ numpy/f2py/rules.py | 7 ++----- numpy/f2py/symbolic.py | 7 +++++++ numpy/f2py/use_rules.py | 11 ++--------- 13 files changed, 41 insertions(+), 74 deletions(-) diff --git a/numpy/f2py/__init__.py b/numpy/f2py/__init__.py index dbe3df27f6ec..e583250f7060 100644 --- a/numpy/f2py/__init__.py +++ b/numpy/f2py/__init__.py @@ -1,13 +1,21 @@ #!/usr/bin/env python3 """Fortran to Python Interface Generator. +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. +Permission to use, modify, and distribute this software is given under the terms +of the NumPy License. + +NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. """ __all__ = ['run_main', 'compile', 'get_include'] import sys import subprocess import os +import warnings +from numpy.exceptions import VisibleDeprecationWarning from . import f2py2e from . import diagnose diff --git a/numpy/f2py/auxfuncs.py b/numpy/f2py/auxfuncs.py index f5cda876e3be..13a1074b447e 100644 --- a/numpy/f2py/auxfuncs.py +++ b/numpy/f2py/auxfuncs.py @@ -1,18 +1,12 @@ -#!/usr/bin/env python3 """ - Auxiliary functions for f2py2e. -Copyright 1999,2000 Pearu Peterson all rights reserved, -Pearu Peterson +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy (BSD style) LICENSE. - NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/07/24 19:01:55 $ -Pearu Peterson - """ import pprint import sys diff --git a/numpy/f2py/capi_maps.py b/numpy/f2py/capi_maps.py index 17776af08c6e..fa477a5b9aca 100644 --- a/numpy/f2py/capi_maps.py +++ b/numpy/f2py/capi_maps.py @@ -1,15 +1,10 @@ -#!/usr/bin/env python3 """ - -Copyright 1999,2000 Pearu Peterson all rights reserved, -Pearu Peterson +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/05/06 10:57:33 $ -Pearu Peterson - """ from . import __version__ f2py_version = __version__.version diff --git a/numpy/f2py/cb_rules.py b/numpy/f2py/cb_rules.py index 761831e00449..721e075b6c73 100644 --- a/numpy/f2py/cb_rules.py +++ b/numpy/f2py/cb_rules.py @@ -1,17 +1,12 @@ -#!/usr/bin/env python3 """ - Build call-back mechanism for f2py2e. -Copyright 2000 Pearu Peterson all rights reserved, -Pearu Peterson +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/07/20 11:27:58 $ -Pearu Peterson - """ from . import __version__ from .auxfuncs import ( diff --git a/numpy/f2py/cfuncs.py b/numpy/f2py/cfuncs.py index c074e9104008..4328a6e5004c 100644 --- a/numpy/f2py/cfuncs.py +++ b/numpy/f2py/cfuncs.py @@ -1,18 +1,14 @@ #!/usr/bin/env python3 """ - C declarations, CPP macros, and C functions for f2py2e. Only required declarations/macros/functions will be used. -Copyright 1999,2000 Pearu Peterson all rights reserved, -Pearu Peterson +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/05/06 11:42:34 $ -Pearu Peterson - """ import sys import copy diff --git a/numpy/f2py/common_rules.py b/numpy/f2py/common_rules.py index 5c86613c1011..64347b737454 100644 --- a/numpy/f2py/common_rules.py +++ b/numpy/f2py/common_rules.py @@ -1,17 +1,12 @@ -#!/usr/bin/env python3 """ - Build common block mechanism for f2py2e. -Copyright 2000 Pearu Peterson all rights reserved, -Pearu Peterson +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/05/06 10:57:33 $ -Pearu Peterson - """ from . import __version__ f2py_version = __version__.version diff --git a/numpy/f2py/crackfortran.py b/numpy/f2py/crackfortran.py index d17b052f9654..ac56d3f2ae6c 100755 --- a/numpy/f2py/crackfortran.py +++ b/numpy/f2py/crackfortran.py @@ -2,14 +2,12 @@ """ crackfortran --- read fortran (77,90) code and extract declaration information. -Copyright 1999-2004 Pearu Peterson all rights reserved, -Pearu Peterson +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/09/27 07:13:49 $ -Pearu Peterson Usage of crackfortran: diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index d9bb61ada7ac..ad34d575c273 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -4,15 +4,12 @@ f2py2e - Fortran to Python C/API generator. 2nd Edition. See __usage__ below. -Copyright 1999--2011 Pearu Peterson all rights reserved, -Pearu Peterson +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/05/06 08:31:19 $ -Pearu Peterson - """ import sys import os @@ -21,6 +18,7 @@ from pathlib import Path from itertools import dropwhile import argparse +import copy from . import crackfortran from . import rules @@ -190,10 +188,10 @@ Version: {f2py_version} numpy Version: {numpy_version} -Requires: Python 3.5 or higher. License: NumPy license (see LICENSE.txt in the NumPy source code) -Copyright 1999 - 2011 Pearu Peterson all rights reserved. -https://web.archive.org/web/20140822061353/http://cens.ioc.ee/projects/f2py2e""" +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. +https://numpy.org/doc/stable/f2py/index.html\n""" def scaninputline(inputline): diff --git a/numpy/f2py/f90mod_rules.py b/numpy/f2py/f90mod_rules.py index 9d4876122da6..2f8a8dc1878a 100644 --- a/numpy/f2py/f90mod_rules.py +++ b/numpy/f2py/f90mod_rules.py @@ -1,17 +1,12 @@ -#!/usr/bin/env python3 """ - Build F90 module support for f2py2e. -Copyright 2000 Pearu Peterson all rights reserved, -Pearu Peterson +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/02/03 19:30:23 $ -Pearu Peterson - """ __version__ = "$Revision: 1.27 $"[10:-1] diff --git a/numpy/f2py/func2subr.py b/numpy/f2py/func2subr.py index 2eedc0ade85e..b9aa9fc007cb 100644 --- a/numpy/f2py/func2subr.py +++ b/numpy/f2py/func2subr.py @@ -1,17 +1,13 @@ -#!/usr/bin/env python3 """ Rules for building C/API module with f2py2e. -Copyright 1999,2000 Pearu Peterson all rights reserved, -Pearu Peterson +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2004/11/26 11:13:06 $ -Pearu Peterson - """ import copy diff --git a/numpy/f2py/rules.py b/numpy/f2py/rules.py index 1bac871024d8..009365e04761 100755 --- a/numpy/f2py/rules.py +++ b/numpy/f2py/rules.py @@ -40,15 +40,12 @@ return buildvalue -Copyright 1999,2000 Pearu Peterson all rights reserved, -Pearu Peterson +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2005/08/30 08:58:42 $ -Pearu Peterson - """ import os, sys import time diff --git a/numpy/f2py/symbolic.py b/numpy/f2py/symbolic.py index b1b9f5b6a10a..67120d79a51e 100644 --- a/numpy/f2py/symbolic.py +++ b/numpy/f2py/symbolic.py @@ -2,6 +2,13 @@ References: - J3/21-007: Draft Fortran 202x. https://j3-fortran.org/doc/year/21/21-007.pdf + +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. +Permission to use, modify, and distribute this software is given under the +terms of the NumPy License. + +NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. """ # To analyze Fortran expressions to solve dimensions specifications, diff --git a/numpy/f2py/use_rules.py b/numpy/f2py/use_rules.py index f1b71e83c252..808b3dd97ec2 100644 --- a/numpy/f2py/use_rules.py +++ b/numpy/f2py/use_rules.py @@ -1,19 +1,12 @@ -#!/usr/bin/env python3 """ - Build 'use others module data' mechanism for f2py2e. -Unfinished. - -Copyright 2000 Pearu Peterson all rights reserved, -Pearu Peterson +Copyright 1999 -- 2011 Pearu Peterson all rights reserved. +Copyright 2011 -- present NumPy Developers. Permission to use, modify, and distribute this software is given under the terms of the NumPy License. NO WARRANTY IS EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. -$Date: 2000/09/10 12:35:43 $ -Pearu Peterson - """ __version__ = "$Revision: 1.3 $"[10:-1] From 6b270fb940d4d36dc8115dba43a926dc312ba70a Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Sun, 24 Dec 2023 13:54:13 -0700 Subject: [PATCH 93/94] MAINT: Update crackfortran.py and f2py2e.py from main --- numpy/f2py/crackfortran.py | 230 ++++++++++++++++++++++++++++++++----- numpy/f2py/f2py2e.py | 112 +++++++++--------- 2 files changed, 255 insertions(+), 87 deletions(-) diff --git a/numpy/f2py/crackfortran.py b/numpy/f2py/crackfortran.py index ac56d3f2ae6c..8d3fc27608bd 100755 --- a/numpy/f2py/crackfortran.py +++ b/numpy/f2py/crackfortran.py @@ -993,6 +993,16 @@ def _resolvenameargspattern(line): def analyzeline(m, case, line): + """ + Reads each line in the input file in sequence and updates global vars. + + Effectively reads and collects information from the input file to the + global variable groupcache, a dictionary containing info about each part + of the fortran module. + + At the end of analyzeline, information is filtered into the correct dict + keys, but parameter values and dimensions are not yet interpreted. + """ global groupcounter, groupname, groupcache, grouplist, filepositiontext global currentfilename, f77modulename, neededinterface, neededmodule global expectbegin, gotnextfile, previous_context @@ -1679,10 +1689,18 @@ def markinnerspaces(line): def updatevars(typespec, selector, attrspec, entitydecl): + """ + Returns last_name, the variable name without special chars, parenthesis + or dimension specifiers. + + Alters groupcache to add the name, typespec, attrspec (and possibly value) + of current variable. + """ global groupcache, groupcounter last_name = None kindselect, charselect, typename = cracktypespec(typespec, selector) + # Clean up outer commas, whitespace and undesired chars from attrspec if attrspec: attrspec = [x.strip() for x in markoutercomma(attrspec).split('@,@')] l = [] @@ -2396,8 +2414,6 @@ def _calc_depend_dict(vars): def get_sorted_names(vars): - """ - """ depend_dict = _calc_depend_dict(vars) names = [] for name in list(depend_dict.keys()): @@ -2450,7 +2466,7 @@ def _selected_real_kind_func(p, r=0, radix=0): if p < 16: return 8 machine = platform.machine().lower() - if machine.startswith(('aarch64', 'alpha', 'arm64', 'loongarch', 'power', 'ppc', 'riscv', 's390x', 'sparc')): + if machine.startswith(('aarch64', 'alpha', 'arm64', 'loongarch', 'mips', 'power', 'ppc', 'riscv', 's390x', 'sparc')): if p <= 33: return 16 else: @@ -2489,6 +2505,7 @@ def get_parameters(vars, global_params={}): # TODO: test .eq., .neq., etc replacements. ]: v = v.replace(*repl) + v = kind_re.sub(r'kind("\1")', v) v = selected_int_kind_re.sub(r'selected_int_kind(\1)', v) @@ -2497,14 +2514,17 @@ def get_parameters(vars, global_params={}): # then we may easily remove those specifiers. # However, it may be that the user uses other specifiers...(!) is_replaced = False + if 'kindselector' in vars[n]: + # Remove kind specifier (including those defined + # by parameters) if 'kind' in vars[n]['kindselector']: orig_v_len = len(v) v = v.replace('_' + vars[n]['kindselector']['kind'], '') # Again, this will be true if even a single specifier # has been replaced, see comment above. is_replaced = len(v) < orig_v_len - + if not is_replaced: if not selected_kind_re.match(v): v_ = v.split('_') @@ -2531,6 +2551,10 @@ def get_parameters(vars, global_params={}): outmess(f'get_parameters[TODO]: ' f'implement evaluation of complex expression {v}\n') + dimspec = ([s.lstrip('dimension').strip() + for s in vars[n]['attrspec'] + if s.startswith('dimension')] or [None])[0] + # Handle _dp for gh-6624 # Also fixes gh-20460 if real16pattern.search(v): @@ -2538,11 +2562,11 @@ def get_parameters(vars, global_params={}): elif real8pattern.search(v): v = 4 try: - params[n] = eval(v, g_params, params) - + params[n] = param_eval(v, g_params, params, dimspec=dimspec) except Exception as msg: params[n] = v - outmess('get_parameters: got "%s" on %s\n' % (msg, repr(v))) + outmess(f'get_parameters: got "{msg}" on {n!r}\n') + if isstring(vars[n]) and isinstance(params[n], int): params[n] = chr(params[n]) nl = n.lower() @@ -2550,8 +2574,7 @@ def get_parameters(vars, global_params={}): params[nl] = params[n] else: print(vars[n]) - outmess( - 'get_parameters:parameter %s does not have value?!\n' % (repr(n))) + outmess(f'get_parameters:parameter {n!r} does not have value?!\n') return params @@ -2560,6 +2583,7 @@ def _eval_length(length, params): return '(*)' return _eval_scalar(length, params) + _is_kind_number = re.compile(r'\d+_').match @@ -2580,6 +2604,10 @@ def _eval_scalar(value, params): def analyzevars(block): + """ + Sets correct dimension information for each variable/parameter + """ + global f90modulevars setmesstext(block) @@ -2608,7 +2636,8 @@ def analyzevars(block): svars.append(n) params = get_parameters(vars, get_useparameters(block)) - + # At this point, params are read and interpreted, but + # the params used to define vars are not yet parsed dep_matches = {} name_match = re.compile(r'[A-Za-z][\w$]*').match for v in list(vars.keys()): @@ -2707,27 +2736,30 @@ def analyzevars(block): check = None if dim and 'dimension' not in vars[n]: vars[n]['dimension'] = [] - for d in rmbadname([x.strip() for x in markoutercomma(dim).split('@,@')]): - star = ':' if d == ':' else '*' + for d in rmbadname( + [x.strip() for x in markoutercomma(dim).split('@,@')] + ): + # d is the expression inside the dimension declaration # Evaluate `d` with respect to params - if d in params: - d = str(params[d]) - for p in params: - re_1 = re.compile(r'(?P.*?)\b' + p + r'\b(?P.*)', re.I) - m = re_1.match(d) - while m: - d = m.group('before') + \ - str(params[p]) + m.group('after') - m = re_1.match(d) - - if d == star: - dl = [star] + try: + # the dimension for this variable depends on a + # previously defined parameter + d = param_parse(d, params) + except (ValueError, IndexError, KeyError): + outmess( + ('analyzevars: could not parse dimension for ' + f'variable {d!r}\n') + ) + + dim_char = ':' if d == ':' else '*' + if d == dim_char: + dl = [dim_char] else: dl = markoutercomma(d, ':').split('@:@') if len(dl) == 2 and '*' in dl: # e.g. dimension(5:*) dl = ['*'] d = '*' - if len(dl) == 1 and dl[0] != star: + if len(dl) == 1 and dl[0] != dim_char: dl = ['1', dl[0]] if len(dl) == 2: d1, d2 = map(symbolic.Expr.parse, dl) @@ -2961,9 +2993,152 @@ def compute_deps(v, deps): del vars[n] return vars + analyzeargs_re_1 = re.compile(r'\A[a-z]+[\w$]*\Z', re.I) +def param_eval(v, g_params, params, dimspec=None): + """ + Creates a dictionary of indices and values for each parameter in a + parameter array to be evaluated later. + + WARNING: It is not possible to initialize multidimensional array + parameters e.g. dimension(-3:1, 4, 3:5) at this point. This is because in + Fortran initialization through array constructor requires the RESHAPE + intrinsic function. Since the right-hand side of the parameter declaration + is not executed in f2py, but rather at the compiled c/fortran extension, + later, it is not possible to execute a reshape of a parameter array. + One issue remains: if the user wants to access the array parameter from + python, we should either + 1) allow them to access the parameter array using python standard indexing + (which is often incompatible with the original fortran indexing) + 2) allow the parameter array to be accessed in python as a dictionary with + fortran indices as keys + We are choosing 2 for now. + """ + if dimspec is None: + try: + p = eval(v, g_params, params) + except Exception as msg: + p = v + outmess(f'param_eval: got "{msg}" on {v!r}\n') + return p + + # This is an array parameter. + # First, we parse the dimension information + if len(dimspec) < 2 or dimspec[::len(dimspec)-1] != "()": + raise ValueError(f'param_eval: dimension {dimspec} can\'t be parsed') + dimrange = dimspec[1:-1].split(',') + if len(dimrange) == 1: + # e.g. dimension(2) or dimension(-1:1) + dimrange = dimrange[0].split(':') + # now, dimrange is a list of 1 or 2 elements + if len(dimrange) == 1: + bound = param_parse(dimrange[0], params) + dimrange = range(1, int(bound)+1) + else: + lbound = param_parse(dimrange[0], params) + ubound = param_parse(dimrange[1], params) + dimrange = range(int(lbound), int(ubound)+1) + else: + raise ValueError(f'param_eval: multidimensional array parameters ' + '{dimspec} not supported') + + # Parse parameter value + v = (v[2:-2] if v.startswith('(/') else v).split(',') + v_eval = [] + for item in v: + try: + item = eval(item, g_params, params) + except Exception as msg: + outmess(f'param_eval: got "{msg}" on {item!r}\n') + v_eval.append(item) + + p = dict(zip(dimrange, v_eval)) + + return p + + +def param_parse(d, params): + """Recursively parse array dimensions. + + Parses the declaration of an array variable or parameter + `dimension` keyword, and is called recursively if the + dimension for this array is a previously defined parameter + (found in `params`). + + Parameters + ---------- + d : str + Fortran expression describing the dimension of an array. + params : dict + Previously parsed parameters declared in the Fortran source file. + + Returns + ------- + out : str + Parsed dimension expression. + + Examples + -------- + + * If the line being analyzed is + + `integer, parameter, dimension(2) :: pa = (/ 3, 5 /)` + + then `d = 2` and we return immediately, with + + >>> d = '2' + >>> param_parse(d, params) + 2 + + * If the line being analyzed is + + `integer, parameter, dimension(pa) :: pb = (/1, 2, 3/)` + + then `d = 'pa'`; since `pa` is a previously parsed parameter, + and `pa = 3`, we call `param_parse` recursively, to obtain + + >>> d = 'pa' + >>> params = {'pa': 3} + >>> param_parse(d, params) + 3 + + * If the line being analyzed is + + `integer, parameter, dimension(pa(1)) :: pb = (/1, 2, 3/)` + + then `d = 'pa(1)'`; since `pa` is a previously parsed parameter, + and `pa(1) = 3`, we call `param_parse` recursively, to obtain + + >>> d = 'pa(1)' + >>> params = dict(pa={1: 3, 2: 5}) + >>> param_parse(d, params) + 3 + """ + if "(" in d: + # this dimension expression is an array + dname = d[:d.find("(")] + ddims = d[d.find("(")+1:d.rfind(")")] + # this dimension expression is also a parameter; + # parse it recursively + index = int(param_parse(ddims, params)) + return str(params[dname][index]) + elif d in params: + return str(params[d]) + else: + for p in params: + re_1 = re.compile( + r'(?P.*?)\b' + p + r'\b(?P.*)', re.I + ) + m = re_1.match(d) + while m: + d = m.group('before') + \ + str(params[p]) + m.group('after') + m = re_1.match(d) + return d + + def expr2name(a, block, args=[]): orig_a = a a_is_expr = not analyzeargs_re_1.match(a) @@ -3216,11 +3391,6 @@ def true_intent_list(var): def vars2fortran(block, vars, args, tab='', as_interface=False): - """ - TODO: - public sub - ... - """ setmesstext(block) ret = '' nout = [] diff --git a/numpy/f2py/f2py2e.py b/numpy/f2py/f2py2e.py index ad34d575c273..ce22b2d8a9ec 100755 --- a/numpy/f2py/f2py2e.py +++ b/numpy/f2py/f2py2e.py @@ -62,12 +62,6 @@ Options: - --2d-numpy Use numpy.f2py tool with NumPy support. [DEFAULT] - --2d-numeric Use f2py2e tool with Numeric support. - --2d-numarray Use f2py2e tool with Numarray support. - --g3-numpy Use 3rd generation f2py from the separate f2py package. - [NOT AVAILABLE YET] - -h Write signatures of the fortran routines to file and exit. You can then edit and use it instead of . If ==stdout then the @@ -128,20 +122,22 @@ -v Print f2py version ID and exit. -build backend options (only effective with -c): +build backend options (only effective with -c) +[NO_MESON] is used to indicate an option not meant to be used +with the meson backend or above Python 3.12: - --fcompiler= Specify Fortran compiler type by vendor - --compiler= Specify C compiler type (as defined by distutils) + --fcompiler= Specify Fortran compiler type by vendor [NO_MESON] + --compiler= Specify distutils C compiler type [NO_MESON] - --help-fcompiler List available Fortran compilers and exit - --f77exec= Specify the path to F77 compiler - --f90exec= Specify the path to F90 compiler + --help-fcompiler List available Fortran compilers and exit [NO_MESON] + --f77exec= Specify the path to F77 compiler [NO_MESON] + --f90exec= Specify the path to F90 compiler [NO_MESON] --f77flags= Specify F77 compiler flags --f90flags= Specify F90 compiler flags - --opt= Specify optimization flags - --arch= Specify architecture specific optimization flags - --noopt Compile without optimization - --noarch Compile without arch-dependent optimization + --opt= Specify optimization flags [NO_MESON] + --arch= Specify architecture specific optimization flags [NO_MESON] + --noopt Compile without optimization [NO_MESON] + --noarch Compile without arch-dependent optimization [NO_MESON] --debug Compile with debugging information --dep @@ -166,7 +162,7 @@ by numpy.distutils/system_info.py. E.g. to link with optimized LAPACK libraries (vecLib on MacOSX, ATLAS elsewhere), use --link-lapack_opt. - See also --help-link switch. + See also --help-link switch. [NO_MESON] -L/path/to/lib/ -l -D -U @@ -196,7 +192,7 @@ def scaninputline(inputline): files, skipfuncs, onlyfuncs, debug = [], [], [], [] - f, f2, f3, f5, f6, f7, f8, f9, f10 = 1, 0, 0, 0, 0, 0, 0, 0, 0 + f, f2, f3, f5, f6, f8, f9, f10 = 1, 0, 0, 0, 0, 0, 0, 0 verbose = 1 emptygen = True dolc = -1 @@ -204,7 +200,7 @@ def scaninputline(inputline): dorestdoc = 0 wrapfuncs = 1 buildpath = '.' - include_paths = [] + include_paths, inputline = get_includes(inputline) signsfile, modulename = None, None options = {'buildpath': buildpath, 'coutput': None, @@ -264,14 +260,6 @@ def scaninputline(inputline): elif l[:8] == '-include': cfuncs.outneeds['userincludes'].append(l[9:-1]) cfuncs.userincludes[l[9:-1]] = '#include ' + l[8:] - elif l[:15] in '--include_paths': - outmess( - 'f2py option --include_paths is deprecated, use --include-paths instead.\n') - f7 = 1 - elif l[:15] in '--include-paths': - # Similar to using -I with -c, however this is - # also used during generation of wrappers - f7 = 1 elif l == '--skip-empty-wrappers': emptygen = False elif l[0] == '-': @@ -286,9 +274,6 @@ def scaninputline(inputline): elif f6: f6 = 0 buildpath = l - elif f7: - f7 = 0 - include_paths.extend(l.split(os.pathsep)) elif f8: f8 = 0 options["coutput"] = l @@ -456,7 +441,7 @@ def run_main(comline_list): fobjhsrc = os.path.join(f2pydir, 'src', 'fortranobject.h') fobjcsrc = os.path.join(f2pydir, 'src', 'fortranobject.c') # gh-22819 -- begin - parser = make_f2py_parser() + parser = make_f2py_compile_parser() args, comline_list = parser.parse_known_args(comline_list) pyf_files, _ = filter_files("", "[.]pyf([.]src|)", comline_list) # Checks that no existing modulename is defined in a pyf file @@ -538,7 +523,35 @@ def get_prefix(module): p = os.path.dirname(os.path.dirname(module.__file__)) return p -def make_f2py_parser(): + +class CombineIncludePaths(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + include_paths_set = set(getattr(namespace, 'include_paths', []) or []) + if option_string == "--include_paths": + outmess("Use --include-paths or -I instead of --include_paths which will be removed") + if option_string == "--include-paths" or option_string == "--include_paths": + include_paths_set.update(values.split(':')) + else: + include_paths_set.add(values) + setattr(namespace, 'include_paths', list(include_paths_set)) + +def include_parser(): + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("-I", dest="include_paths", action=CombineIncludePaths) + parser.add_argument("--include-paths", dest="include_paths", action=CombineIncludePaths) + parser.add_argument("--include_paths", dest="include_paths", action=CombineIncludePaths) + return parser + +def get_includes(iline): + iline = (' '.join(iline)).split() + parser = include_parser() + args, remain = parser.parse_known_args(iline) + ipaths = args.include_paths + if args.include_paths is None: + ipaths = [] + return ipaths, remain + +def make_f2py_compile_parser(): parser = argparse.ArgumentParser(add_help=False) parser.add_argument("--dep", action="append", dest="dependencies") parser.add_argument("--backend", choices=['meson', 'distutils'], default='distutils') @@ -548,7 +561,7 @@ def make_f2py_parser(): def preparse_sysargv(): # To keep backwards bug compatibility, newer flags are handled by argparse, # and `sys.argv` is passed to the rest of `f2py` as is. - parser = make_f2py_parser() + parser = make_f2py_compile_parser() args, remaining_argv = parser.parse_known_args() sys.argv = [sys.argv[0]] + remaining_argv @@ -665,19 +678,19 @@ def run_compile(): if '--quiet' in f2py_flags: setup_flags.append('--quiet') + # Ugly filter to remove everything but sources sources = sys.argv[1:] - for optname in ['--include_paths', '--include-paths', '--f2cmap']: - if optname in sys.argv: - i = sys.argv.index(optname) - f2py_flags.extend(sys.argv[i:i + 2]) - del sys.argv[i + 1], sys.argv[i] - sources = sys.argv[1:] + f2cmapopt = '--f2cmap' + if f2cmapopt in sys.argv: + i = sys.argv.index(f2cmapopt) + f2py_flags.extend(sys.argv[i:i + 2]) + del sys.argv[i + 1], sys.argv[i] + sources = sys.argv[1:] pyf_files, _sources = filter_files("", "[.]pyf([.]src|)", sources) sources = pyf_files + _sources modulename = validate_modulename(pyf_files, modulename) extra_objects, sources = filter_files('', '[.](o|a|so|dylib)', sources) - include_dirs, sources = filter_files('-I', '', sources, remove_prefix=1) library_dirs, sources = filter_files('-L', '', sources, remove_prefix=1) libraries, sources = filter_files('-l', '', sources, remove_prefix=1) undef_macros, sources = filter_files('-U', '', sources, remove_prefix=1) @@ -700,6 +713,8 @@ def run_compile(): else: run_main(f" {' '.join(f2py_flags)} {' '.join(pyf_files)}".split()) + # Order matters here, includes are needed for run_main above + include_dirs, sources = get_includes(sources) # Now use the builder builder = build_backend( modulename, @@ -747,23 +762,6 @@ def main(): show_all() return - # Probably outdated options that were not working before 1.16 - if '--g3-numpy' in sys.argv[1:]: - sys.stderr.write("G3 f2py support is not implemented, yet.\\n") - sys.exit(1) - elif '--2e-numeric' in sys.argv[1:]: - sys.argv.remove('--2e-numeric') - elif '--2e-numarray' in sys.argv[1:]: - # Note that this errors becaust the -DNUMARRAY argument is - # not recognized. Just here for back compatibility and the - # error message. - sys.argv.append("-DNUMARRAY") - sys.argv.remove('--2e-numarray') - elif '--2e-numpy' in sys.argv[1:]: - sys.argv.remove('--2e-numpy') - else: - pass - if '-c' in sys.argv[1:]: run_compile() else: From c8d8c240b2bcec41c9b7c9e5ca25bf5945b1da07 Mon Sep 17 00:00:00 2001 From: Charles Harris Date: Mon, 1 Jan 2024 11:48:37 -0700 Subject: [PATCH 94/94] REL: Prepare for the NumPy 1.26.3 release - Create 1.26.3-changelog.rst - Update 1.26.3-notes.rst - Update .mailmap - Skip test that fails on internal Python 3.12 bug. [wheel build] --- .mailmap | 4 ++ doc/changelog/1.26.3-changelog.rst | 73 ++++++++++++++++++++++++ doc/source/release/1.26.3-notes.rst | 88 ++++++++++++++++++++++++++++- numpy/tests/test_warnings.py | 3 + 4 files changed, 165 insertions(+), 3 deletions(-) create mode 100644 doc/changelog/1.26.3-changelog.rst diff --git a/.mailmap b/.mailmap index 21f4af76ecc3..2d910fe98fea 100644 --- a/.mailmap +++ b/.mailmap @@ -85,6 +85,8 @@ Allan Haldane Al-Baraa El-Hag <48454648+a-elhag@users.noreply.github.com> Alok Singhal Alok Singhal Alyssa Quek +Andrea Bianchi +Andrea Bianchi andrea-bia Ankit Dwivedi Ankit Dwivedi Amir Sarabadani @@ -301,6 +303,8 @@ Jerome Kelleher Jessé Pires Jessi J Zhao <35235453+jessijzhao@users.noreply.github.com> João Fontes Gonçalves +Johann Rohwer +Johann Rohwer jmrohwer Johnathon Cusick Jhong-Ken Chen (陳仲肯) Jhong-Ken Chen (陳仲肯) <37182101+kennychenfs@users.noreply.github.com> diff --git a/doc/changelog/1.26.3-changelog.rst b/doc/changelog/1.26.3-changelog.rst new file mode 100644 index 000000000000..959c89f65bfd --- /dev/null +++ b/doc/changelog/1.26.3-changelog.rst @@ -0,0 +1,73 @@ + +Contributors +============ + +A total of 18 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* @DWesl +* @Illviljan +* Alexander Grund +* Andrea Bianchi + +* Charles Harris +* Daniel Vanzo +* Johann Rohwer + +* Matti Picus +* Nathan Goldbaum +* Peter Hawkins +* Raghuveer Devulapalli +* Ralf Gommers +* Rohit Goswami +* Sayed Adel +* Sebastian Berg +* Stefano Rivera + +* Thomas A Caswell +* matoro + +Pull requests merged +==================== + +A total of 42 pull requests were merged for this release. + +* `#25130 `__: MAINT: prepare 1.26.x for further development +* `#25188 `__: TYP: add None to ``__getitem__`` in ``numpy.array_api`` +* `#25189 `__: BLD,BUG: quadmath required where available [f2py] +* `#25190 `__: BUG: alpha doesn't use REAL(10) +* `#25191 `__: BUG: Fix FP overflow error in division when the divisor is scalar +* `#25192 `__: MAINT: Pin scipy-openblas version. +* `#25201 `__: BUG: Fix f2py to enable use of string optional inout argument +* `#25202 `__: BUG: Fix -fsanitize=alignment issue in numpy/_core/src/multiarray/arraytypes.c.src +* `#25203 `__: TST: Explicitly pass NumPy path to cython during tests (also... +* `#25204 `__: BUG: fix issues with ``newaxis`` and ``linalg.solve`` in ``numpy.array_api`` +* `#25205 `__: BUG: Disallow shadowed modulenames +* `#25217 `__: BUG: Handle common blocks with kind specifications from modules +* `#25218 `__: BUG: Fix moving compiled executable to root with f2py -c on Windows +* `#25219 `__: BUG: Fix single to half-precision conversion on PPC64/VSX3 +* `#25227 `__: TST: f2py: fix issue in test skip condition +* `#25240 `__: Revert "MAINT: Pin scipy-openblas version." +* `#25249 `__: MAINT: do not use ``long`` type +* `#25377 `__: TST: PyPy needs another gc.collect on latest versions +* `#25378 `__: CI: Install Lapack runtime on Cygwin. +* `#25379 `__: MAINT: Bump conda-incubator/setup-miniconda from 2.2.0 to 3.0.1 +* `#25380 `__: BLD: update vendored Meson for AIX shared library fix +* `#25419 `__: MAINT: Init ``base`` in cpu_avx512_kn +* `#25420 `__: BUG: Fix failing test_features on SapphireRapids +* `#25422 `__: BUG: Fix non-contiguous memory load when ARM/Neon is enabled +* `#25428 `__: MAINT,BUG: Never import distutils above 3.12 [f2py] +* `#25452 `__: MAINT: make the import-time check for old Accelerate more specific +* `#25458 `__: BUG: fix macOS version checks for Accelerate support +* `#25465 `__: MAINT: Bump actions/setup-node and larsoner/circleci-artifacts-redirector-action +* `#25466 `__: BUG: avoid seg fault from OOB access in RandomState.set_state() +* `#25467 `__: BUG: Fix two errors related to not checking for failed allocations +* `#25468 `__: BUG: Fix regression with ``f2py`` wrappers when modules and subroutines... +* `#25475 `__: BUG: Fix build issues on SPR +* `#25478 `__: BLD: fix uninitialized variable warnings from simd/neon/memory.h +* `#25480 `__: BUG: Handle ``iso_c_type`` mappings more consistently +* `#25481 `__: BUG: Fix module name bug in signature files [urgent] [f2py] +* `#25482 `__: BUG: Handle .pyf.src and fix SciPy [urgent] +* `#25483 `__: DOC: ``f2py`` rewrite with ``meson`` details +* `#25485 `__: BUG: Add external library handling for meson [f2py] +* `#25486 `__: MAINT: Run f2py's meson backend with the same python that ran... +* `#25489 `__: MAINT: Update ``numpy/f2py/_backends`` from main. +* `#25490 `__: MAINT: Easy updates of ``f2py/*.py`` from main. +* `#25491 `__: MAINT: Update crackfortran.py and f2py2e.py from main diff --git a/doc/source/release/1.26.3-notes.rst b/doc/source/release/1.26.3-notes.rst index 68fc303779e2..fd3ebc899c34 100644 --- a/doc/source/release/1.26.3-notes.rst +++ b/doc/source/release/1.26.3-notes.rst @@ -5,15 +5,97 @@ NumPy 1.26.3 Release Notes ========================== NumPy 1.26.3 is a maintenance release that fixes bugs and regressions -discovered after the 1.26.2 release. The 1.26.release series is the last -planned minor release series before NumPy 2.0. The Python versions supported by -this release are 3.9-3.12. +discovered after the 1.26.2 release. The most notable changes are the f2py bug +fixes. The Python versions supported by this release are 3.9-3.12. + + +Compatibility +============= + +``f2py`` will no longer accept ambiguous ``-m`` and ``.pyf`` CLI combinations. +When more than one ``.pyf`` file is passed, an error is raised. When both ``-m`` +and a ``.pyf`` is passed, a warning is emitted and the ``-m`` provided name is +ignored. + + +Improvements +============ + +``f2py`` now handles ``common`` blocks which have ``kind`` specifications from +modules. This further expands the usability of intrinsics like +``iso_fortran_env`` and ``iso_c_binding``. Contributors ============ +A total of 18 people contributed to this release. People with a "+" by their +names contributed a patch for the first time. + +* @DWesl +* @Illviljan +* Alexander Grund +* Andrea Bianchi + +* Charles Harris +* Daniel Vanzo +* Johann Rohwer + +* Matti Picus +* Nathan Goldbaum +* Peter Hawkins +* Raghuveer Devulapalli +* Ralf Gommers +* Rohit Goswami +* Sayed Adel +* Sebastian Berg +* Stefano Rivera + +* Thomas A Caswell +* matoro + Pull requests merged ==================== +A total of 42 pull requests were merged for this release. + +* `#25130 `__: MAINT: prepare 1.26.x for further development +* `#25188 `__: TYP: add None to ``__getitem__`` in ``numpy.array_api`` +* `#25189 `__: BLD,BUG: quadmath required where available [f2py] +* `#25190 `__: BUG: alpha doesn't use REAL(10) +* `#25191 `__: BUG: Fix FP overflow error in division when the divisor is scalar +* `#25192 `__: MAINT: Pin scipy-openblas version. +* `#25201 `__: BUG: Fix f2py to enable use of string optional inout argument +* `#25202 `__: BUG: Fix -fsanitize=alignment issue in numpy/_core/src/multiarray/arraytypes.c.src +* `#25203 `__: TST: Explicitly pass NumPy path to cython during tests (also... +* `#25204 `__: BUG: fix issues with ``newaxis`` and ``linalg.solve`` in ``numpy.array_api`` +* `#25205 `__: BUG: Disallow shadowed modulenames +* `#25217 `__: BUG: Handle common blocks with kind specifications from modules +* `#25218 `__: BUG: Fix moving compiled executable to root with f2py -c on Windows +* `#25219 `__: BUG: Fix single to half-precision conversion on PPC64/VSX3 +* `#25227 `__: TST: f2py: fix issue in test skip condition +* `#25240 `__: Revert "MAINT: Pin scipy-openblas version." +* `#25249 `__: MAINT: do not use ``long`` type +* `#25377 `__: TST: PyPy needs another gc.collect on latest versions +* `#25378 `__: CI: Install Lapack runtime on Cygwin. +* `#25379 `__: MAINT: Bump conda-incubator/setup-miniconda from 2.2.0 to 3.0.1 +* `#25380 `__: BLD: update vendored Meson for AIX shared library fix +* `#25419 `__: MAINT: Init ``base`` in cpu_avx512_kn +* `#25420 `__: BUG: Fix failing test_features on SapphireRapids +* `#25422 `__: BUG: Fix non-contiguous memory load when ARM/Neon is enabled +* `#25428 `__: MAINT,BUG: Never import distutils above 3.12 [f2py] +* `#25452 `__: MAINT: make the import-time check for old Accelerate more specific +* `#25458 `__: BUG: fix macOS version checks for Accelerate support +* `#25465 `__: MAINT: Bump actions/setup-node and larsoner/circleci-artifacts-redirector-action +* `#25466 `__: BUG: avoid seg fault from OOB access in RandomState.set_state() +* `#25467 `__: BUG: Fix two errors related to not checking for failed allocations +* `#25468 `__: BUG: Fix regression with ``f2py`` wrappers when modules and subroutines... +* `#25475 `__: BUG: Fix build issues on SPR +* `#25478 `__: BLD: fix uninitialized variable warnings from simd/neon/memory.h +* `#25480 `__: BUG: Handle ``iso_c_type`` mappings more consistently +* `#25481 `__: BUG: Fix module name bug in signature files [urgent] [f2py] +* `#25482 `__: BUG: Handle .pyf.src and fix SciPy [urgent] +* `#25483 `__: DOC: ``f2py`` rewrite with ``meson`` details +* `#25485 `__: BUG: Add external library handling for meson [f2py] +* `#25486 `__: MAINT: Run f2py's meson backend with the same python that ran... +* `#25489 `__: MAINT: Update ``numpy/f2py/_backends`` from main. +* `#25490 `__: MAINT: Easy updates of ``f2py/*.py`` from main. +* `#25491 `__: MAINT: Update crackfortran.py and f2py2e.py from main diff --git a/numpy/tests/test_warnings.py b/numpy/tests/test_warnings.py index d7a6d880cbdb..ee5124c5d511 100644 --- a/numpy/tests/test_warnings.py +++ b/numpy/tests/test_warnings.py @@ -5,6 +5,7 @@ import pytest from pathlib import Path +import sys import ast import tokenize import numpy @@ -56,6 +57,8 @@ def visit_Call(self, node): @pytest.mark.slow +@pytest.mark.skipif(sys.version_info >= (3, 12), + reason="Deprecation warning in ast") def test_warning_calls(): # combined "ignore" and stacklevel error base = Path(numpy.__file__).parent