From 25b4acdc0df373a4621d6e8eda9933ada6cba75f Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Tue, 25 Oct 2022 14:27:37 +0000 Subject: [PATCH 1/8] Sets cache support to false, removes upper req limit --- setup.py | 2 +- sqlalchemy_bigquery/base.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4afe003e..3d876748 100644 --- a/setup.py +++ b/setup.py @@ -91,7 +91,7 @@ def readme(): # https://github.com/googleapis/python-bigquery-sqlalchemy/issues/386 # and # https://github.com/googleapis/python-bigquery-sqlalchemy/issues/385 - "sqlalchemy>=1.2.0,<=1.4.27", + "sqlalchemy>=1.2.0", "future", ], extras_require=extras, diff --git a/sqlalchemy_bigquery/base.py b/sqlalchemy_bigquery/base.py index 3a26b330..32247a45 100644 --- a/sqlalchemy_bigquery/base.py +++ b/sqlalchemy_bigquery/base.py @@ -733,6 +733,7 @@ class BigQueryDialect(DefaultDialect): supports_default_values = False supports_empty_insert = False supports_multivalues_insert = True + supports_statement_cache = False supports_unicode_statements = True supports_unicode_binds = True supports_native_decimal = True From 36c2d0cffd8997e86df0176f5e43cba9aed04a46 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Fri, 17 Mar 2023 16:24:45 +0000 Subject: [PATCH 2/8] feat: remove dependency on bqstorage and pyarrow --- README.rst | 14 ++++++++++++ noxfile.py | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++- setup.py | 27 ++++++++++++++++------ 3 files changed, 99 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 7898cc67..0e0d30bc 100644 --- a/README.rst +++ b/README.rst @@ -81,6 +81,20 @@ Windows \Scripts\activate \Scripts\pip.exe install sqlalchemy-bigquery + +Installations when processing large datasets +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When handling large datasets, you may see speed increases by also installing the +`bqstorage` dependencies. See the instructions above about creating a virtual +environment and then install `sqlalchemy-bigquery` using the `bqstorage` extras: + +.. code-block:: console + + source /bin/activate + /bin/pip install sqlalchemy-bigquery[bqstorage] + + Usage ----- diff --git a/noxfile.py b/noxfile.py index 457800d6..7c2b15b5 100644 --- a/noxfile.py +++ b/noxfile.py @@ -50,10 +50,12 @@ "3.8": [ "tests", "alembic", + "bqstorage", ], "3.11": [ "tests", "geography", + "bqstorage", ], } @@ -73,10 +75,12 @@ "3.8": [ "tests", "alembic", + "bqstorage", ], "3.11": [ "tests", "geography", + "bqstorage", ], } @@ -179,7 +183,7 @@ def install_unittest_dependencies(session, *constraints): session.install("-e", ".", *constraints) -def default(session): +def default(session, install_extras=True): # Install all test dependencies, then install this package in-place. constraints_path = str( @@ -187,6 +191,14 @@ def default(session): ) install_unittest_dependencies(session, "-c", constraints_path) + if install_extras and session.python == "3.11": + install_target = ".[geography,alembic,tests,bqstorage]" + elif install_extras: + install_target = ".[all]" + else: + install_target = "." + session.install("-e", install_target, "-c", constraints_path) + # Run py.test against the unit tests. session.run( "py.test", @@ -209,6 +221,13 @@ def unit(session): default(session) +@nox.session(python=[UNIT_TEST_PYTHON_VERSIONS[0], UNIT_TEST_PYTHON_VERSIONS[-1]]) +def unit_noextras(session): + """Run the unit test suite.""" + + default(session, install_extras=False) + + def install_systemtest_dependencies(session, *constraints): # Use pre-release gRPC for system tests. @@ -283,6 +302,51 @@ def system(session): ) +@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +def system_noextras(session): + """Run the system test suite.""" + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + system_test_path = os.path.join("tests", "system.py") + system_test_folder_path = os.path.join("tests", "system") + + # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true. + if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false": + session.skip("RUN_SYSTEM_TESTS is set to false, skipping") + # Install pyopenssl for mTLS testing. + if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true": + session.install("pyopenssl") + + system_test_exists = os.path.exists(system_test_path) + system_test_folder_exists = os.path.exists(system_test_folder_path) + # Sanity check: only run tests if found. + if not system_test_exists and not system_test_folder_exists: + session.skip("System tests were not found") + + global SYSTEM_TEST_EXTRAS_BY_PYTHON + SYSTEM_TEST_EXTRAS_BY_PYTHON = False + install_systemtest_dependencies(session, "-c", constraints_path) + + # Run py.test against the system tests. + if system_test_exists: + session.run( + "py.test", + "--quiet", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_path, + *session.posargs, + ) + if system_test_folder_exists: + session.run( + "py.test", + "--quiet", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_folder_path, + *session.posargs, + ) + + @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS[-1]) def compliance(session): """Run the SQLAlchemy dialect-compliance system tests""" diff --git a/setup.py b/setup.py index bed5195c..026013aa 100644 --- a/setup.py +++ b/setup.py @@ -45,11 +45,26 @@ def readme(): return f.read() -extras = dict( - geography=["GeoAlchemy2", "shapely"], - alembic=["alembic"], - tests=["packaging", "pytz"], -) +extras = { + "geography": ["GeoAlchemy2", "shapely"], + "alembic": ["alembic"], + "tests": ["packaging", "pytz"], + # Keep the no-op bqstorage extra for backward compatibility. + # See: https://github.com/googleapis/python-bigquery/issues/757 + "bqstorage": [ + "google-cloud-bigquery-storage >= 2.0.0, <3.0.0dev", + # Due to an issue in pip's dependency resolver, the `grpc` extra is not + # installed, even though `google-cloud-bigquery-storage` specifies it + # as `google-api-core[grpc]`. We thus need to explicitly specify it here. + # See: https://github.com/googleapis/python-bigquery/issues/83 The + # grpc.Channel.close() method isn't added until 1.32.0. + # https://github.com/grpc/grpc/pull/15254 + "grpcio >= 1.47.0, < 2.0dev", + "grpcio >= 1.49.1, < 2.0dev; python_version>='3.11'", + "pyarrow >= 3.0.0", + ], +} + extras["all"] = set(itertools.chain.from_iterable(extras.values())) setup( @@ -85,9 +100,7 @@ def readme(): # https://github.com/googleapis/google-cloud-python/issues/10566 "google-auth>=1.25.0,<3.0.0dev", # Work around pip wack. "google-cloud-bigquery>=2.25.2,<4.0.0dev", - "google-cloud-bigquery-storage>=2.0.0,<3.0.0dev", "packaging", - "pyarrow>=3.0.0", "sqlalchemy>=1.2.0,<2.0.0dev", "future", ], From 28ad8af898b234884d3f7149fa82b07c504e3af0 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Mon, 20 Mar 2023 20:30:25 +0000 Subject: [PATCH 3/8] minor tweaks based on comments, remove unneeded constraints file --- noxfile.py | 7 ------- testing/constraints-3.6.txt | 12 ------------ testing/constraints-3.7.txt | 2 +- tests/unit/test_geography.py | 4 ++-- 4 files changed, 3 insertions(+), 22 deletions(-) delete mode 100644 testing/constraints-3.6.txt diff --git a/noxfile.py b/noxfile.py index 7c2b15b5..a6c38339 100644 --- a/noxfile.py +++ b/noxfile.py @@ -221,13 +221,6 @@ def unit(session): default(session) -@nox.session(python=[UNIT_TEST_PYTHON_VERSIONS[0], UNIT_TEST_PYTHON_VERSIONS[-1]]) -def unit_noextras(session): - """Run the unit test suite.""" - - default(session, install_extras=False) - - def install_systemtest_dependencies(session, *constraints): # Use pre-release gRPC for system tests. diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt deleted file mode 100644 index 9d2df4fe..00000000 --- a/testing/constraints-3.6.txt +++ /dev/null @@ -1,12 +0,0 @@ -# This constraints file is used to check that lower bounds -# are correct in setup.py -# List *all* library dependencies and extras in this file. -# Pin the version to the lower bound. -# -# e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", -sqlalchemy==1.2.0 -google-auth==1.25.0 -google-cloud-bigquery==2.25.2 -google-cloud-bigquery-storage==2.0.0 -google-api-core==1.31.5 -pyarrow==3.0.0 diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt index 9d2df4fe..1d0a1b72 100644 --- a/testing/constraints-3.7.txt +++ b/testing/constraints-3.7.txt @@ -6,7 +6,7 @@ # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", sqlalchemy==1.2.0 google-auth==1.25.0 -google-cloud-bigquery==2.25.2 +google-cloud-bigquery==3.3.6 google-cloud-bigquery-storage==2.0.0 google-api-core==1.31.5 pyarrow==3.0.0 diff --git a/tests/unit/test_geography.py b/tests/unit/test_geography.py index 25d3c605..fb856dca 100644 --- a/tests/unit/test_geography.py +++ b/tests/unit/test_geography.py @@ -104,7 +104,7 @@ def test_geoalchemy2_core(faux_conn, last_query): try: conn.execute( - select([lake_table.c.name, lake_table.c.geog.ST_AREA().label("area")]) + select([lake_table.c.name, lake_table.c.geog.ST_Area().label("area")]) ) except Exception: pass # sqlite had no special functions :) @@ -169,7 +169,7 @@ def test_calling_st_functions_that_dont_take_geographies(faux_conn, last_query): from sqlalchemy import select, func try: - faux_conn.execute(select([func.ST_GEOGFROMTEXT("point(0 0)")])) + faux_conn.execute(select([func.ST_GeogFromText("point(0 0)")])) except Exception: pass # sqlite had no special functions :) From 35a5a6483a707b38ab7557b1bcdec51b22bb1418 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Tue, 21 Mar 2023 11:32:12 +0000 Subject: [PATCH 4/8] updates coverage flags --- sqlalchemy_bigquery/__init__.py | 2 +- sqlalchemy_bigquery/_types.py | 4 ++-- sqlalchemy_bigquery/base.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sqlalchemy_bigquery/__init__.py b/sqlalchemy_bigquery/__init__.py index 2739dfa2..a9162e73 100644 --- a/sqlalchemy_bigquery/__init__.py +++ b/sqlalchemy_bigquery/__init__.py @@ -68,7 +68,7 @@ try: from .geography import GEOGRAPHY, WKB, WKT # noqa -except ImportError: +except ImportError: # pragma: no cover pass else: __all__.extend(["GEOGRAPHY", "WKB", "WKT"]) diff --git a/sqlalchemy_bigquery/_types.py b/sqlalchemy_bigquery/_types.py index 4e18dc2a..b79d7f90 100644 --- a/sqlalchemy_bigquery/_types.py +++ b/sqlalchemy_bigquery/_types.py @@ -24,7 +24,7 @@ try: from .geography import GEOGRAPHY -except ImportError: +except ImportError: # pragma: no cover pass from ._struct import STRUCT @@ -69,7 +69,7 @@ try: _type_map["GEOGRAPHY"] = GEOGRAPHY -except NameError: +except NameError: # pragma: no cover pass STRUCT_FIELD_TYPES = "RECORD", "STRUCT" diff --git a/sqlalchemy_bigquery/base.py b/sqlalchemy_bigquery/base.py index 843f6312..d436dc76 100644 --- a/sqlalchemy_bigquery/base.py +++ b/sqlalchemy_bigquery/base.py @@ -1055,7 +1055,7 @@ def __init__(self, *args, **kwargs): try: import alembic # noqa -except ImportError: +except ImportError: # pragma: no cover pass else: from alembic.ddl import impl From 2743d860e709848c027a5e5c7e710f0a3599f099 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Tue, 21 Mar 2023 13:19:05 +0000 Subject: [PATCH 5/8] experiment with no cover, cause issues --- sqlalchemy_bigquery/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlalchemy_bigquery/base.py b/sqlalchemy_bigquery/base.py index d436dc76..25837dc7 100644 --- a/sqlalchemy_bigquery/base.py +++ b/sqlalchemy_bigquery/base.py @@ -1053,7 +1053,7 @@ def __init__(self, *args, **kwargs): dialect = BigQueryDialect -try: +try: # pragma: no cover import alembic # noqa except ImportError: # pragma: no cover pass From 8377b3d4b8777297c6394edd0921e89a8a6e23f3 Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Tue, 21 Mar 2023 18:43:14 +0000 Subject: [PATCH 6/8] updates no cover pragma in all caps --- sqlalchemy_bigquery/__init__.py | 2 +- sqlalchemy_bigquery/_types.py | 4 ++-- sqlalchemy_bigquery/base.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sqlalchemy_bigquery/__init__.py b/sqlalchemy_bigquery/__init__.py index a9162e73..55253049 100644 --- a/sqlalchemy_bigquery/__init__.py +++ b/sqlalchemy_bigquery/__init__.py @@ -68,7 +68,7 @@ try: from .geography import GEOGRAPHY, WKB, WKT # noqa -except ImportError: # pragma: no cover +except ImportError: # pragma: NO COVER pass else: __all__.extend(["GEOGRAPHY", "WKB", "WKT"]) diff --git a/sqlalchemy_bigquery/_types.py b/sqlalchemy_bigquery/_types.py index b79d7f90..8399e978 100644 --- a/sqlalchemy_bigquery/_types.py +++ b/sqlalchemy_bigquery/_types.py @@ -24,7 +24,7 @@ try: from .geography import GEOGRAPHY -except ImportError: # pragma: no cover +except ImportError: # pragma: NO COVER pass from ._struct import STRUCT @@ -69,7 +69,7 @@ try: _type_map["GEOGRAPHY"] = GEOGRAPHY -except NameError: # pragma: no cover +except NameError: # pragma: NO COVER pass STRUCT_FIELD_TYPES = "RECORD", "STRUCT" diff --git a/sqlalchemy_bigquery/base.py b/sqlalchemy_bigquery/base.py index 25837dc7..b9874243 100644 --- a/sqlalchemy_bigquery/base.py +++ b/sqlalchemy_bigquery/base.py @@ -1053,9 +1053,9 @@ def __init__(self, *args, **kwargs): dialect = BigQueryDialect -try: # pragma: no cover +try: import alembic # noqa -except ImportError: # pragma: no cover +except ImportError: # pragma: NO COVER pass else: from alembic.ddl import impl From 7d50252c292bd701ce76fbaef28ce303c5c553fc Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Tue, 21 Mar 2023 20:20:58 +0000 Subject: [PATCH 7/8] fixes linting error --- sqlalchemy_bigquery/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlalchemy_bigquery/base.py b/sqlalchemy_bigquery/base.py index b9874243..4fae670e 100644 --- a/sqlalchemy_bigquery/base.py +++ b/sqlalchemy_bigquery/base.py @@ -1055,7 +1055,7 @@ def __init__(self, *args, **kwargs): try: import alembic # noqa -except ImportError: # pragma: NO COVER +except ImportError: # pragma: NO COVER pass else: from alembic.ddl import impl From e6377c72c5c8b715977f1d1c612b1c407ed961db Mon Sep 17 00:00:00 2001 From: Chalmer Lowe Date: Tue, 21 Mar 2023 21:22:35 +0000 Subject: [PATCH 8/8] updates owlbot --- owlbot.py | 86 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 2 deletions(-) diff --git a/owlbot.py b/owlbot.py index a2882583..4cc564d4 100644 --- a/owlbot.py +++ b/owlbot.py @@ -29,8 +29,8 @@ # ---------------------------------------------------------------------------- extras = ["tests"] extras_by_python = { - "3.8": ["tests", "alembic"], - "3.11": ["tests", "geography"], + "3.8": ["tests", "alembic", "bqstorage"], + "3.11": ["tests", "geography", "bqstorage"], } templated_files = common.py_library( unit_test_python_versions=["3.7", "3.8", "3.9", "3.10", "3.11"], @@ -95,6 +95,14 @@ ) +s.replace( + ["noxfile.py"], + r"def default\(session\)", + "def default(session, install_extras=True)", +) + + + def place_before(path, text, *before_text, escape=None): replacement = "\n".join(before_text) + "\n" + text @@ -117,6 +125,23 @@ def place_before(path, text, *before_text, escape=None): ) +install_logic = ''' + if install_extras and session.python == "3.11": + install_target = ".[geography,alembic,tests,bqstorage]" + elif install_extras: + install_target = ".[all]" + else: + install_target = "." + session.install("-e", install_target, "-c", constraints_path) +''' + +place_before( + "noxfile.py", + "# Run py.test against the unit tests.", + install_logic, +) + + # Maybe we can get rid of this when we don't need pytest-rerunfailures, # which we won't need when BQ retries itself: # https://github.com/googleapis/python-bigquery/pull/837 @@ -189,6 +214,63 @@ def compliance(session): s.replace(["noxfile.py"], '"alabaster"', '"alabaster", "geoalchemy2", "shapely"') +system_noextras = ''' +@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) +def system_noextras(session): + """Run the system test suite.""" + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + system_test_path = os.path.join("tests", "system.py") + system_test_folder_path = os.path.join("tests", "system") + + # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true. + if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false": + session.skip("RUN_SYSTEM_TESTS is set to false, skipping") + # Install pyopenssl for mTLS testing. + if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true": + session.install("pyopenssl") + + system_test_exists = os.path.exists(system_test_path) + system_test_folder_exists = os.path.exists(system_test_folder_path) + # Sanity check: only run tests if found. + if not system_test_exists and not system_test_folder_exists: + session.skip("System tests were not found") + + global SYSTEM_TEST_EXTRAS_BY_PYTHON + SYSTEM_TEST_EXTRAS_BY_PYTHON = False + install_systemtest_dependencies(session, "-c", constraints_path) + + # Run py.test against the system tests. + if system_test_exists: + session.run( + "py.test", + "--quiet", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_path, + *session.posargs, + ) + if system_test_folder_exists: + session.run( + "py.test", + "--quiet", + f"--junitxml=system_{session.python}_sponge_log.xml", + system_test_folder_path, + *session.posargs, + ) + + +''' + + +place_before( + "noxfile.py", + "@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS[-1])\n" + "def compliance(session):", + system_noextras, + escape="()[]", + ) + # Add DB config for SQLAlchemy dialect test suite. # https://github.com/googleapis/python-bigquery-sqlalchemy/issues/89