From 6ffcef605b4ac7805cafa7c79285c7127cc4a49a Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 26 Aug 2021 12:56:44 -0600 Subject: [PATCH 01/16] chore: Update multiple dependencies (#306) * chore(deps): update dependency google-cloud-bigquery to v2.25.0 * chore(deps): update dependency geoalchemy2 to v0.9.4 * chore(deps): update dependency pluggy to v1 * chore(deps): update dependency importlib-metadata to v4.7.0 * chore(deps): update dependency sqlalchemy-bigquery to v1.1.0 * chore(deps): update dependency opentelemetry-api to v1.5.0 * chore(deps): update dependency opentelemetry-sdk to v1.5.0 * update other open-telementry packages * Revert pluggy change, pytest doesn't allow it yet. * Look, a newer version! * Look, a newer version! Co-authored-by: Renovate Bot Co-authored-by: Anthonios Partheniou --- samples/snippets/requirements-test.txt | 2 +- samples/snippets/requirements.txt | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index d51249fe..de5fce55 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1,6 +1,6 @@ attrs==21.2.0 google-cloud-testutils==1.0.0 -importlib-metadata==4.6.4 +importlib-metadata==4.7.1 iniconfig==1.1.1 packaging==21.0 pluggy==0.13.1 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 50a070fb..65f50ec6 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -12,11 +12,11 @@ dataclasses==0.6; python_version < '3.7' Deprecated==1.2.12 Fiona==1.8.20 future==0.18.2 -GeoAlchemy2==0.9.3 +GeoAlchemy2==0.9.4 geopandas==0.9.0 google-api-core==2.0.0 google-auth==2.0.1 -google-cloud-bigquery==2.24.1 +google-cloud-bigquery==2.25.0 google-cloud-bigquery-storage==2.6.3 google-cloud-core==2.0.0 google-crc32c==1.1.2 @@ -26,16 +26,16 @@ greenlet==1.1.1 grpcio==1.39.0 idna==3.2 immutables==0.16 -importlib-metadata==4.6.4 +importlib-metadata==4.7.1 libcst==0.3.20 munch==2.5.0 mypy-extensions==0.4.3 numpy==1.19.5; python_version < '3.7' numpy==1.21.2; python_version >= '3.7' -opentelemetry-api==1.4.1 -opentelemetry-instrumentation==0.23b2 -opentelemetry-sdk==1.4.1 -opentelemetry-semantic-conventions==0.23b2 +opentelemetry-api==1.5.0 +opentelemetry-instrumentation==0.24b0 +opentelemetry-sdk==1.5.0 +opentelemetry-semantic-conventions==0.24b0 packaging==21.0 pandas==1.1.5; python_version < '3.7' pandas==1.3.2; python_version >= '3.7' @@ -56,7 +56,7 @@ rsa==4.7.2 Shapely==1.7.1 six==1.16.0 SQLAlchemy==1.4.23 -sqlalchemy-bigquery==1.0.0 +sqlalchemy-bigquery==1.1.0 tqdm==4.62.2 typing-extensions==3.10.0.0 typing-inspect==0.7.1 From 123318269876e7f76c7f0f2daa5f5b365026cd3f Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 26 Aug 2021 16:22:52 -0600 Subject: [PATCH 02/16] fix: the unnest function lost needed type information (#298) --- sqlalchemy_bigquery/base.py | 17 +++++++++ tests/unit/test_sqlalchemy_bigquery.py | 51 ++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/sqlalchemy_bigquery/base.py b/sqlalchemy_bigquery/base.py index e4f86e7b..e03d074e 100644 --- a/sqlalchemy_bigquery/base.py +++ b/sqlalchemy_bigquery/base.py @@ -35,6 +35,8 @@ from google.api_core.exceptions import NotFound import sqlalchemy +import sqlalchemy.sql.expression +import sqlalchemy.sql.functions import sqlalchemy.sql.sqltypes import sqlalchemy.sql.type_api from sqlalchemy.exc import NoSuchTableError @@ -1092,6 +1094,21 @@ def get_view_definition(self, connection, view_name, schema=None, **kw): return view.view_query +class unnest(sqlalchemy.sql.functions.GenericFunction): + def __init__(self, *args, **kwargs): + expr = kwargs.pop("expr", None) + if expr is not None: + args = (expr,) + args + if len(args) != 1: + raise TypeError("The unnest function requires a single argument.") + arg = args[0] + if isinstance(arg, sqlalchemy.sql.expression.ColumnElement): + if not isinstance(arg.type, sqlalchemy.sql.sqltypes.ARRAY): + raise TypeError("The argument to unnest must have an ARRAY type.") + self.type = arg.type.item_type + super().__init__(*args, **kwargs) + + dialect = BigQueryDialect try: diff --git a/tests/unit/test_sqlalchemy_bigquery.py b/tests/unit/test_sqlalchemy_bigquery.py index a4c81367..75cbec42 100644 --- a/tests/unit/test_sqlalchemy_bigquery.py +++ b/tests/unit/test_sqlalchemy_bigquery.py @@ -10,6 +10,7 @@ from google.cloud import bigquery from google.cloud.bigquery.dataset import DatasetListItem from google.cloud.bigquery.table import TableListItem +import packaging.version import pytest import sqlalchemy @@ -178,3 +179,53 @@ def test_follow_dialect_attribute_convention(): assert sqlalchemy_bigquery.dialect is sqlalchemy_bigquery.BigQueryDialect assert sqlalchemy_bigquery.base.dialect is sqlalchemy_bigquery.BigQueryDialect + + +@pytest.mark.parametrize( + "args,kw,error", + [ + ((), {}, "The unnest function requires a single argument."), + ((1, 1), {}, "The unnest function requires a single argument."), + ((1,), {"expr": 1}, "The unnest function requires a single argument."), + ((1, 1), {"expr": 1}, "The unnest function requires a single argument."), + ( + (), + {"expr": sqlalchemy.Column("x", sqlalchemy.String)}, + "The argument to unnest must have an ARRAY type.", + ), + ( + (sqlalchemy.Column("x", sqlalchemy.String),), + {}, + "The argument to unnest must have an ARRAY type.", + ), + ], +) +def test_unnest_function_errors(args, kw, error): + # Make sure the unnest function is registered with SQLAlchemy, which + # happens when sqlalchemy_bigquery is imported. + import sqlalchemy_bigquery # noqa + + with pytest.raises(TypeError, match=error): + sqlalchemy.func.unnest(*args, **kw) + + +@pytest.mark.parametrize( + "args,kw", + [ + ((), {"expr": sqlalchemy.Column("x", sqlalchemy.ARRAY(sqlalchemy.String))}), + ((sqlalchemy.Column("x", sqlalchemy.ARRAY(sqlalchemy.String)),), {}), + ], +) +def test_unnest_function(args, kw): + # Make sure the unnest function is registered with SQLAlchemy, which + # happens when sqlalchemy_bigquery is imported. + import sqlalchemy_bigquery # noqa + + f = sqlalchemy.func.unnest(*args, **kw) + assert isinstance(f.type, sqlalchemy.String) + if packaging.version.parse(sqlalchemy.__version__) >= packaging.version.parse( + "1.4" + ): + assert isinstance( + sqlalchemy.select([f]).subquery().c.unnest.type, sqlalchemy.String + ) From 06c3bcc2014b5cd974a4744d53a3f6b33d67406a Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 27 Aug 2021 15:14:27 +0200 Subject: [PATCH 03/16] chore(deps): update dependency google-cloud-bigquery to v2.25.1 (#307) Co-authored-by: Jim Fulton --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 65f50ec6..3183c5d6 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -16,7 +16,7 @@ GeoAlchemy2==0.9.4 geopandas==0.9.0 google-api-core==2.0.0 google-auth==2.0.1 -google-cloud-bigquery==2.25.0 +google-cloud-bigquery==2.25.1 google-cloud-bigquery-storage==2.6.3 google-cloud-core==2.0.0 google-crc32c==1.1.2 From 0fad7b036910d516bbed8ad14368db8bc0243923 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Fri, 27 Aug 2021 09:55:40 -0600 Subject: [PATCH 04/16] chore: migrate default branch from master to main (#301) --- .kokoro/build.sh | 2 +- .kokoro/test-samples-impl.sh | 2 +- CONTRIBUTING.rst | 12 +++++----- docs/conf.py | 16 ++++--------- owlbot.py | 46 +++++++++++++++++++++++++++++++++++- release-procedure.md | 10 ++++---- 6 files changed, 63 insertions(+), 25 deletions(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index b625604b..2a2874e5 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -41,7 +41,7 @@ python3 -m pip install --upgrade --quiet nox python3 -m nox --version # If this is a continuous build, send the test log to the FlakyBot. -# See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot. +# See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot. if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]]; then cleanup() { chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh index 311a8d54..8a324c9c 100755 --- a/.kokoro/test-samples-impl.sh +++ b/.kokoro/test-samples-impl.sh @@ -80,7 +80,7 @@ for file in samples/**/requirements.txt; do EXIT=$? # If this is a periodic build, send the test log to the FlakyBot. - # See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot. + # See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot. if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot $KOKORO_GFILE_DIR/linux_amd64/flakybot diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index b6278e10..2083c729 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -50,9 +50,9 @@ You'll have to create a development environment using a Git checkout: # Configure remotes such that you can pull changes from the googleapis/python-bigquery-sqlalchemy # repository into your local repository. $ git remote add upstream git@github.com:googleapis/python-bigquery-sqlalchemy.git - # fetch and merge changes from upstream into master + # fetch and merge changes from upstream into main $ git fetch upstream - $ git merge upstream/master + $ git merge upstream/main Now your local repo is set up such that you will push changes to your GitHub repo, from which you can submit a pull request. @@ -110,12 +110,12 @@ Coding Style variables:: export GOOGLE_CLOUD_TESTING_REMOTE="upstream" - export GOOGLE_CLOUD_TESTING_BRANCH="master" + export GOOGLE_CLOUD_TESTING_BRANCH="main" By doing this, you are specifying the location of the most up-to-date version of ``python-bigquery-sqlalchemy``. The the suggested remote name ``upstream`` should point to the official ``googleapis`` checkout and the - the branch should be the main branch on that remote (``master``). + the branch should be the main branch on that remote (``main``). - This repository contains configuration for the `pre-commit `__ tool, which automates checking @@ -209,7 +209,7 @@ The `description on PyPI`_ for the project comes directly from the ``README``. Due to the reStructuredText (``rst``) parser used by PyPI, relative links which will work on GitHub (e.g. ``CONTRIBUTING.rst`` instead of -``https://github.com/googleapis/python-bigquery-sqlalchemy/blob/master/CONTRIBUTING.rst``) +``https://github.com/googleapis/python-bigquery-sqlalchemy/blob/main/CONTRIBUTING.rst``) may cause problems creating links or rendering the description. .. _description on PyPI: https://pypi.org/project/sqlalchemy-bigquery @@ -234,7 +234,7 @@ We support: Supported versions can be found in our ``noxfile.py`` `config`_. -.. _config: https://github.com/googleapis/python-bigquery-sqlalchemy/blob/master/noxfile.py +.. _config: https://github.com/googleapis/python-bigquery-sqlalchemy/blob/main/noxfile.py We also explicitly decided to support Python 3 beginning with version 3.6. diff --git a/docs/conf.py b/docs/conf.py index 6e45e97c..620d2a06 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -76,8 +76,8 @@ # The encoding of source files. # source_encoding = 'utf-8-sig' -# The master toctree document. -master_doc = "index" +# The root toctree document. +root_doc = "index" # General information about the project. project = "sqlalchemy-bigquery" @@ -280,7 +280,7 @@ # author, documentclass [howto, manual, or own class]). latex_documents = [ ( - master_doc, + root_doc, "sqlalchemy-bigquery.tex", "sqlalchemy-bigquery Documentation", author, @@ -314,13 +314,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ( - master_doc, - "sqlalchemy-bigquery", - "sqlalchemy-bigquery Documentation", - [author], - 1, - ) + (root_doc, "sqlalchemy-bigquery", "sqlalchemy-bigquery Documentation", [author], 1,) ] # If true, show URL addresses after external links. @@ -334,7 +328,7 @@ # dir menu entry, description, category) texinfo_documents = [ ( - master_doc, + root_doc, "sqlalchemy-bigquery", "sqlalchemy-bigquery Documentation", author, diff --git a/owlbot.py b/owlbot.py index b6cfe7ff..f8633351 100644 --- a/owlbot.py +++ b/owlbot.py @@ -172,7 +172,6 @@ def compliance(session): # Add DB config for SQLAlchemy dialect test suite. -# https://github.com/sqlalchemy/sqlalchemy/blob/master/README.dialects.rst # https://github.com/googleapis/python-bigquery-sqlalchemy/issues/89 s.replace( ["setup.cfg"], @@ -195,6 +194,51 @@ def compliance(session): python.py_samples(skip_readmes=True) +# ---------------------------------------------------------------------------- +# Remove the replacements below once https://github.com/googleapis/synthtool/pull/1188 is merged + +# Update googleapis/repo-automation-bots repo to main in .kokoro/*.sh files +s.replace(".kokoro/*.sh", "repo-automation-bots/tree/master", "repo-automation-bots/tree/main") + +# Customize CONTRIBUTING.rst to replace master with main +s.replace( + "CONTRIBUTING.rst", + "fetch and merge changes from upstream into master", + "fetch and merge changes from upstream into main", +) + +s.replace( + "CONTRIBUTING.rst", + "git merge upstream/master", + "git merge upstream/main", +) + +s.replace( + "CONTRIBUTING.rst", + """export GOOGLE_CLOUD_TESTING_BRANCH=\"master\"""", + """export GOOGLE_CLOUD_TESTING_BRANCH=\"main\"""", +) + +s.replace( + "CONTRIBUTING.rst", + "remote \(``master``\)", + "remote (``main``)", +) + +s.replace( + "CONTRIBUTING.rst", + "blob/master/CONTRIBUTING.rst", + "blob/main/CONTRIBUTING.rst", +) + +s.replace( + "CONTRIBUTING.rst", + "blob/master/noxfile.py", + "blob/main/noxfile.py", +) + +s.replace("docs/conf.py", "master", "root") + # ---------------------------------------------------------------------------- # Final cleanup # ---------------------------------------------------------------------------- diff --git a/release-procedure.md b/release-procedure.md index ba5291dd..c3d68f31 100644 --- a/release-procedure.md +++ b/release-procedure.md @@ -1,10 +1,10 @@ # sqlalchemy-bigquery release procedure -* Checkout master branch +* Checkout main branch - git fetch upstream master - git checkout master - git rebase -i upstream/master + git fetch upstream main + git checkout main + git rebase -i upstream/main * Update version number in `setup.py` @@ -13,7 +13,7 @@ * Commit and push git commit -m "Release x.x.x" - git push upstream master + git push upstream main * Build the package From 0443d7267a9330cba095193b4f9635574c8f7b05 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 30 Aug 2021 16:54:26 +0000 Subject: [PATCH 05/16] chore(python): disable dependency dashboard (#313) --- .github/.OwlBot.lock.yaml | 2 +- renovate.json | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index a9fcd07c..b75186cf 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:9743664022bd63a8084be67f144898314c7ca12f0a03e422ac17c733c129d803 + digest: sha256:d6761eec279244e57fe9d21f8343381a01d3632c034811a72f68b83119e58c69 diff --git a/renovate.json b/renovate.json index c0489556..9fa8816f 100644 --- a/renovate.json +++ b/renovate.json @@ -1,6 +1,8 @@ { "extends": [ - "config:base", ":preserveSemverRanges" + "config:base", + ":preserveSemverRanges", + ":disableDependencyDashboard" ], "ignorePaths": [".pre-commit-config.yaml"], "pip_requirements": { From 0f6e2bfd901592cd3670dc01de22856f552fa828 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Tue, 31 Aug 2021 09:42:38 -0600 Subject: [PATCH 06/16] chore: Update some dependencies (#319) --- dev_requirements.txt | 2 +- samples/snippets/requirements-test.txt | 10 +++++----- samples/snippets/requirements.txt | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index 036eedd7..a092f5b0 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -2,6 +2,6 @@ sqlalchemy>=1.1.9 google-cloud-bigquery>=1.6.0 future==0.18.2 -pytest==6.2.4 +pytest==6.2.5 pytest-flake8==1.0.7 pytz==2021.1 \ No newline at end of file diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index de5fce55..1d27f79b 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1,12 +1,12 @@ attrs==21.2.0 -google-cloud-testutils==1.0.0 -importlib-metadata==4.7.1 +google-cloud-testutils==1.1.0 +importlib-metadata==4.8.1 iniconfig==1.1.1 packaging==21.0 -pluggy==0.13.1 +pluggy==1.0.0 py==1.10.0 pyparsing==2.4.7 -pytest==6.2.4 +pytest==6.2.5 toml==0.10.2 -typing-extensions==3.10.0.0 +typing-extensions==3.10.0.2 zipp==3.5.0 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 3183c5d6..ef7a6f26 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -19,14 +19,14 @@ google-auth==2.0.1 google-cloud-bigquery==2.25.1 google-cloud-bigquery-storage==2.6.3 google-cloud-core==2.0.0 -google-crc32c==1.1.2 +google-crc32c==1.1.3 google-resumable-media==2.0.0 googleapis-common-protos==1.53.0 greenlet==1.1.1 grpcio==1.39.0 idna==3.2 immutables==0.16 -importlib-metadata==4.7.1 +importlib-metadata==4.8.1 libcst==0.3.20 munch==2.5.0 mypy-extensions==0.4.3 @@ -58,7 +58,7 @@ six==1.16.0 SQLAlchemy==1.4.23 sqlalchemy-bigquery==1.1.0 tqdm==4.62.2 -typing-extensions==3.10.0.0 +typing-extensions==3.10.0.2 typing-inspect==0.7.1 urllib3==1.26.6 wrapt==1.12.1 From a976491b5048f2d02bdb6ff2b6d8fe8a056a5464 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 1 Sep 2021 17:53:12 +0200 Subject: [PATCH 07/16] chore(deps): update dependency google-auth to v2.0.2 (#320) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index ef7a6f26..91857bb9 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -15,7 +15,7 @@ future==0.18.2 GeoAlchemy2==0.9.4 geopandas==0.9.0 google-api-core==2.0.0 -google-auth==2.0.1 +google-auth==2.0.2 google-cloud-bigquery==2.25.1 google-cloud-bigquery-storage==2.6.3 google-cloud-core==2.0.0 From b54467246f944664bdbfc2b773c46ed9948f4e04 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 1 Sep 2021 22:49:39 +0200 Subject: [PATCH 08/16] chore(deps): update dependency google-cloud-bigquery to v2.25.2 (#323) --- samples/snippets/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 91857bb9..e38cfa20 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -16,7 +16,7 @@ GeoAlchemy2==0.9.4 geopandas==0.9.0 google-api-core==2.0.0 google-auth==2.0.2 -google-cloud-bigquery==2.25.1 +google-cloud-bigquery==2.25.2 google-cloud-bigquery-storage==2.6.3 google-cloud-core==2.0.0 google-crc32c==1.1.3 From e4842148a3250e5fdcff7527aa0a0c6dcb51335a Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 1 Sep 2021 22:34:13 +0000 Subject: [PATCH 09/16] chore(python): group renovate prs (#324) --- .github/.OwlBot.lock.yaml | 2 +- renovate.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index b75186cf..ef3cb34f 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:d6761eec279244e57fe9d21f8343381a01d3632c034811a72f68b83119e58c69 + digest: sha256:1456ea2b3b523ccff5e13030acef56d1de28f21249c62aa0f196265880338fa7 diff --git a/renovate.json b/renovate.json index 9fa8816f..c21036d3 100644 --- a/renovate.json +++ b/renovate.json @@ -1,6 +1,7 @@ { "extends": [ "config:base", + "group:all", ":preserveSemverRanges", ":disableDependencyDashboard" ], From 06c88cb466719c044822cbcb1afb3c751e83f914 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 2 Sep 2021 02:08:29 +0200 Subject: [PATCH 10/16] chore(deps): update all dependencies (#328) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![WhiteSource Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [google-api-core](https://togithub.com/googleapis/python-api-core) | `==2.0.0` -> `==2.0.1` | [![age](https://badges.renovateapi.com/packages/pypi/google-api-core/2.0.1/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/pypi/google-api-core/2.0.1/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/pypi/google-api-core/2.0.1/compatibility-slim/2.0.0)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/pypi/google-api-core/2.0.1/confidence-slim/2.0.0)](https://docs.renovatebot.com/merge-confidence/) | | [google-cloud-bigquery](https://togithub.com/googleapis/python-bigquery) | `==2.25.2` -> `==2.26.0` | [![age](https://badges.renovateapi.com/packages/pypi/google-cloud-bigquery/2.26.0/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/pypi/google-cloud-bigquery/2.26.0/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/pypi/google-cloud-bigquery/2.26.0/compatibility-slim/2.25.2)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/pypi/google-cloud-bigquery/2.26.0/confidence-slim/2.25.2)](https://docs.renovatebot.com/merge-confidence/) | | [google-resumable-media](https://togithub.com/googleapis/google-resumable-media-python) | `==2.0.0` -> `==2.0.1` | [![age](https://badges.renovateapi.com/packages/pypi/google-resumable-media/2.0.1/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/pypi/google-resumable-media/2.0.1/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/pypi/google-resumable-media/2.0.1/compatibility-slim/2.0.0)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/pypi/google-resumable-media/2.0.1/confidence-slim/2.0.0)](https://docs.renovatebot.com/merge-confidence/) | --- ### Release Notes
googleapis/python-api-core ### [`v2.0.1`](https://togithub.com/googleapis/python-api-core/blob/master/CHANGELOG.md#​201-httpswwwgithubcomgoogleapispython-api-corecomparev200v201-2021-08-31) [Compare Source](https://togithub.com/googleapis/python-api-core/compare/v2.0.0...v2.0.1)
googleapis/python-bigquery ### [`v2.26.0`](https://togithub.com/googleapis/python-bigquery/blob/master/CHANGELOG.md#​2260-httpswwwgithubcomgoogleapispython-bigquerycomparev2252v2260-2021-09-01) [Compare Source](https://togithub.com/googleapis/python-bigquery/compare/v2.25.2...v2.26.0) ##### Features - set the X-Server-Timeout header when timeout is set ([#​927](https://www.togithub.com/googleapis/python-bigquery/issues/927)) ([ba02f24](https://www.github.com/googleapis/python-bigquery/commit/ba02f248ba9c449c34859579a4011f4bfd2f4a93)) ##### Bug Fixes - guard imports against unsupported pyarrow versions ([#​934](https://www.togithub.com/googleapis/python-bigquery/issues/934)) ([b289076](https://www.github.com/googleapis/python-bigquery/commit/b28907693bbe889becc1b9c8963f0a7e1ee6c35a)) ##### [2.25.2](https://www.github.com/googleapis/python-bigquery/compare/v2.25.1...v2.25.2) (2021-08-31) ##### Bug Fixes - error inserting DataFrame with REPEATED field ([#​925](https://www.togithub.com/googleapis/python-bigquery/issues/925)) ([656d2fa](https://www.github.com/googleapis/python-bigquery/commit/656d2fa6f870573a21235c83463752a2d084caba)) - underscores weren't allowed in struct field names when passing parameters to the DB API ([#​930](https://www.togithub.com/googleapis/python-bigquery/issues/930)) ([fcb0bc6](https://www.github.com/googleapis/python-bigquery/commit/fcb0bc68c972c2c98bb8542f54e9228308177ecb)) ##### Documentation - update docstring for bigquery_create_routine sample ([#​883](https://www.togithub.com/googleapis/python-bigquery/issues/883)) ([#​917](https://www.togithub.com/googleapis/python-bigquery/issues/917)) ([e2d12b7](https://www.github.com/googleapis/python-bigquery/commit/e2d12b795ef2dc51b0ee36f1b3000edb1e64ce05)) ##### [2.25.1](https://www.github.com/googleapis/python-bigquery/compare/v2.25.0...v2.25.1) (2021-08-25) ##### Bug Fixes - populate default `timeout` and retry after client-side timeout ([#​896](https://www.togithub.com/googleapis/python-bigquery/issues/896)) ([b508809](https://www.github.com/googleapis/python-bigquery/commit/b508809c0f887575274309a463e763c56ddd017d)) - use REST API in cell magic when requested ([#​892](https://www.togithub.com/googleapis/python-bigquery/issues/892)) ([1cb3e55](https://www.github.com/googleapis/python-bigquery/commit/1cb3e55253e824e3a1da5201f6ec09065fb6b627))
googleapis/google-resumable-media-python ### [`v2.0.1`](https://togithub.com/googleapis/google-resumable-media-python/blob/master/CHANGELOG.md#​201-httpswwwgithubcomgoogleapisgoogle-resumable-media-pythoncomparev200v201-2021-08-30) [Compare Source](https://togithub.com/googleapis/google-resumable-media-python/compare/v2.0.0...v2.0.1)
--- ### Configuration 📅 **Schedule**: At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 👻 **Immortal**: This PR will be recreated if closed unmerged. Get [config help](https://togithub.com/renovatebot/renovate/discussions) if that's undesired. --- - [ ] If you want to rebase/retry this PR, check this box. --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/python-bigquery-sqlalchemy). --- samples/snippets/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index e38cfa20..2933a4c1 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -14,13 +14,13 @@ Fiona==1.8.20 future==0.18.2 GeoAlchemy2==0.9.4 geopandas==0.9.0 -google-api-core==2.0.0 +google-api-core==2.0.1 google-auth==2.0.2 -google-cloud-bigquery==2.25.2 +google-cloud-bigquery==2.26.0 google-cloud-bigquery-storage==2.6.3 google-cloud-core==2.0.0 google-crc32c==1.1.3 -google-resumable-media==2.0.0 +google-resumable-media==2.0.1 googleapis-common-protos==1.53.0 greenlet==1.1.1 grpcio==1.39.0 From 9b2acbb277b340af933bce920b010b717380b912 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 2 Sep 2021 21:30:47 +0200 Subject: [PATCH 11/16] chore(deps): update all dependencies (#329) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![WhiteSource Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [google-cloud-bigquery-storage](https://togithub.com/googleapis/python-bigquery-storage) | `==2.6.3` -> `==2.7.0` | [![age](https://badges.renovateapi.com/packages/pypi/google-cloud-bigquery-storage/2.7.0/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/pypi/google-cloud-bigquery-storage/2.7.0/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/pypi/google-cloud-bigquery-storage/2.7.0/compatibility-slim/2.6.3)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/pypi/google-cloud-bigquery-storage/2.7.0/confidence-slim/2.6.3)](https://docs.renovatebot.com/merge-confidence/) | | [google-crc32c](https://togithub.com/googleapis/python-crc32c) | `==1.1.3` -> `==1.1.4` | [![age](https://badges.renovateapi.com/packages/pypi/google-crc32c/1.1.4/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/pypi/google-crc32c/1.1.4/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/pypi/google-crc32c/1.1.4/compatibility-slim/1.1.3)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/pypi/google-crc32c/1.1.4/confidence-slim/1.1.3)](https://docs.renovatebot.com/merge-confidence/) | --- ### Release Notes
googleapis/python-bigquery-storage ### [`v2.7.0`](https://togithub.com/googleapis/python-bigquery-storage/blob/master/CHANGELOG.md#​270-httpswwwgithubcomgoogleapispython-bigquery-storagecomparev263v270-2021-09-02) [Compare Source](https://togithub.com/googleapis/python-bigquery-storage/compare/v2.6.3...v2.7.0) ##### Features - **v1beta2:** Align ReadRows timeout with other versions of the API ([#​293](https://www.togithub.com/googleapis/python-bigquery-storage/issues/293)) ([43e36a1](https://www.github.com/googleapis/python-bigquery-storage/commit/43e36a13ece8d876763d88bad0252a1b2421c52a)) ##### Documentation - **v1beta2:** Align session length with public documentation ([43e36a1](https://www.github.com/googleapis/python-bigquery-storage/commit/43e36a13ece8d876763d88bad0252a1b2421c52a)) ##### [2.6.3](https://www.github.com/googleapis/python-bigquery-storage/compare/v2.6.2...v2.6.3) (2021-08-06) ##### Bug Fixes - resume read stream on `Unknown` transport-layer exception ([#​263](https://www.togithub.com/googleapis/python-bigquery-storage/issues/263)) ([127caa0](https://www.github.com/googleapis/python-bigquery-storage/commit/127caa06144b9cec04b23914b561be6a264bcb36)) ##### [2.6.2](https://www.github.com/googleapis/python-bigquery-storage/compare/v2.6.1...v2.6.2) (2021-07-28) ##### Bug Fixes - enable self signed jwt for grpc ([#​249](https://www.togithub.com/googleapis/python-bigquery-storage/issues/249)) ([a7e8d91](https://www.github.com/googleapis/python-bigquery-storage/commit/a7e8d913fc3de67a3f38ecbd35af2f9d1a33aa8d)) ##### Documentation - remove duplicate code samples ([#​246](https://www.togithub.com/googleapis/python-bigquery-storage/issues/246)) ([303f273](https://www.github.com/googleapis/python-bigquery-storage/commit/303f2732ced38e491df92e965dd37bac24a61d2f)) - add Samples section to CONTRIBUTING.rst ([#​241](https://www.togithub.com/googleapis/python-bigquery-storage/issues/241)) ([5d02358](https://www.github.com/googleapis/python-bigquery-storage/commit/5d02358fbd397cafcc1169d829859fe2dd568645)) ##### [2.6.1](https://www.github.com/googleapis/python-bigquery-storage/compare/v2.6.0...v2.6.1) (2021-07-20) ##### Bug Fixes - **deps:** pin 'google-{api,cloud}-core', 'google-auth' to allow 2.x versions ([#​240](https://www.togithub.com/googleapis/python-bigquery-storage/issues/240)) ([8f848e1](https://www.github.com/googleapis/python-bigquery-storage/commit/8f848e18379085160492cdd2d12dc8de50a46c8e)) ##### Documentation - pandas DataFrame samples are more standalone ([#​224](https://www.togithub.com/googleapis/python-bigquery-storage/issues/224)) ([4026997](https://www.github.com/googleapis/python-bigquery-storage/commit/4026997d7a286b63ed2b969c0bd49de59635326d))
googleapis/python-crc32c ### [`v1.1.4`](https://togithub.com/googleapis/python-crc32c/blob/master/CHANGELOG.md#​114-httpswwwgithubcomgoogleapispython-crc32ccomparev114v114-2021-09-02) [Compare Source](https://togithub.com/googleapis/python-crc32c/compare/v1.1.3...v1.1.4)
--- ### Configuration 📅 **Schedule**: At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 👻 **Immortal**: This PR will be recreated if closed unmerged. Get [config help](https://togithub.com/renovatebot/renovate/discussions) if that's undesired. --- - [ ] If you want to rebase/retry this PR, check this box. --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/python-bigquery-sqlalchemy). --- samples/snippets/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 2933a4c1..29985c23 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -17,9 +17,9 @@ geopandas==0.9.0 google-api-core==2.0.1 google-auth==2.0.2 google-cloud-bigquery==2.26.0 -google-cloud-bigquery-storage==2.6.3 +google-cloud-bigquery-storage==2.7.0 google-cloud-core==2.0.0 -google-crc32c==1.1.3 +google-crc32c==1.1.4 google-resumable-media==2.0.1 googleapis-common-protos==1.53.0 greenlet==1.1.1 From 853d2879ef0f6399c25742b29e122e70a7f2dfa4 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 3 Sep 2021 08:59:27 -0400 Subject: [PATCH 12/16] chore: use public post processor image (#333) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: use public post processor image * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 4 ++-- .github/.OwlBot.yaml | 2 +- CONTRIBUTING.rst | 6 +++--- owlbot.py | 45 --------------------------------------- 4 files changed, 6 insertions(+), 51 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index ef3cb34f..7b6cc310 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: - image: gcr.io/repo-automation-bots/owlbot-python:latest - digest: sha256:1456ea2b3b523ccff5e13030acef56d1de28f21249c62aa0f196265880338fa7 + image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest + digest: sha256:a3a85c2e0b3293068e47b1635b178f7e3d3845f2cfb8722de6713d4bbafdcd1d diff --git a/.github/.OwlBot.yaml b/.github/.OwlBot.yaml index 243bb8bf..3cbe64b8 100644 --- a/.github/.OwlBot.yaml +++ b/.github/.OwlBot.yaml @@ -13,7 +13,7 @@ # limitations under the License. docker: - image: gcr.io/repo-automation-bots/owlbot-python:latest + image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest begin-after-commit-hash: be22498ce258bf2d5fe12fd696d3ad9a2b6c430e diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 2083c729..779aa5a1 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -113,9 +113,9 @@ Coding Style export GOOGLE_CLOUD_TESTING_BRANCH="main" By doing this, you are specifying the location of the most up-to-date - version of ``python-bigquery-sqlalchemy``. The the suggested remote name ``upstream`` - should point to the official ``googleapis`` checkout and the - the branch should be the main branch on that remote (``main``). + version of ``python-bigquery-sqlalchemy``. The + remote name ``upstream`` should point to the official ``googleapis`` + checkout and the branch should be the default branch on that remote (``main``). - This repository contains configuration for the `pre-commit `__ tool, which automates checking diff --git a/owlbot.py b/owlbot.py index f8633351..dcec1b04 100644 --- a/owlbot.py +++ b/owlbot.py @@ -194,51 +194,6 @@ def compliance(session): python.py_samples(skip_readmes=True) -# ---------------------------------------------------------------------------- -# Remove the replacements below once https://github.com/googleapis/synthtool/pull/1188 is merged - -# Update googleapis/repo-automation-bots repo to main in .kokoro/*.sh files -s.replace(".kokoro/*.sh", "repo-automation-bots/tree/master", "repo-automation-bots/tree/main") - -# Customize CONTRIBUTING.rst to replace master with main -s.replace( - "CONTRIBUTING.rst", - "fetch and merge changes from upstream into master", - "fetch and merge changes from upstream into main", -) - -s.replace( - "CONTRIBUTING.rst", - "git merge upstream/master", - "git merge upstream/main", -) - -s.replace( - "CONTRIBUTING.rst", - """export GOOGLE_CLOUD_TESTING_BRANCH=\"master\"""", - """export GOOGLE_CLOUD_TESTING_BRANCH=\"main\"""", -) - -s.replace( - "CONTRIBUTING.rst", - "remote \(``master``\)", - "remote (``main``)", -) - -s.replace( - "CONTRIBUTING.rst", - "blob/master/CONTRIBUTING.rst", - "blob/main/CONTRIBUTING.rst", -) - -s.replace( - "CONTRIBUTING.rst", - "blob/master/noxfile.py", - "blob/main/noxfile.py", -) - -s.replace("docs/conf.py", "master", "root") - # ---------------------------------------------------------------------------- # Final cleanup # ---------------------------------------------------------------------------- From a42283c5f617c8e0fa4cb3a22277c3a21a0688df Mon Sep 17 00:00:00 2001 From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Date: Tue, 7 Sep 2021 11:28:11 -0600 Subject: [PATCH 13/16] chore: reference main branch of google-cloud-python (#334) Adjust google-cloud-python links to reference main branch. --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 56e64921..9b76eab6 100644 --- a/README.rst +++ b/README.rst @@ -9,7 +9,7 @@ SQLAlchemy Dialect for BigQuery - `Product Documentation`_ .. |GA| image:: https://img.shields.io/badge/support-GA-gold.svg - :target: https://github.com/googleapis/google-cloud-python/blob/master/README.rst#general-availability + :target: https://github.com/googleapis/google-cloud-python/blob/main/README.rst#general-availability .. |pypi| image:: https://img.shields.io/pypi/v/sqlalchemy-bigquery.svg :target: https://pypi.org/project/sqlalchemy-bigquery/ .. |versions| image:: https://img.shields.io/pypi/pyversions/sqlalchemy-bigquery.svg From 6624b10ded73bbca6f40af73aaeaceb95c381b63 Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Thu, 9 Sep 2021 09:50:04 -0600 Subject: [PATCH 14/16] feat: STRUCT and ARRAY support (#318) --- docs/alembic.rst | 2 +- docs/index.rst | 1 + docs/struct.rst | 69 +++++++ samples/snippets/STRUCT.py | 90 +++++++++ samples/snippets/STRUCT_test.py | 27 +++ .../{test_geography.py => geography_test.py} | 0 setup.py | 2 +- sqlalchemy_bigquery/__init__.py | 4 +- sqlalchemy_bigquery/_struct.py | 148 ++++++++++++++ sqlalchemy_bigquery/_types.py | 141 +++++++++++++ sqlalchemy_bigquery/base.py | 123 +----------- sqlalchemy_bigquery/geography.py | 2 +- testing/constraints-3.6.txt | 2 +- tests/system/conftest.py | 13 ++ tests/system/test__struct.py | 189 ++++++++++++++++++ tests/system/test_sqlalchemy_bigquery.py | 33 ++- tests/unit/conftest.py | 5 + tests/unit/test__struct.py | 152 ++++++++++++++ tests/unit/test_catalog_functions.py | 36 ++-- tests/unit/test_dialect_types.py | 4 +- tests/unit/test_select.py | 19 +- 21 files changed, 903 insertions(+), 159 deletions(-) create mode 100644 docs/struct.rst create mode 100644 samples/snippets/STRUCT.py create mode 100644 samples/snippets/STRUCT_test.py rename samples/snippets/{test_geography.py => geography_test.py} (100%) create mode 100644 sqlalchemy_bigquery/_struct.py create mode 100644 sqlalchemy_bigquery/_types.py create mode 100644 tests/system/test__struct.py create mode 100644 tests/unit/test__struct.py diff --git a/docs/alembic.rst b/docs/alembic.rst index e83953a0..8b5df741 100644 --- a/docs/alembic.rst +++ b/docs/alembic.rst @@ -43,7 +43,7 @@ Supported operations: `_ Note that some of the operations above have limited capability, again -do to `BigQuery limitations +due to `BigQuery limitations `_. The `execute` operation allows access to BigQuery-specific diff --git a/docs/index.rst b/docs/index.rst index 4fe42891..c70b2d2f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,6 +3,7 @@ :maxdepth: 2 README + struct geography alembic reference diff --git a/docs/struct.rst b/docs/struct.rst new file mode 100644 index 00000000..9b2d5724 --- /dev/null +++ b/docs/struct.rst @@ -0,0 +1,69 @@ +Working with BigQuery STRUCT data +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The BigQuery `STRUCT data type +`_ +provided data that are collections of named fields. + +`sqlalchemy-bigquery` provided a STRUCT type that can be used to +define tables with STRUCT columns: + +.. literalinclude:: samples/snippets/STRUCT.py + :language: python + :dedent: 4 + :start-after: [START bigquery_sqlalchemy_create_table_with_struct] + :end-before: [END bigquery_sqlalchemy_create_table_with_struct] + +`STRUCT` types can be nested, as in this example. Struct fields can +be defined in two ways: + +- Fields can be provided as keyword arguments, as in the `cylinder` + and `horsepower` fields in this example. + +- Fields can be provided as name-type tuples provided as positional + arguments, as with the `count` and `compression` fields in this example. + +STRUCT columns are automatically created when existing database tables +containing STRUCT columns are introspected. + +Struct data are represented in Python as Python dictionaries: + +.. literalinclude:: samples/snippets/STRUCT.py + :language: python + :dedent: 4 + :start-after: [START bigquery_sqlalchemy_insert_struct] + :end-before: [END bigquery_sqlalchemy_insert_struct] + +When querying struct fields, you can use attribute access syntax: + +.. literalinclude:: samples/snippets/STRUCT.py + :language: python + :dedent: 4 + :start-after: [START bigquery_sqlalchemy_query_struct] + :end-before: [END bigquery_sqlalchemy_query_struct] + +or mapping access: + +.. literalinclude:: samples/snippets/STRUCT.py + :language: python + :dedent: 4 + :start-after: [START bigquery_sqlalchemy_query_getitem] + :end-before: [END bigquery_sqlalchemy_query_getitem] + +and field names are case insensitive: + +.. literalinclude:: samples/snippets/STRUCT.py + :language: python + :dedent: 4 + :start-after: [START bigquery_sqlalchemy_query_STRUCT] + :end-before: [END bigquery_sqlalchemy_query_STRUCT] + +When using attribute-access syntax, field names may conflict with +column attribute names. For example SQLAlchemy columns have `name` +and `type` attributes, among others. When accessing a field whose name +conflicts with a column attribute name, either use mapping access, or +spell the field name with upper-case letters. + + + + diff --git a/samples/snippets/STRUCT.py b/samples/snippets/STRUCT.py new file mode 100644 index 00000000..ce59f90b --- /dev/null +++ b/samples/snippets/STRUCT.py @@ -0,0 +1,90 @@ +# Copyright (c) 2021 The sqlalchemy-bigquery Authors +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +def example(engine): + # fmt: off + # [START bigquery_sqlalchemy_create_table_with_struct] + from sqlalchemy.ext.declarative import declarative_base + from sqlalchemy import Column, String, Integer, Float + from sqlalchemy_bigquery import STRUCT + + Base = declarative_base() + + class Car(Base): + __tablename__ = "Cars" + + model = Column(String, primary_key=True) + engine = Column( + STRUCT( + cylinder=STRUCT(("count", Integer), + ("compression", Float)), + horsepower=Integer) + ) + + # [END bigquery_sqlalchemy_create_table_with_struct] + Car.__table__.create(engine) + + # [START bigquery_sqlalchemy_insert_struct] + from sqlalchemy.orm import sessionmaker + + Session = sessionmaker(bind=engine) + session = Session() + + sebring = Car(model="Sebring", + engine=dict( + cylinder=dict( + count=6, + compression=18.0), + horsepower=235)) + townc = Car(model="Town and Counttry", + engine=dict( + cylinder=dict( + count=6, + compression=16.0), + horsepower=251)) + xj8 = Car(model="XJ8", + engine=dict( + cylinder=dict( + count=8, + compression=10.75), + horsepower=575)) + + session.add_all((sebring, townc, xj8)) + session.commit() + + # [END bigquery_sqlalchemy_insert_struct] + + # [START bigquery_sqlalchemy_query_struct] + sixes = session.query(Car).filter(Car.engine.cylinder.count == 6) + # [END bigquery_sqlalchemy_query_struct] + sixes1 = list(sixes) + + # [START bigquery_sqlalchemy_query_STRUCT] + sixes = session.query(Car).filter(Car.engine.CYLINDER.COUNT == 6) + # [END bigquery_sqlalchemy_query_STRUCT] + sixes2 = list(sixes) + + # [START bigquery_sqlalchemy_query_getitem] + sixes = session.query(Car).filter(Car.engine["cylinder"]["count"] == 6) + # [END bigquery_sqlalchemy_query_getitem] + # fmt: on + sixes3 = list(sixes) + + return sixes1, sixes2, sixes3 diff --git a/samples/snippets/STRUCT_test.py b/samples/snippets/STRUCT_test.py new file mode 100644 index 00000000..5a5c6515 --- /dev/null +++ b/samples/snippets/STRUCT_test.py @@ -0,0 +1,27 @@ +# Copyright (c) 2021 The sqlalchemy-bigquery Authors +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +def test_struct(engine): + from . import STRUCT + + sixeses = STRUCT.example(engine) + + for sixes in sixeses: + assert sorted(car.model for car in sixes) == ["Sebring", "Town and Counttry"] diff --git a/samples/snippets/test_geography.py b/samples/snippets/geography_test.py similarity index 100% rename from samples/snippets/test_geography.py rename to samples/snippets/geography_test.py diff --git a/setup.py b/setup.py index f70c3a0d..7efe0f9b 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ def readme(): # Until this issue is closed # https://github.com/googleapis/google-cloud-python/issues/10566 "google-auth>=1.25.0,<3.0.0dev", # Work around pip wack. - "google-cloud-bigquery>=2.24.1", + "google-cloud-bigquery>=2.25.2,<3.0.0dev", "sqlalchemy>=1.2.0,<1.5.0dev", "future", ], diff --git a/sqlalchemy_bigquery/__init__.py b/sqlalchemy_bigquery/__init__.py index f0defda1..f0d8a6c6 100644 --- a/sqlalchemy_bigquery/__init__.py +++ b/sqlalchemy_bigquery/__init__.py @@ -23,7 +23,7 @@ from .version import __version__ # noqa from .base import BigQueryDialect, dialect # noqa -from .base import ( +from ._types import ( ARRAY, BIGNUMERIC, BOOL, @@ -38,6 +38,7 @@ NUMERIC, RECORD, STRING, + STRUCT, TIME, TIMESTAMP, ) @@ -58,6 +59,7 @@ "NUMERIC", "RECORD", "STRING", + "STRUCT", "TIME", "TIMESTAMP", ] diff --git a/sqlalchemy_bigquery/_struct.py b/sqlalchemy_bigquery/_struct.py new file mode 100644 index 00000000..a3d9aba4 --- /dev/null +++ b/sqlalchemy_bigquery/_struct.py @@ -0,0 +1,148 @@ +# Copyright (c) 2021 The sqlalchemy-bigquery Authors +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +from typing import Mapping, Tuple + +import packaging.version +import sqlalchemy.sql.default_comparator +import sqlalchemy.sql.sqltypes +import sqlalchemy.types + +from . import base + +sqlalchemy_1_4_or_more = packaging.version.parse( + sqlalchemy.__version__ +) >= packaging.version.parse("1.4") + +if sqlalchemy_1_4_or_more: + import sqlalchemy.sql.coercions + import sqlalchemy.sql.roles + + +def _get_subtype_col_spec(type_): + global _get_subtype_col_spec + + type_compiler = base.dialect.type_compiler(base.dialect()) + _get_subtype_col_spec = type_compiler.process + return _get_subtype_col_spec(type_) + + +class STRUCT(sqlalchemy.sql.sqltypes.Indexable, sqlalchemy.types.UserDefinedType): + """ + A type for BigQuery STRUCT/RECORD data + + See https://googleapis.dev/python/sqlalchemy-bigquery/latest/struct.html + """ + + # See https://docs.sqlalchemy.org/en/14/core/custom_types.html#creating-new-types + + def __init__( + self, + *fields: Tuple[str, sqlalchemy.types.TypeEngine], + **kwfields: Mapping[str, sqlalchemy.types.TypeEngine], + ): + # Note that because: + # https://docs.python.org/3/whatsnew/3.6.html#pep-468-preserving-keyword-argument-order + # We know that `kwfields` preserves order. + self._STRUCT_fields = tuple( + ( + name, + type_ if isinstance(type_, sqlalchemy.types.TypeEngine) else type_(), + ) + for (name, type_) in (fields + tuple(kwfields.items())) + ) + + self._STRUCT_byname = { + name.lower(): type_ for (name, type_) in self._STRUCT_fields + } + + def __repr__(self): + fields = ", ".join( + f"{name}={repr(type_)}" for name, type_ in self._STRUCT_fields + ) + return f"STRUCT({fields})" + + def get_col_spec(self, **kw): + fields = ", ".join( + f"{name} {_get_subtype_col_spec(type_)}" + for name, type_ in self._STRUCT_fields + ) + return f"STRUCT<{fields}>" + + def bind_processor(self, dialect): + return dict + + class Comparator(sqlalchemy.sql.sqltypes.Indexable.Comparator): + def _setup_getitem(self, name): + if not isinstance(name, str): + raise TypeError( + f"STRUCT fields can only be accessed with strings field names," + f" not {repr(name)}." + ) + subtype = self.expr.type._STRUCT_byname.get(name.lower()) + if subtype is None: + raise KeyError(name) + operator = struct_getitem_op + index = _field_index(self, name, operator) + return operator, index, subtype + + def __getattr__(self, name): + if name.lower() in self.expr.type._STRUCT_byname: + return self[name] + + comparator_factory = Comparator + + +# In the implementations of _field_index below, we're stealing from +# the JSON type implementation, but the code to steal changed in +# 1.4. :/ + +if sqlalchemy_1_4_or_more: + + def _field_index(self, name, operator): + return sqlalchemy.sql.coercions.expect( + sqlalchemy.sql.roles.BinaryElementRole, + name, + expr=self.expr, + operator=operator, + bindparam_type=sqlalchemy.types.String(), + ) + + +else: + + def _field_index(self, name, operator): + return sqlalchemy.sql.default_comparator._check_literal( + self.expr, operator, name, bindparam_type=sqlalchemy.types.String(), + ) + + +def struct_getitem_op(a, b): + raise NotImplementedError() + + +sqlalchemy.sql.default_comparator.operator_lookup[ + struct_getitem_op.__name__ +] = sqlalchemy.sql.default_comparator.operator_lookup["json_getitem_op"] + + +class SQLCompiler: + def visit_struct_getitem_op_binary(self, binary, operator_, **kw): + left = self.process(binary.left, **kw) + return f"{left}.{binary.right.value}" diff --git a/sqlalchemy_bigquery/_types.py b/sqlalchemy_bigquery/_types.py new file mode 100644 index 00000000..4e18dc2a --- /dev/null +++ b/sqlalchemy_bigquery/_types.py @@ -0,0 +1,141 @@ +# Copyright (c) 2021 The sqlalchemy-bigquery Authors +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import sqlalchemy.types +import sqlalchemy.util + +from google.cloud.bigquery.schema import SchemaField + +try: + from .geography import GEOGRAPHY +except ImportError: + pass + +from ._struct import STRUCT + +_type_map = { + "ARRAY": sqlalchemy.types.ARRAY, + "BIGNUMERIC": sqlalchemy.types.Numeric, + "BOOLEAN": sqlalchemy.types.Boolean, + "BOOL": sqlalchemy.types.Boolean, + "BYTES": sqlalchemy.types.BINARY, + "DATETIME": sqlalchemy.types.DATETIME, + "DATE": sqlalchemy.types.DATE, + "FLOAT64": sqlalchemy.types.Float, + "FLOAT": sqlalchemy.types.Float, + "INT64": sqlalchemy.types.Integer, + "INTEGER": sqlalchemy.types.Integer, + "NUMERIC": sqlalchemy.types.Numeric, + "RECORD": STRUCT, + "STRING": sqlalchemy.types.String, + "STRUCT": STRUCT, + "TIMESTAMP": sqlalchemy.types.TIMESTAMP, + "TIME": sqlalchemy.types.TIME, +} + +# By convention, dialect-provided types are spelled with all upper case. +ARRAY = _type_map["ARRAY"] +BIGNUMERIC = _type_map["NUMERIC"] +BOOLEAN = _type_map["BOOLEAN"] +BOOL = _type_map["BOOL"] +BYTES = _type_map["BYTES"] +DATETIME = _type_map["DATETIME"] +DATE = _type_map["DATE"] +FLOAT64 = _type_map["FLOAT64"] +FLOAT = _type_map["FLOAT"] +INT64 = _type_map["INT64"] +INTEGER = _type_map["INTEGER"] +NUMERIC = _type_map["NUMERIC"] +RECORD = _type_map["RECORD"] +STRING = _type_map["STRING"] +TIMESTAMP = _type_map["TIMESTAMP"] +TIME = _type_map["TIME"] + +try: + _type_map["GEOGRAPHY"] = GEOGRAPHY +except NameError: + pass + +STRUCT_FIELD_TYPES = "RECORD", "STRUCT" + + +def _get_transitive_schema_fields(fields): + """ + Recurse into record type and return all the nested field names. + As contributed by @sumedhsakdeo on issue #17 + """ + results = [] + for field in fields: + results += [field] + if field.field_type in STRUCT_FIELD_TYPES: + sub_fields = [ + SchemaField.from_api_repr( + dict(f.to_api_repr(), name=f"{field.name}.{f.name}") + ) + for f in field.fields + ] + results += _get_transitive_schema_fields(sub_fields) + return results + + +def _get_sqla_column_type(field): + try: + coltype = _type_map[field.field_type] + except KeyError: + sqlalchemy.util.warn( + "Did not recognize type '%s' of column '%s'" + % (field.field_type, field.name) + ) + coltype = sqlalchemy.types.NullType + else: + if field.field_type.endswith("NUMERIC"): + coltype = coltype(precision=field.precision, scale=field.scale) + elif field.field_type == "STRING" or field.field_type == "BYTES": + coltype = coltype(field.max_length) + elif field.field_type == "RECORD" or field.field_type == "STRUCT": + coltype = STRUCT( + *( + (subfield.name, _get_sqla_column_type(subfield)) + for subfield in field.fields + ) + ) + else: + coltype = coltype() + + if field.mode == "REPEATED": + coltype = ARRAY(coltype) + + return coltype + + +def get_columns(bq_schema): + fields = _get_transitive_schema_fields(bq_schema) + return [ + { + "name": field.name, + "type": _get_sqla_column_type(field), + "nullable": field.mode == "NULLABLE" or field.mode == "REPEATED", + "comment": field.description, + "default": None, + "precision": field.precision, + "scale": field.scale, + "max_length": field.max_length, + } + for field in fields + ] diff --git a/sqlalchemy_bigquery/base.py b/sqlalchemy_bigquery/base.py index e03d074e..f5b1d515 100644 --- a/sqlalchemy_bigquery/base.py +++ b/sqlalchemy_bigquery/base.py @@ -30,7 +30,6 @@ from google import auth import google.api_core.exceptions from google.cloud.bigquery import dbapi -from google.cloud.bigquery.schema import SchemaField from google.cloud.bigquery.table import TableReference from google.api_core.exceptions import NotFound @@ -40,7 +39,7 @@ import sqlalchemy.sql.sqltypes import sqlalchemy.sql.type_api from sqlalchemy.exc import NoSuchTableError -from sqlalchemy import types, util +from sqlalchemy import util from sqlalchemy.sql.compiler import ( SQLCompiler, GenericTypeCompiler, @@ -55,12 +54,7 @@ import re from .parse_url import parse_url -from sqlalchemy_bigquery import _helpers - -try: - from .geography import GEOGRAPHY -except ImportError: - pass +from . import _helpers, _struct, _types FIELD_ILLEGAL_CHARACTERS = re.compile(r"[^\w]+") @@ -117,49 +111,6 @@ def format_label(self, label, name=None): return result -_type_map = { - "ARRAY": types.ARRAY, - "BIGNUMERIC": types.Numeric, - "BOOLEAN": types.Boolean, - "BOOL": types.Boolean, - "BYTES": types.BINARY, - "DATETIME": types.DATETIME, - "DATE": types.DATE, - "FLOAT64": types.Float, - "FLOAT": types.Float, - "INT64": types.Integer, - "INTEGER": types.Integer, - "NUMERIC": types.Numeric, - "RECORD": types.JSON, - "STRING": types.String, - "TIMESTAMP": types.TIMESTAMP, - "TIME": types.TIME, -} - -# By convention, dialect-provided types are spelled with all upper case. -ARRAY = _type_map["ARRAY"] -BIGNUMERIC = _type_map["NUMERIC"] -BOOLEAN = _type_map["BOOLEAN"] -BOOL = _type_map["BOOL"] -BYTES = _type_map["BYTES"] -DATETIME = _type_map["DATETIME"] -DATE = _type_map["DATE"] -FLOAT64 = _type_map["FLOAT64"] -FLOAT = _type_map["FLOAT"] -INT64 = _type_map["INT64"] -INTEGER = _type_map["INTEGER"] -NUMERIC = _type_map["NUMERIC"] -RECORD = _type_map["RECORD"] -STRING = _type_map["STRING"] -TIMESTAMP = _type_map["TIMESTAMP"] -TIME = _type_map["TIME"] - -try: - _type_map["GEOGRAPHY"] = GEOGRAPHY -except NameError: - pass - - class BigQueryExecutionContext(DefaultExecutionContext): def create_cursor(self): # Set arraysize @@ -227,7 +178,7 @@ def pre_exec(self): ) -class BigQueryCompiler(SQLCompiler): +class BigQueryCompiler(_struct.SQLCompiler, SQLCompiler): compound_keywords = SQLCompiler.compound_keywords.copy() compound_keywords[selectable.CompoundSelect.UNION] = "UNION DISTINCT" @@ -543,9 +494,6 @@ def visit_bindparam( type_.scale = -t.exponent bq_type = self.dialect.type_compiler.process(type_) - if bq_type[-1] == ">" and bq_type.startswith("ARRAY<"): - # Values get arrayified at a lower level. - bq_type = bq_type[6:-1] bq_type = self.__remove_type_parameter(bq_type) assert_(param != "%s", f"Unexpected param: {param}") @@ -566,6 +514,11 @@ def visit_bindparam( return param + def visit_getitem_binary(self, binary, operator_, **kw): + left = self.process(binary.left, **kw) + right = self.process(binary.right, **kw) + return f"{left}[OFFSET({right})]" + class BigQueryTypeCompiler(GenericTypeCompiler): def visit_INTEGER(self, type_, **kw): @@ -836,14 +789,6 @@ def create_connect_args(self, url): ) return ([client], {}) - def _json_deserializer(self, row): - """JSON deserializer for RECORD types. - - The DB-API layer already deserializes JSON to a dictionary, so this - just returns the input. - """ - return row - def _get_table_or_view_names(self, connection, table_type, schema=None): current_schema = schema or self.dataset_id get_table_name = ( @@ -968,59 +913,9 @@ def has_table(self, connection, table_name, schema=None): except NoSuchTableError: return False - def _get_columns_helper(self, columns, cur_columns): - """ - Recurse into record type and return all the nested field names. - As contributed by @sumedhsakdeo on issue #17 - """ - results = [] - for col in columns: - results += [col] - if col.field_type == "RECORD": - cur_columns.append(col) - fields = [ - SchemaField.from_api_repr( - dict(f.to_api_repr(), name=f"{col.name}.{f.name}") - ) - for f in col.fields - ] - results += self._get_columns_helper(fields, cur_columns) - cur_columns.pop() - return results - def get_columns(self, connection, table_name, schema=None, **kw): table = self._get_table(connection, table_name, schema) - columns = self._get_columns_helper(table.schema, []) - result = [] - for col in columns: - try: - coltype = _type_map[col.field_type] - except KeyError: - util.warn( - "Did not recognize type '%s' of column '%s'" - % (col.field_type, col.name) - ) - coltype = types.NullType - - if col.field_type.endswith("NUMERIC"): - coltype = coltype(precision=col.precision, scale=col.scale) - elif col.field_type == "STRING" or col.field_type == "BYTES": - coltype = coltype(col.max_length) - - result.append( - { - "name": col.name, - "type": types.ARRAY(coltype) if col.mode == "REPEATED" else coltype, - "nullable": col.mode == "NULLABLE" or col.mode == "REPEATED", - "comment": col.description, - "default": None, - "precision": col.precision, - "scale": col.scale, - "max_length": col.max_length, - } - ) - - return result + return _types.get_columns(table.schema) def get_table_comment(self, connection, table_name, schema=None, **kw): table = self._get_table(connection, table_name, schema) diff --git a/sqlalchemy_bigquery/geography.py b/sqlalchemy_bigquery/geography.py index 9a10c236..16384dd4 100644 --- a/sqlalchemy_bigquery/geography.py +++ b/sqlalchemy_bigquery/geography.py @@ -95,7 +95,7 @@ class Lake(Base): name = Column(String) geog = column(GEOGRAPHY) - + See https://googleapis.dev/python/sqlalchemy-bigquery/latest/geography.html """ def __init__(self): diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt index e5ed0b2a..60421130 100644 --- a/testing/constraints-3.6.txt +++ b/testing/constraints-3.6.txt @@ -6,5 +6,5 @@ # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", sqlalchemy==1.2.0 google-auth==1.25.0 -google-cloud-bigquery==2.24.1 +google-cloud-bigquery==2.25.2 google-api-core==1.30.0 diff --git a/tests/system/conftest.py b/tests/system/conftest.py index d9db14ab..7bf76a2d 100644 --- a/tests/system/conftest.py +++ b/tests/system/conftest.py @@ -26,6 +26,8 @@ from google.cloud import bigquery import test_utils.prefixer +from sqlalchemy_bigquery import BigQueryDialect + prefixer = test_utils.prefixer.Prefixer("python-bigquery-sqlalchemy", "tests/system") DATA_DIR = pathlib.Path(__file__).parent / "data" @@ -140,6 +142,17 @@ def cleanup_datasets(bigquery_client: bigquery.Client): ) +@pytest.fixture(scope="session") +def engine(): + engine = sqlalchemy.create_engine("bigquery://", echo=True) + return engine + + +@pytest.fixture(scope="session") +def dialect(): + return BigQueryDialect() + + @pytest.fixture def metadata(): return sqlalchemy.MetaData() diff --git a/tests/system/test__struct.py b/tests/system/test__struct.py new file mode 100644 index 00000000..24863056 --- /dev/null +++ b/tests/system/test__struct.py @@ -0,0 +1,189 @@ +# Copyright (c) 2021 The sqlalchemy-bigquery Authors +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import datetime + +import packaging.version +import pytest +import sqlalchemy + +import sqlalchemy_bigquery + + +def test_struct(engine, bigquery_dataset, metadata): + conn = engine.connect() + table = sqlalchemy.Table( + f"{bigquery_dataset}.test_struct", + metadata, + sqlalchemy.Column( + "person", + sqlalchemy_bigquery.STRUCT( + name=sqlalchemy.String, + children=sqlalchemy.ARRAY( + sqlalchemy_bigquery.STRUCT( + name=sqlalchemy.String, bdate=sqlalchemy.DATE + ) + ), + ), + ), + ) + metadata.create_all(engine) + + conn.execute( + table.insert().values( + person=dict( + name="bob", + children=[dict(name="billy", bdate=datetime.date(2020, 1, 1))], + ) + ) + ) + + assert list(conn.execute(sqlalchemy.select([table]))) == [ + ( + { + "name": "bob", + "children": [{"name": "billy", "bdate": datetime.date(2020, 1, 1)}], + }, + ) + ] + assert list(conn.execute(sqlalchemy.select([table.c.person.NAME]))) == [("bob",)] + assert list(conn.execute(sqlalchemy.select([table.c.person.children[0]]))) == [ + ({"name": "billy", "bdate": datetime.date(2020, 1, 1)},) + ] + assert list( + conn.execute(sqlalchemy.select([table.c.person.children[0].bdate])) + ) == [(datetime.date(2020, 1, 1),)] + assert list( + conn.execute( + sqlalchemy.select([table]).where(table.c.person.children[0].NAME == "billy") + ) + ) == [ + ( + { + "name": "bob", + "children": [{"name": "billy", "bdate": datetime.date(2020, 1, 1)}], + }, + ) + ] + assert ( + list( + conn.execute( + sqlalchemy.select([table]).where( + table.c.person.children[0].NAME == "sally" + ) + ) + ) + == [] + ) + + +def test_complex_literals_pr_67(engine, bigquery_dataset, metadata): + # https://github.com/googleapis/python-bigquery-sqlalchemy/pull/67 + + # Simple select example: + + table_name = f"{bigquery_dataset}.test_comples_literals_pr_67" + engine.execute( + f""" + create table {table_name} as ( + select 'a' as id, + struct(1 as x__count, 2 as y__count, 3 as z__count) as dimensions + ) + """ + ) + + table = sqlalchemy.Table(table_name, metadata, autoload_with=engine) + + got = str( + sqlalchemy.select([(table.c.dimensions.x__count + 5).label("c")]).compile( + engine + ) + ) + want = ( + f"SELECT (`{table_name}`.`dimensions`.x__count) + %(param_1:INT64)s AS `c` \n" + f"FROM `{table_name}`" + ) + + assert got == want + + # Hopefully, "Example doing a pivot" is addressed by + # test_unnest_and_struct_access_233 below :) + + +@pytest.mark.skipif( + packaging.version.parse(sqlalchemy.__version__) < packaging.version.parse("1.4"), + reason="unnest (and other table-valued-function) support required version 1.4", +) +def test_unnest_and_struct_access_233(engine, bigquery_dataset, metadata): + # https://github.com/googleapis/python-bigquery-sqlalchemy/issues/233 + + from sqlalchemy import Table, select, Column, ARRAY, String, func + from sqlalchemy.orm import sessionmaker + from sqlalchemy_bigquery import STRUCT + + conn = engine.connect() + + mock_table = Table(f"{bigquery_dataset}.Mock", metadata, Column("mock_id", String)) + another_mock_table = Table( + f"{bigquery_dataset}.AnotherMock", + metadata, + Column("objects", ARRAY(STRUCT(object_id=String))), + ) + metadata.create_all(engine) + + conn.execute( + mock_table.insert(), dict(mock_id="x"), dict(mock_id="y"), dict(mock_id="z"), + ) + conn.execute( + another_mock_table.insert(), + dict(objects=[dict(object_id="x"), dict(object_id="y"), dict(object_id="q")]), + ) + + subquery = select( + func.unnest(another_mock_table.c.objects).alias("another_mock_objects").column + ).subquery() + + join = mock_table.join( + subquery, subquery.c.another_mock_objects["object_id"] == mock_table.c.mock_id, + ) + + query = select(mock_table).select_from(join) + + got = str(query.compile(engine)) + want = ( + f"SELECT `{bigquery_dataset}.Mock`.`mock_id` \n" + f"FROM `{bigquery_dataset}.Mock` " + f"JOIN (" + f"SELECT `another_mock_objects` \n" + f"FROM " + f"`{bigquery_dataset}.AnotherMock` `{bigquery_dataset}.AnotherMock_1`, " + f"unnest(`{bigquery_dataset}.AnotherMock_1`.`objects`)" + f" AS `another_mock_objects`" + f") AS `anon_1` " + f"ON " + f"(`anon_1`.`another_mock_objects`.object_id) = " + f"`{bigquery_dataset}.Mock`.`mock_id`" + ) + assert got == want + + Session = sessionmaker(bind=engine) + session = Session() + results = sorted(session.execute(query)) + + assert results == [("x",), ("y",)] diff --git a/tests/system/test_sqlalchemy_bigquery.py b/tests/system/test_sqlalchemy_bigquery.py index d8622020..564c5e68 100644 --- a/tests/system/test_sqlalchemy_bigquery.py +++ b/tests/system/test_sqlalchemy_bigquery.py @@ -18,9 +18,10 @@ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -*- coding: utf-8 -*- -from __future__ import unicode_literals -from sqlalchemy_bigquery import BigQueryDialect +import datetime +import decimal + from sqlalchemy.engine import create_engine from sqlalchemy.schema import Table, MetaData, Column from sqlalchemy.ext.declarative import declarative_base @@ -32,8 +33,8 @@ from pytz import timezone import pytest import sqlalchemy -import datetime -import decimal + +import sqlalchemy_bigquery ONE_ROW_CONTENTS_EXPANDED = [ 588, @@ -98,17 +99,24 @@ {"name": "bytes", "type": types.BINARY(), "nullable": True, "default": None}, { "name": "record", - "type": types.JSON(), + "type": sqlalchemy_bigquery.STRUCT(name=types.String, age=types.Integer), "nullable": True, "default": None, "comment": "In Standard SQL this data type is a STRUCT.", }, {"name": "record.name", "type": types.String(), "nullable": True, "default": None}, {"name": "record.age", "type": types.Integer(), "nullable": True, "default": None}, - {"name": "nested_record", "type": types.JSON(), "nullable": True, "default": None}, + { + "name": "nested_record", + "type": sqlalchemy_bigquery.STRUCT( + record=sqlalchemy_bigquery.STRUCT(name=types.String, age=types.Integer) + ), + "nullable": True, + "default": None, + }, { "name": "nested_record.record", - "type": types.JSON(), + "type": sqlalchemy_bigquery.STRUCT(name=types.String, age=types.Integer), "nullable": True, "default": None, }, @@ -133,17 +141,6 @@ ] -@pytest.fixture(scope="session") -def engine(): - engine = create_engine("bigquery://", echo=True) - return engine - - -@pytest.fixture(scope="session") -def dialect(): - return BigQueryDialect() - - @pytest.fixture(scope="session") def engine_using_test_dataset(bigquery_dataset): engine = create_engine(f"bigquery:///{bigquery_dataset}", echo=True) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 886e9aee..d311a134 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -42,6 +42,11 @@ ) +@pytest.fixture() +def engine(): + return sqlalchemy.create_engine("bigquery://myproject/mydataset") + + @pytest.fixture() def faux_conn(): test_data = dict(execute=[]) diff --git a/tests/unit/test__struct.py b/tests/unit/test__struct.py new file mode 100644 index 00000000..ee096fb5 --- /dev/null +++ b/tests/unit/test__struct.py @@ -0,0 +1,152 @@ +# Copyright (c) 2017 The sqlalchemy-bigquery Authors +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +import datetime + +import pytest + +import sqlalchemy + + +def _test_struct(): + from sqlalchemy_bigquery import STRUCT + + return STRUCT( + name=sqlalchemy.String, + children=sqlalchemy.ARRAY( + STRUCT(name=sqlalchemy.String, bdate=sqlalchemy.DATE) + ), + ) + + +def test_struct_colspec(): + assert _test_struct().get_col_spec() == ( + "STRUCT>>" + ) + + +def test_struct_repr(): + assert repr(_test_struct()) == ( + "STRUCT(name=String(), children=ARRAY(STRUCT(name=String(), bdate=DATE())))" + ) + + +def test_bind_processor(): + assert _test_struct().bind_processor(None) is dict + + +def _col(): + return sqlalchemy.Table( + "t", sqlalchemy.MetaData(), sqlalchemy.Column("person", _test_struct()), + ).c.person + + +@pytest.mark.parametrize( + "expr,sql", + [ + (_col()["name"], "`t`.`person`.name"), + (_col()["Name"], "`t`.`person`.Name"), + (_col().NAME, "`t`.`person`.NAME"), + (_col().children, "`t`.`person`.children"), + ( + # SQLAlchemy doesn't add the label in this case for some reason. + # TODO: why? + # https://github.com/googleapis/python-bigquery-sqlalchemy/issues/336 + _col().children[0].label("anon_1"), + "(`t`.`person`.children)[OFFSET(%(param_1:INT64)s)]", + ), + ( + _col().children[0]["bdate"], + "((`t`.`person`.children)[OFFSET(%(param_1:INT64)s)]).bdate", + ), + ( + _col().children[0].bdate, + "((`t`.`person`.children)[OFFSET(%(param_1:INT64)s)]).bdate", + ), + ], +) +def test_struct_traversal_project(engine, expr, sql): + sql = f"SELECT {sql} AS `anon_1` \nFROM `t`" + assert str(sqlalchemy.select([expr]).compile(engine)) == sql + + +@pytest.mark.parametrize( + "expr,sql", + [ + (_col()["name"] == "x", "(`t`.`person`.name) = %(param_1:STRING)s"), + (_col()["Name"] == "x", "(`t`.`person`.Name) = %(param_1:STRING)s"), + (_col().NAME == "x", "(`t`.`person`.NAME) = %(param_1:STRING)s"), + ( + _col().children[0] == dict(name="foo", bdate=datetime.date(2020, 1, 1)), + "(`t`.`person`.children)[OFFSET(%(param_1:INT64)s)]" + " = %(param_2:STRUCT)s", + ), + ( + _col().children[0] == dict(name="foo", bdate=datetime.date(2020, 1, 1)), + "(`t`.`person`.children)[OFFSET(%(param_1:INT64)s)]" + " = %(param_2:STRUCT)s", + ), + ( + _col().children[0]["bdate"] == datetime.date(2021, 8, 30), + "(((`t`.`person`.children)[OFFSET(%(param_1:INT64)s)]).bdate)" + " = %(param_2:DATE)s", + ), + ( + _col().children[0].bdate == datetime.date(2021, 8, 30), + "(((`t`.`person`.children)[OFFSET(%(param_1:INT64)s)]).bdate)" + " = %(param_2:DATE)s", + ), + ], +) +def test_struct_traversal_filter(engine, expr, sql, param=1): + want = f"SELECT `t`.`person` \nFROM `t`, `t` \nWHERE {sql}" + got = str(sqlalchemy.select([_col()]).where(expr).compile(engine)) + assert got == want + + +def test_struct_insert_type_info(engine, metadata): + t = sqlalchemy.Table("t", metadata, sqlalchemy.Column("person", _test_struct())) + got = str( + t.insert() + .values( + person=dict( + name="bob", + children=[dict(name="billy", bdate=datetime.date(2020, 1, 1))], + ) + ) + .compile(engine) + ) + + assert got == ( + "INSERT INTO `t` (`person`) VALUES (%(person:" + "STRUCT>>" + ")s)" + ) + + +def test_struct_non_string_field_access(engine): + with pytest.raises( + TypeError, + match="STRUCT fields can only be accessed with strings field names, not 42", + ): + _col()[42] + + +def test_struct_bad_name(engine): + with pytest.raises(KeyError, match="42"): + _col()["42"] diff --git a/tests/unit/test_catalog_functions.py b/tests/unit/test_catalog_functions.py index 6613ae57..fd7d0d63 100644 --- a/tests/unit/test_catalog_functions.py +++ b/tests/unit/test_catalog_functions.py @@ -165,8 +165,8 @@ def test_get_table_comment(faux_conn): ("STRING(42)", sqlalchemy.types.String(42), dict(max_length=42)), ("BYTES", sqlalchemy.types.BINARY(), ()), ("BYTES(42)", sqlalchemy.types.BINARY(42), dict(max_length=42)), - ("INT64", sqlalchemy.types.Integer, ()), - ("FLOAT64", sqlalchemy.types.Float, ()), + ("INT64", sqlalchemy.types.Integer(), ()), + ("FLOAT64", sqlalchemy.types.Float(), ()), ("NUMERIC", sqlalchemy.types.NUMERIC(), ()), ("NUMERIC(4)", sqlalchemy.types.NUMERIC(4), dict(precision=4)), ("NUMERIC(4, 2)", sqlalchemy.types.NUMERIC(4, 2), dict(precision=4, scale=2)), @@ -177,11 +177,11 @@ def test_get_table_comment(faux_conn): sqlalchemy.types.NUMERIC(42, 2), dict(precision=42, scale=2), ), - ("BOOL", sqlalchemy.types.Boolean, ()), - ("TIMESTAMP", sqlalchemy.types.TIMESTAMP, ()), - ("DATE", sqlalchemy.types.DATE, ()), - ("TIME", sqlalchemy.types.TIME, ()), - ("DATETIME", sqlalchemy.types.DATETIME, ()), + ("BOOL", sqlalchemy.types.Boolean(), ()), + ("TIMESTAMP", sqlalchemy.types.TIMESTAMP(), ()), + ("DATE", sqlalchemy.types.DATE(), ()), + ("TIME", sqlalchemy.types.TIME(), ()), + ("DATETIME", sqlalchemy.types.DATETIME(), ()), ("THURSDAY", sqlalchemy.types.NullType, ()), ], ) @@ -207,6 +207,8 @@ def test_get_table_columns(faux_conn, btype, atype, extra): def test_get_table_columns_special_cases(faux_conn): + from sqlalchemy_bigquery import STRUCT + cursor = faux_conn.connection.cursor() cursor.execute("create table foo (s STRING, n INT64 not null, r RECORD)") client = faux_conn.connection._client @@ -218,10 +220,10 @@ def test_get_table_columns_special_cases(faux_conn): ) actual = faux_conn.dialect.get_columns(faux_conn, "foo") - stype = actual[0].pop("type") - assert isinstance(stype, sqlalchemy.types.ARRAY) - assert isinstance(stype.item_type, sqlalchemy.types.String) - assert actual == [ + for a in actual: + a["type"] = repr(a["type"]) + + expected = [ { "comment": "a fine column", "default": None, @@ -230,13 +232,14 @@ def test_get_table_columns_special_cases(faux_conn): "max_length": None, "precision": None, "scale": None, + "type": repr(sqlalchemy.types.ARRAY(sqlalchemy.types.String())), }, { "comment": None, "default": None, "name": "n", "nullable": False, - "type": sqlalchemy.types.Integer, + "type": repr(sqlalchemy.types.Integer()), "max_length": None, "precision": None, "scale": None, @@ -246,7 +249,9 @@ def test_get_table_columns_special_cases(faux_conn): "default": None, "name": "r", "nullable": True, - "type": sqlalchemy.types.JSON, + "type": repr( + STRUCT(i=sqlalchemy.types.Integer(), f=sqlalchemy.types.Float()) + ), "max_length": None, "precision": None, "scale": None, @@ -256,7 +261,7 @@ def test_get_table_columns_special_cases(faux_conn): "default": None, "name": "r.i", "nullable": True, - "type": sqlalchemy.types.Integer, + "type": repr(sqlalchemy.types.Integer()), "max_length": None, "precision": None, "scale": None, @@ -266,12 +271,13 @@ def test_get_table_columns_special_cases(faux_conn): "default": None, "name": "r.f", "nullable": True, - "type": sqlalchemy.types.Float, + "type": repr(sqlalchemy.types.Float()), "max_length": None, "precision": None, "scale": None, }, ] + assert actual == expected def test_has_table(faux_conn): diff --git a/tests/unit/test_dialect_types.py b/tests/unit/test_dialect_types.py index a1af7c47..47ffd94a 100644 --- a/tests/unit/test_dialect_types.py +++ b/tests/unit/test_dialect_types.py @@ -24,7 +24,7 @@ def test_types_import(): """Demonstrate behavior of importing types independent of any other import.""" dialect_module = importlib.import_module("sqlalchemy_bigquery") - base_module = importlib.import_module("sqlalchemy_bigquery.base") - custom_types = getattr(base_module, "_type_map") + _types_module = importlib.import_module("sqlalchemy_bigquery._types") + custom_types = getattr(_types_module, "_type_map") for type_name, type_value in custom_types.items(): assert getattr(dialect_module, type_name) == type_value diff --git a/tests/unit/test_select.py b/tests/unit/test_select.py index 474fc9d9..641677a4 100644 --- a/tests/unit/test_select.py +++ b/tests/unit/test_select.py @@ -129,9 +129,6 @@ def test_typed_parameters(faux_conn, type_, val, btype, vrep): faux_conn.execute(table.insert().values(**{col_name: val})) - if btype.startswith("ARRAY<"): - btype = btype[6:-1] - ptype = btype[: btype.index("(")] if "(" in btype else btype assert faux_conn.test_data["execute"][-1] == ( @@ -173,8 +170,12 @@ def test_typed_parameters(faux_conn, type_, val, btype, vrep): ) -def test_select_json(faux_conn, metadata): - table = sqlalchemy.Table("t", metadata, sqlalchemy.Column("x", sqlalchemy.JSON)) +def test_select_struct(faux_conn, metadata): + from sqlalchemy_bigquery import STRUCT + + table = sqlalchemy.Table( + "t", metadata, sqlalchemy.Column("x", STRUCT(y=sqlalchemy.Integer)), + ) faux_conn.ex("create table t (x RECORD)") faux_conn.ex("""insert into t values ('{"y": 1}')""") @@ -430,3 +431,11 @@ def test_unnest_w_no_table_references(faux_conn, alias): assert " ".join(compiled.strip().split()) == ( "SELECT `anon_1` FROM unnest(%(unnest_1)s) AS `anon_1`" ) + + +def test_array_indexing(engine, metadata): + t = sqlalchemy.Table( + "t", metadata, sqlalchemy.Column("a", sqlalchemy.ARRAY(sqlalchemy.String)), + ) + got = str(sqlalchemy.select([t.c.a[0]]).compile(engine)) + assert got == "SELECT `t`.`a`[OFFSET(%(a_1:INT64)s)] AS `anon_1` \nFROM `t`" From 31ea69314809554c13566e977060d9fb255edbb5 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 9 Sep 2021 19:24:27 +0200 Subject: [PATCH 15/16] chore(deps): update all dependencies (#330) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![WhiteSource Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [Deprecated](https://togithub.com/tantale/deprecated) | `==1.2.12` -> `==1.2.13` | [![age](https://badges.renovateapi.com/packages/pypi/Deprecated/1.2.13/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/pypi/Deprecated/1.2.13/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/pypi/Deprecated/1.2.13/compatibility-slim/1.2.12)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/pypi/Deprecated/1.2.13/confidence-slim/1.2.12)](https://docs.renovatebot.com/merge-confidence/) | | [google-crc32c](https://togithub.com/googleapis/python-crc32c) | `==1.1.4` -> `==1.1.5` | [![age](https://badges.renovateapi.com/packages/pypi/google-crc32c/1.1.5/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/pypi/google-crc32c/1.1.5/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/pypi/google-crc32c/1.1.5/compatibility-slim/1.1.4)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/pypi/google-crc32c/1.1.5/confidence-slim/1.1.4)](https://docs.renovatebot.com/merge-confidence/) | | [google-resumable-media](https://togithub.com/googleapis/google-resumable-media-python) | `==2.0.1` -> `==2.0.2` | [![age](https://badges.renovateapi.com/packages/pypi/google-resumable-media/2.0.2/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/pypi/google-resumable-media/2.0.2/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/pypi/google-resumable-media/2.0.2/compatibility-slim/2.0.1)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/pypi/google-resumable-media/2.0.2/confidence-slim/2.0.1)](https://docs.renovatebot.com/merge-confidence/) | | [grpcio](https://grpc.io) | `==1.39.0` -> `==1.40.0` | [![age](https://badges.renovateapi.com/packages/pypi/grpcio/1.40.0/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/pypi/grpcio/1.40.0/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/pypi/grpcio/1.40.0/compatibility-slim/1.39.0)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/pypi/grpcio/1.40.0/confidence-slim/1.39.0)](https://docs.renovatebot.com/merge-confidence/) | --- ### Release Notes
tantale/deprecated ### [`v1.2.13`](https://togithub.com/tantale/deprecated/blob/master/CHANGELOG.rst#v1213-2021-09-05) [Compare Source](https://togithub.com/tantale/deprecated/compare/v1.2.12...v1.2.13) \==================== Bug fix release ## Fix - Fix [#​45](https://togithub.com/tantale/deprecated/issues/45): Change the signature of the :func:`~deprecated.sphinx.deprecated` decorator to reflect the valid use cases. - Fix [#​48](https://togithub.com/tantale/deprecated/issues/48): Fix `versionadded` and `versionchanged` decorators: do not return a decorator factory, but a Wrapt adapter. ## Other - Fix configuration for AppVeyor: simplify the test scripts and set the version format to match the current version. - Change configuration for Tox: - change the requirements for `pip` to "pip >= 9.0.3, < 21" (Python 2.7, 3.4 and 3.5). - install `typing` when building on Python 3.4 (required by Pytest->Attrs). - run unit tests on Wrapt 1.13 (release candidate). - Migrating project to `travis-ci.com `\_.
googleapis/python-crc32c ### [`v1.1.5`](https://togithub.com/googleapis/python-crc32c/blob/master/CHANGELOG.md#​115-httpswwwgithubcomgoogleapispython-crc32ccomparev114v115-2021-09-07) [Compare Source](https://togithub.com/googleapis/python-crc32c/compare/v1.1.4...v1.1.5)
googleapis/google-resumable-media-python ### [`v2.0.2`](https://togithub.com/googleapis/google-resumable-media-python/blob/master/CHANGELOG.md#​202-httpswwwgithubcomgoogleapisgoogle-resumable-media-pythoncomparev201v202-2021-09-02) [Compare Source](https://togithub.com/googleapis/google-resumable-media-python/compare/v2.0.1...v2.0.2)
--- ### Configuration 📅 **Schedule**: At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Renovate will not automatically rebase this PR, because other commits have been found. 👻 **Immortal**: This PR will be recreated if closed unmerged. Get [config help](https://togithub.com/renovatebot/renovate/discussions) if that's undesired. --- - [ ] If you want to rebase/retry this PR, check this box. --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/python-bigquery-sqlalchemy). --- samples/snippets/requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 29985c23..78b9ee50 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -9,7 +9,7 @@ click-plugins==1.1.1 cligj==0.7.2 contextvars==2.4 dataclasses==0.6; python_version < '3.7' -Deprecated==1.2.12 +Deprecated==1.2.13 Fiona==1.8.20 future==0.18.2 GeoAlchemy2==0.9.4 @@ -19,11 +19,11 @@ google-auth==2.0.2 google-cloud-bigquery==2.26.0 google-cloud-bigquery-storage==2.7.0 google-cloud-core==2.0.0 -google-crc32c==1.1.4 -google-resumable-media==2.0.1 +google-crc32c==1.1.2 +google-resumable-media==2.0.2 googleapis-common-protos==1.53.0 greenlet==1.1.1 -grpcio==1.39.0 +grpcio==1.40.0 idna==3.2 immutables==0.16 importlib-metadata==4.8.1 From f6d27990e782f0630a60bbbed48efc987e0bbdf3 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 9 Sep 2021 18:56:31 +0000 Subject: [PATCH 16/16] chore: release 1.2.0 (#338) :robot: I have created a release \*beep\* \*boop\* --- ## [1.2.0](https://www.github.com/googleapis/python-bigquery-sqlalchemy/compare/v1.1.0...v1.2.0) (2021-09-09) ### Features * STRUCT and ARRAY support ([#318](https://www.github.com/googleapis/python-bigquery-sqlalchemy/issues/318)) ([6624b10](https://www.github.com/googleapis/python-bigquery-sqlalchemy/commit/6624b10ded73bbca6f40af73aaeaceb95c381b63)) ### Bug Fixes * the unnest function lost needed type information ([#298](https://www.github.com/googleapis/python-bigquery-sqlalchemy/issues/298)) ([1233182](https://www.github.com/googleapis/python-bigquery-sqlalchemy/commit/123318269876e7f76c7f0f2daa5f5b365026cd3f)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- CHANGELOG.md | 12 ++++++++++++ sqlalchemy_bigquery/version.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c4d72a6..74b032b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,18 @@ Older versions of this project were distributed as [pybigquery][0]. [2]: https://pypi.org/project/pybigquery/#history +## [1.2.0](https://www.github.com/googleapis/python-bigquery-sqlalchemy/compare/v1.1.0...v1.2.0) (2021-09-09) + + +### Features + +* STRUCT and ARRAY support ([#318](https://www.github.com/googleapis/python-bigquery-sqlalchemy/issues/318)) ([6624b10](https://www.github.com/googleapis/python-bigquery-sqlalchemy/commit/6624b10ded73bbca6f40af73aaeaceb95c381b63)) + + +### Bug Fixes + +* the unnest function lost needed type information ([#298](https://www.github.com/googleapis/python-bigquery-sqlalchemy/issues/298)) ([1233182](https://www.github.com/googleapis/python-bigquery-sqlalchemy/commit/123318269876e7f76c7f0f2daa5f5b365026cd3f)) + ## [1.1.0](https://www.github.com/googleapis/python-bigquery-sqlalchemy/compare/v1.0.0...v1.1.0) (2021-08-25) diff --git a/sqlalchemy_bigquery/version.py b/sqlalchemy_bigquery/version.py index ef8460f5..f7a8338b 100644 --- a/sqlalchemy_bigquery/version.py +++ b/sqlalchemy_bigquery/version.py @@ -17,4 +17,4 @@ # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -__version__ = "1.1.0" +__version__ = "1.2.0"