From 8e4a7b4ecf7c03d4c7b8ce24480d004c903f2573 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 30 Mar 2022 04:54:59 -0500 Subject: [PATCH 1/9] chore: add branch protection rules for v2 (#1181) --- .github/sync-repo-settings.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index 73cc3bcef..220c031b2 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -1,6 +1,5 @@ # https://github.com/googleapis/repo-automation-bots/tree/main/packages/sync-repo-settings -# Allow merge commits to sync main and v3 with fewer conflicts. -mergeCommitAllowed: true +mergeCommitAllowed: false # Rules for main branch protection branchProtectionRules: # Identifies the protection rule pattern. Name of the branch to be protected. @@ -16,8 +15,8 @@ branchProtectionRules: - 'Samples - Lint' - 'Samples - Python 3.7' - 'Samples - Python 3.8' -- pattern: v3 - requiresLinearHistory: false +- pattern: v2 + requiresLinearHistory: true requiresCodeOwnerReviews: true requiresStrictStatusChecks: true requiredStatusCheckContexts: From b18c8fd24d2b68934478fde3841f6551e2f404b4 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 30 Mar 2022 15:03:58 +0200 Subject: [PATCH 2/9] chore(deps): update all dependencies (#1164) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore(deps): update all dependencies * revert * revert * revert * revert * revert * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Anthonios Partheniou Co-authored-by: Owl Bot --- samples/geography/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index fed8be7f9..54d0985cb 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -10,13 +10,13 @@ db-dtypes==0.4.0 Fiona==1.8.21 geojson==2.5.0 geopandas==0.10.2 -google-api-core==2.6.1 +google-api-core==2.7.0 google-auth==2.6.0 google-cloud-bigquery==2.34.2 google-cloud-bigquery-storage==2.12.0 google-cloud-core==2.2.3 google-crc32c==1.3.0 -google-resumable-media==2.3.1 +google-resumable-media==2.3.2 googleapis-common-protos==1.55.0 grpcio==1.44.0 idna==3.3 From fe16ec06f8d90caa575ef82e82961b2ed199dad2 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 30 Mar 2022 16:07:32 +0200 Subject: [PATCH 3/9] chore(deps): update all dependencies (#1183) * chore(deps): update all dependencies * revert environment specific pins * fix typo * revert environment specific pins * revert environment specific pins Co-authored-by: Anthonios Partheniou --- samples/geography/requirements-test.txt | 2 +- samples/geography/requirements.txt | 16 ++++++++-------- samples/magics/requirements-test.txt | 2 +- samples/magics/requirements.txt | 12 ++++++------ samples/snippets/requirements-test.txt | 2 +- samples/snippets/requirements.txt | 12 ++++++------ 6 files changed, 23 insertions(+), 23 deletions(-) diff --git a/samples/geography/requirements-test.txt b/samples/geography/requirements-test.txt index 4bd417eba..5e29de931 100644 --- a/samples/geography/requirements-test.txt +++ b/samples/geography/requirements-test.txt @@ -1,2 +1,2 @@ -pytest==7.0.1 +pytest==7.1.1 mock==4.0.3 diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index 54d0985cb..3ad2b7722 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -2,22 +2,22 @@ attrs==21.4.0 certifi==2021.10.8 cffi==1.15.0 charset-normalizer==2.0.12 -click==8.0.4 +click==8.1.0 click-plugins==1.1.1 cligj==0.7.2 dataclasses==0.8; python_version < '3.7' -db-dtypes==0.4.0 +db-dtypes==1.0.0 Fiona==1.8.21 geojson==2.5.0 geopandas==0.10.2 -google-api-core==2.7.0 -google-auth==2.6.0 +google-api-core==2.7.1 +google-auth==2.6.2 google-cloud-bigquery==2.34.2 -google-cloud-bigquery-storage==2.12.0 +google-cloud-bigquery-storage==2.13.0 google-cloud-core==2.2.3 google-crc32c==1.3.0 google-resumable-media==2.3.2 -googleapis-common-protos==1.55.0 +googleapis-common-protos==1.56.0 grpcio==1.44.0 idna==3.3 libcst==0.4.1 @@ -34,7 +34,7 @@ pyasn1-modules==0.2.8 pycparser==2.21 pyparsing==3.0.7 python-dateutil==2.8.2 -pytz==2021.3 +pytz==2022.1 PyYAML==6.0 requests==2.27.1 rsa==4.8 @@ -42,4 +42,4 @@ Shapely==1.8.1.post1 six==1.16.0 typing-extensions==4.1.1 typing-inspect==0.7.1 -urllib3==1.26.8 +urllib3==1.26.9 diff --git a/samples/magics/requirements-test.txt b/samples/magics/requirements-test.txt index bafc3de2a..c5864d4f7 100644 --- a/samples/magics/requirements-test.txt +++ b/samples/magics/requirements-test.txt @@ -1,3 +1,3 @@ google-cloud-testutils==1.3.1 -pytest==7.0.1 +pytest==7.1.1 mock==4.0.3 diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt index 5c54ecd83..1516125ff 100644 --- a/samples/magics/requirements.txt +++ b/samples/magics/requirements.txt @@ -1,13 +1,13 @@ -db-dtypes==0.4.0 -google-cloud-bigquery-storage==2.12.0 -google-auth-oauthlib==0.5.0 +db-dtypes==1.0.0 +google-cloud-bigquery-storage==2.13.0 +google-auth-oauthlib==0.5.1 grpcio==1.44.0 ipython==7.31.1; python_version == '3.7' ipython==8.0.1; python_version == '3.8' -ipython==8.1.1; python_version >= '3.9' +ipython==8.2.0; python_version >= '3.9' matplotlib==3.5.1 pandas==1.3.5; python_version == '3.7' pandas==1.4.1; python_version >= '3.8' pyarrow==7.0.0 -pytz==2021.3 -typing-extensions==3.10.0.2 +pytz==2022.1 +typing-extensions==4.1.1 diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index bafc3de2a..c5864d4f7 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1,3 +1,3 @@ google-cloud-testutils==1.3.1 -pytest==7.0.1 +pytest==7.1.1 mock==4.0.3 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 5c54ecd83..1516125ff 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,13 +1,13 @@ -db-dtypes==0.4.0 -google-cloud-bigquery-storage==2.12.0 -google-auth-oauthlib==0.5.0 +db-dtypes==1.0.0 +google-cloud-bigquery-storage==2.13.0 +google-auth-oauthlib==0.5.1 grpcio==1.44.0 ipython==7.31.1; python_version == '3.7' ipython==8.0.1; python_version == '3.8' -ipython==8.1.1; python_version >= '3.9' +ipython==8.2.0; python_version >= '3.9' matplotlib==3.5.1 pandas==1.3.5; python_version == '3.7' pandas==1.4.1; python_version >= '3.8' pyarrow==7.0.0 -pytz==2021.3 -typing-extensions==3.10.0.2 +pytz==2022.1 +typing-extensions==4.1.1 From 346fbc42a4cb047b1bf1ccdff31e116fb38f01d9 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 30 Mar 2022 17:03:44 +0200 Subject: [PATCH 4/9] chore: use === in requirements.txt for environment specific pins (#1184) * chore(deps): update all dependencies * chore(deps): use === in requirements.txt for environment specific pins Co-authored-by: Anthonios Partheniou --- samples/geography/requirements.txt | 2 +- samples/magics/requirements.txt | 6 +++--- samples/snippets/requirements.txt | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index 3ad2b7722..0347ea61d 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -24,7 +24,7 @@ libcst==0.4.1 munch==2.5.0 mypy-extensions==0.4.3 packaging==21.3 -pandas==1.3.5; python_version == '3.7' +pandas==1.3.5; python_version === '3.7' pandas==1.4.1; python_version >= '3.8' proto-plus==1.20.3 protobuf==3.19.4 diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt index 1516125ff..da15b025b 100644 --- a/samples/magics/requirements.txt +++ b/samples/magics/requirements.txt @@ -2,11 +2,11 @@ db-dtypes==1.0.0 google-cloud-bigquery-storage==2.13.0 google-auth-oauthlib==0.5.1 grpcio==1.44.0 -ipython==7.31.1; python_version == '3.7' -ipython==8.0.1; python_version == '3.8' +ipython==7.31.1; python_version === '3.7' +ipython==8.0.1; python_version === '3.8' ipython==8.2.0; python_version >= '3.9' matplotlib==3.5.1 -pandas==1.3.5; python_version == '3.7' +pandas==1.3.5; python_version === '3.7' pandas==1.4.1; python_version >= '3.8' pyarrow==7.0.0 pytz==2022.1 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 1516125ff..da15b025b 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -2,11 +2,11 @@ db-dtypes==1.0.0 google-cloud-bigquery-storage==2.13.0 google-auth-oauthlib==0.5.1 grpcio==1.44.0 -ipython==7.31.1; python_version == '3.7' -ipython==8.0.1; python_version == '3.8' +ipython==7.31.1; python_version === '3.7' +ipython==8.0.1; python_version === '3.8' ipython==8.2.0; python_version >= '3.9' matplotlib==3.5.1 -pandas==1.3.5; python_version == '3.7' +pandas==1.3.5; python_version === '3.7' pandas==1.4.1; python_version >= '3.8' pyarrow==7.0.0 pytz==2022.1 From 212d7ec1f0740d04c26fb3ceffc9a4dd9eed6756 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Wed, 30 Mar 2022 11:41:44 -0400 Subject: [PATCH 5/9] fix(deps): restore dependency on python-dateutil (#1187) * fix(deps): restore dependency on python-dateutil * fix(deps): remove redundante proto-plus Co-authored-by: Tim Swast --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 62fb3bbb3..86eb2d41d 100644 --- a/setup.py +++ b/setup.py @@ -42,8 +42,8 @@ "google-cloud-core >= 1.4.1, <3.0.0dev", "google-resumable-media >= 0.6.0, < 3.0dev", "packaging >= 14.3", - "proto-plus >= 1.10.0", # For the legacy proto-based types. "protobuf >= 3.12.0", # For the legacy proto-based types. + "python-dateutil >= 2.7.2, <3.0dev", "pyarrow >= 3.0.0, < 8.0dev", "requests >= 2.18.0, < 3.0.0dev", ] From 65fc7333cbe8f67cc883e9e99f79125037397d4e Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 30 Mar 2022 18:35:47 +0200 Subject: [PATCH 6/9] chore(deps): update all dependencies (#1185) * chore(deps): update all dependencies * chore: revert version bump for environment specific pins Co-authored-by: Anthonios Partheniou --- samples/geography/requirements.txt | 4 ++-- samples/magics/requirements.txt | 6 +++--- samples/snippets/requirements.txt | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index 0347ea61d..1b1b008e2 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -12,7 +12,7 @@ geojson==2.5.0 geopandas==0.10.2 google-api-core==2.7.1 google-auth==2.6.2 -google-cloud-bigquery==2.34.2 +google-cloud-bigquery==3.0.0 google-cloud-bigquery-storage==2.13.0 google-cloud-core==2.2.3 google-crc32c==1.3.0 @@ -24,7 +24,7 @@ libcst==0.4.1 munch==2.5.0 mypy-extensions==0.4.3 packaging==21.3 -pandas==1.3.5; python_version === '3.7' +pandas===1.3.5; python_version == '3.7' pandas==1.4.1; python_version >= '3.8' proto-plus==1.20.3 protobuf==3.19.4 diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt index da15b025b..94ce22b00 100644 --- a/samples/magics/requirements.txt +++ b/samples/magics/requirements.txt @@ -2,11 +2,11 @@ db-dtypes==1.0.0 google-cloud-bigquery-storage==2.13.0 google-auth-oauthlib==0.5.1 grpcio==1.44.0 -ipython==7.31.1; python_version === '3.7' -ipython==8.0.1; python_version === '3.8' +ipython===7.31.1; python_version == '3.7' +ipython===8.0.1; python_version == '3.8' ipython==8.2.0; python_version >= '3.9' matplotlib==3.5.1 -pandas==1.3.5; python_version === '3.7' +pandas===1.3.5; python_version == '3.7' pandas==1.4.1; python_version >= '3.8' pyarrow==7.0.0 pytz==2022.1 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index da15b025b..94ce22b00 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -2,11 +2,11 @@ db-dtypes==1.0.0 google-cloud-bigquery-storage==2.13.0 google-auth-oauthlib==0.5.1 grpcio==1.44.0 -ipython==7.31.1; python_version === '3.7' -ipython==8.0.1; python_version === '3.8' +ipython===7.31.1; python_version == '3.7' +ipython===8.0.1; python_version == '3.8' ipython==8.2.0; python_version >= '3.9' matplotlib==3.5.1 -pandas==1.3.5; python_version === '3.7' +pandas===1.3.5; python_version == '3.7' pandas==1.4.1; python_version >= '3.8' pyarrow==7.0.0 pytz==2022.1 From b4c7f5ac17dc9273bfc260396552138d12229871 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 30 Mar 2022 18:24:13 +0000 Subject: [PATCH 7/9] chore(python): add E231 to .flake8 ignore list (#1190) Source-Link: https://github.com/googleapis/synthtool/commit/7ff4aad2ec5af0380e8bd6da1fa06eaadf24ec81 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:462782b0b492346b2d9099aaff52206dd30bc8e031ea97082e6facecc2373244 --- .flake8 | 2 +- .github/.OwlBot.lock.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.flake8 b/.flake8 index 29227d4cf..2e4387498 100644 --- a/.flake8 +++ b/.flake8 @@ -16,7 +16,7 @@ # Generated by synthtool. DO NOT EDIT! [flake8] -ignore = E203, E266, E501, W503 +ignore = E203, E231, E266, E501, W503 exclude = # Exclude generated code. **/proto/** diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 87dd00611..9e0a9356b 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:7cffbc10910c3ab1b852c05114a08d374c195a81cdec1d4a67a1d129331d0bfe + digest: sha256:462782b0b492346b2d9099aaff52206dd30bc8e031ea97082e6facecc2373244 From 433391097bae57dd12a93db18fc2bab573d8f128 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 30 Mar 2022 14:20:16 -0500 Subject: [PATCH 8/9] fix(deps): raise exception when pandas is installed but db-dtypes is not (#1191) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `db-dtypes` is already present in the `pandas` "extras", but this PR ensures that if pandas is present and db-dtypes is not, a more understandable error message is raised. ``` google/cloud/bigquery/_pandas_helpers.py:991: ValueError ____________________________________ test_list_rows_nullable_scalars_extreme_dtypes[10] _____________________________________ # Copyright 2019 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Shared helper functions for connecting BigQuery and pandas.""" import concurrent.futures from datetime import datetime import functools from itertools import islice import logging import queue import warnings try: import pandas # type: ignore pandas_import_exception = None except ImportError as exc: # pragma: NO COVER pandas = None pandas_import_exception = exc else: import numpy try: > import db_dtypes # type: ignore E ModuleNotFoundError: No module named 'db_dtypes' google/cloud/bigquery/_pandas_helpers.py:36: ModuleNotFoundError The above exception was the direct cause of the following exception: bigquery_client = scalars_extreme_table = 'swast-scratch.python_bigquery_tests_system_20220330160830_ffff89.scalars_extreme_jsonl0x3ffeb' max_results = 10 @pytest.mark.parametrize( ("max_results",), ( (None,), (10,), ), # Use BQ Storage API. # Use REST API. ) def test_list_rows_nullable_scalars_extreme_dtypes( bigquery_client, scalars_extreme_table, max_results ): # TODO(GH#836): Avoid INTERVAL columns until they are supported by the # BigQuery Storage API and pyarrow. schema = [ bigquery.SchemaField("bool_col", enums.SqlTypeNames.BOOLEAN), bigquery.SchemaField("bignumeric_col", enums.SqlTypeNames.BIGNUMERIC), bigquery.SchemaField("bytes_col", enums.SqlTypeNames.BYTES), bigquery.SchemaField("date_col", enums.SqlTypeNames.DATE), bigquery.SchemaField("datetime_col", enums.SqlTypeNames.DATETIME), bigquery.SchemaField("float64_col", enums.SqlTypeNames.FLOAT64), bigquery.SchemaField("geography_col", enums.SqlTypeNames.GEOGRAPHY), bigquery.SchemaField("int64_col", enums.SqlTypeNames.INT64), bigquery.SchemaField("numeric_col", enums.SqlTypeNames.NUMERIC), bigquery.SchemaField("string_col", enums.SqlTypeNames.STRING), bigquery.SchemaField("time_col", enums.SqlTypeNames.TIME), bigquery.SchemaField("timestamp_col", enums.SqlTypeNames.TIMESTAMP), ] df = bigquery_client.list_rows( scalars_extreme_table, max_results=max_results, selected_fields=schema, > ).to_dataframe() tests/system/test_pandas.py:1084: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ google/cloud/bigquery/table.py:1925: in to_dataframe _pandas_helpers.verify_pandas_imports() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ def verify_pandas_imports(): if pandas is None: raise ValueError(_NO_PANDAS_ERROR) from pandas_import_exception if db_dtypes is None: > raise ValueError(_NO_DB_TYPES_ERROR) from db_dtypes_import_exception E ValueError: Please install the 'db-dtypes' package to use this function. google/cloud/bigquery/_pandas_helpers.py:991: ValueError ``` Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-bigquery/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) Fixes #1188 🦕 --- google/cloud/bigquery/_pandas_helpers.py | 33 +++++++++++++++++++----- google/cloud/bigquery/table.py | 20 +++++--------- tests/unit/test__pandas_helpers.py | 13 ++++++++++ tests/unit/test_table.py | 8 +++--- 4 files changed, 49 insertions(+), 25 deletions(-) diff --git a/google/cloud/bigquery/_pandas_helpers.py b/google/cloud/bigquery/_pandas_helpers.py index 17de6830a..cc0ee75ff 100644 --- a/google/cloud/bigquery/_pandas_helpers.py +++ b/google/cloud/bigquery/_pandas_helpers.py @@ -24,16 +24,25 @@ try: import pandas # type: ignore -except ImportError: # pragma: NO COVER + + pandas_import_exception = None +except ImportError as exc: # pragma: NO COVER pandas = None - date_dtype_name = time_dtype_name = "" # Use '' rather than None because pytype + pandas_import_exception = exc else: import numpy - from db_dtypes import DateDtype, TimeDtype # type: ignore +try: + import db_dtypes # type: ignore + + date_dtype_name = db_dtypes.DateDtype.name + time_dtype_name = db_dtypes.TimeDtype.name + db_dtypes_import_exception = None +except ImportError as exc: # pragma: NO COVER + db_dtypes = None + db_dtypes_import_exception = exc + date_dtype_name = time_dtype_name = "" # Use '' rather than None because pytype - date_dtype_name = DateDtype.name - time_dtype_name = TimeDtype.name import pyarrow # type: ignore import pyarrow.parquet # type: ignore @@ -84,6 +93,9 @@ def _to_wkb(v): _MAX_QUEUE_SIZE_DEFAULT = object() # max queue size sentinel for BQ Storage downloads +_NO_PANDAS_ERROR = "Please install the 'pandas' package to use this function." +_NO_DB_TYPES_ERROR = "Please install the 'db-dtypes' package to use this function." + _PANDAS_DTYPE_TO_BQ = { "bool": "BOOLEAN", "datetime64[ns, UTC]": "TIMESTAMP", @@ -290,13 +302,13 @@ def types_mapper(arrow_data_type): not date_as_object and pyarrow.types.is_date(arrow_data_type) ): - return DateDtype() + return db_dtypes.DateDtype() elif pyarrow.types.is_integer(arrow_data_type): return pandas.Int64Dtype() elif pyarrow.types.is_time(arrow_data_type): - return TimeDtype() + return db_dtypes.TimeDtype() return types_mapper @@ -970,3 +982,10 @@ def dataframe_to_json_generator(dataframe): output[column] = value yield output + + +def verify_pandas_imports(): + if pandas is None: + raise ValueError(_NO_PANDAS_ERROR) from pandas_import_exception + if db_dtypes is None: + raise ValueError(_NO_DB_TYPES_ERROR) from db_dtypes_import_exception diff --git a/google/cloud/bigquery/table.py b/google/cloud/bigquery/table.py index ed4f214ce..5a4de6a01 100644 --- a/google/cloud/bigquery/table.py +++ b/google/cloud/bigquery/table.py @@ -28,8 +28,6 @@ import pandas # type: ignore except ImportError: # pragma: NO COVER pandas = None -else: - import db_dtypes # type: ignore # noqa import pyarrow # type: ignore @@ -69,10 +67,6 @@ from google.cloud.bigquery.dataset import DatasetReference -_NO_PANDAS_ERROR = ( - "The pandas library is not installed, please install " - "pandas to use the to_dataframe() function." -) _NO_GEOPANDAS_ERROR = ( "The geopandas library is not installed, please install " "geopandas to use the to_geodataframe() function." @@ -1818,8 +1812,8 @@ def to_dataframe_iterable( ValueError: If the :mod:`pandas` library cannot be imported. """ - if pandas is None: - raise ValueError(_NO_PANDAS_ERROR) + _pandas_helpers.verify_pandas_imports() + if dtypes is None: dtypes = {} @@ -1928,8 +1922,8 @@ def to_dataframe( :mod:`shapely` library cannot be imported. """ - if pandas is None: - raise ValueError(_NO_PANDAS_ERROR) + _pandas_helpers.verify_pandas_imports() + if geography_as_object and shapely is None: raise ValueError(_NO_SHAPELY_ERROR) @@ -2181,8 +2175,7 @@ def to_dataframe( Returns: pandas.DataFrame: An empty :class:`~pandas.DataFrame`. """ - if pandas is None: - raise ValueError(_NO_PANDAS_ERROR) + _pandas_helpers.verify_pandas_imports() return pandas.DataFrame() def to_geodataframe( @@ -2238,8 +2231,7 @@ def to_dataframe_iterable( ValueError: If the :mod:`pandas` library cannot be imported. """ - if pandas is None: - raise ValueError(_NO_PANDAS_ERROR) + _pandas_helpers.verify_pandas_imports() return iter((pandas.DataFrame(),)) def to_arrow_iterable( diff --git a/tests/unit/test__pandas_helpers.py b/tests/unit/test__pandas_helpers.py index 5b2fadaf1..1a3f918eb 100644 --- a/tests/unit/test__pandas_helpers.py +++ b/tests/unit/test__pandas_helpers.py @@ -1751,3 +1751,16 @@ def test_bq_to_arrow_field_metadata(module_under_test, field_type, metadata): ).metadata == metadata ) + + +def test_verify_pandas_imports_no_pandas(module_under_test, monkeypatch): + monkeypatch.setattr(module_under_test, "pandas", None) + with pytest.raises(ValueError, match="Please install the 'pandas' package"): + module_under_test.verify_pandas_imports() + + +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") +def test_verify_pandas_imports_no_db_dtypes(module_under_test, monkeypatch): + monkeypatch.setattr(module_under_test, "db_dtypes", None) + with pytest.raises(ValueError, match="Please install the 'db-dtypes' package"): + module_under_test.verify_pandas_imports() diff --git a/tests/unit/test_table.py b/tests/unit/test_table.py index 5241230a4..66bc1d3db 100644 --- a/tests/unit/test_table.py +++ b/tests/unit/test_table.py @@ -1836,7 +1836,7 @@ def test_to_arrow_iterable(self): self.assertEqual(record_batch.num_rows, 0) self.assertEqual(record_batch.num_columns, 0) - @mock.patch("google.cloud.bigquery.table.pandas", new=None) + @mock.patch("google.cloud.bigquery._pandas_helpers.pandas", new=None) def test_to_dataframe_error_if_pandas_is_none(self): row_iterator = self._make_one() with self.assertRaises(ValueError): @@ -1849,7 +1849,7 @@ def test_to_dataframe(self): self.assertIsInstance(df, pandas.DataFrame) self.assertEqual(len(df), 0) # verify the number of rows - @mock.patch("google.cloud.bigquery.table.pandas", new=None) + @mock.patch("google.cloud.bigquery._pandas_helpers.pandas", new=None) def test_to_dataframe_iterable_error_if_pandas_is_none(self): row_iterator = self._make_one() with self.assertRaises(ValueError): @@ -2967,7 +2967,7 @@ def test_to_dataframe_iterable_w_bqstorage_max_results_warning(self): assert isinstance(dataframes[0], pandas.DataFrame) assert isinstance(dataframes[1], pandas.DataFrame) - @mock.patch("google.cloud.bigquery.table.pandas", new=None) + @mock.patch("google.cloud.bigquery._pandas_helpers.pandas", new=None) def test_to_dataframe_iterable_error_if_pandas_is_none(self): from google.cloud.bigquery.schema import SchemaField @@ -3339,7 +3339,7 @@ def test_to_dataframe_datetime_objects(self): self.assertEqual(df["ts"][0].date(), datetime.date(1336, 3, 23)) self.assertEqual(df["date"][0], datetime.date(1111, 1, 1)) - @mock.patch("google.cloud.bigquery.table.pandas", new=None) + @mock.patch("google.cloud.bigquery._pandas_helpers.pandas", new=None) def test_to_dataframe_error_if_pandas_is_none(self): from google.cloud.bigquery.schema import SchemaField From c3740acd2c1363e32d56884974a30c4fbc09bd72 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Wed, 30 Mar 2022 19:48:13 +0000 Subject: [PATCH 9/9] chore(main): release 3.0.1 (#1189) :robot: I have created a release *beep* *boop* --- ### [3.0.1](https://github.com/googleapis/python-bigquery/compare/v3.0.0...v3.0.1) (2022-03-30) ### Bug Fixes * **deps:** raise exception when pandas is installed but db-dtypes is not ([#1191](https://github.com/googleapis/python-bigquery/issues/1191)) ([4333910](https://github.com/googleapis/python-bigquery/commit/433391097bae57dd12a93db18fc2bab573d8f128)) * **deps:** restore dependency on python-dateutil ([#1187](https://github.com/googleapis/python-bigquery/issues/1187)) ([212d7ec](https://github.com/googleapis/python-bigquery/commit/212d7ec1f0740d04c26fb3ceffc9a4dd9eed6756)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- CHANGELOG.md | 8 ++++++++ google/cloud/bigquery/version.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5c8447da..ca99c969f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ [1]: https://pypi.org/project/google-cloud-bigquery/#history +### [3.0.1](https://github.com/googleapis/python-bigquery/compare/v3.0.0...v3.0.1) (2022-03-30) + + +### Bug Fixes + +* **deps:** raise exception when pandas is installed but db-dtypes is not ([#1191](https://github.com/googleapis/python-bigquery/issues/1191)) ([4333910](https://github.com/googleapis/python-bigquery/commit/433391097bae57dd12a93db18fc2bab573d8f128)) +* **deps:** restore dependency on python-dateutil ([#1187](https://github.com/googleapis/python-bigquery/issues/1187)) ([212d7ec](https://github.com/googleapis/python-bigquery/commit/212d7ec1f0740d04c26fb3ceffc9a4dd9eed6756)) + ## [3.0.0](https://github.com/googleapis/python-bigquery/compare/v2.34.3...v3.0.0) (2022-03-29) diff --git a/google/cloud/bigquery/version.py b/google/cloud/bigquery/version.py index d6f7def8c..ad3213664 100644 --- a/google/cloud/bigquery/version.py +++ b/google/cloud/bigquery/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "3.0.0" +__version__ = "3.0.1"