diff --git a/.flake8 b/.flake8 index 29227d4cf..2e4387498 100644 --- a/.flake8 +++ b/.flake8 @@ -16,7 +16,7 @@ # Generated by synthtool. DO NOT EDIT! [flake8] -ignore = E203, E266, E501, W503 +ignore = E203, E231, E266, E501, W503 exclude = # Exclude generated code. **/proto/** diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 87dd00611..9e0a9356b 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,4 +13,4 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:7cffbc10910c3ab1b852c05114a08d374c195a81cdec1d4a67a1d129331d0bfe + digest: sha256:462782b0b492346b2d9099aaff52206dd30bc8e031ea97082e6facecc2373244 diff --git a/.github/sync-repo-settings.yaml b/.github/sync-repo-settings.yaml index 73cc3bcef..220c031b2 100644 --- a/.github/sync-repo-settings.yaml +++ b/.github/sync-repo-settings.yaml @@ -1,6 +1,5 @@ # https://github.com/googleapis/repo-automation-bots/tree/main/packages/sync-repo-settings -# Allow merge commits to sync main and v3 with fewer conflicts. -mergeCommitAllowed: true +mergeCommitAllowed: false # Rules for main branch protection branchProtectionRules: # Identifies the protection rule pattern. Name of the branch to be protected. @@ -16,8 +15,8 @@ branchProtectionRules: - 'Samples - Lint' - 'Samples - Python 3.7' - 'Samples - Python 3.8' -- pattern: v3 - requiresLinearHistory: false +- pattern: v2 + requiresLinearHistory: true requiresCodeOwnerReviews: true requiresStrictStatusChecks: true requiredStatusCheckContexts: diff --git a/CHANGELOG.md b/CHANGELOG.md index c5c8447da..ca99c969f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,14 @@ [1]: https://pypi.org/project/google-cloud-bigquery/#history +### [3.0.1](https://github.com/googleapis/python-bigquery/compare/v3.0.0...v3.0.1) (2022-03-30) + + +### Bug Fixes + +* **deps:** raise exception when pandas is installed but db-dtypes is not ([#1191](https://github.com/googleapis/python-bigquery/issues/1191)) ([4333910](https://github.com/googleapis/python-bigquery/commit/433391097bae57dd12a93db18fc2bab573d8f128)) +* **deps:** restore dependency on python-dateutil ([#1187](https://github.com/googleapis/python-bigquery/issues/1187)) ([212d7ec](https://github.com/googleapis/python-bigquery/commit/212d7ec1f0740d04c26fb3ceffc9a4dd9eed6756)) + ## [3.0.0](https://github.com/googleapis/python-bigquery/compare/v2.34.3...v3.0.0) (2022-03-29) diff --git a/google/cloud/bigquery/_pandas_helpers.py b/google/cloud/bigquery/_pandas_helpers.py index 17de6830a..cc0ee75ff 100644 --- a/google/cloud/bigquery/_pandas_helpers.py +++ b/google/cloud/bigquery/_pandas_helpers.py @@ -24,16 +24,25 @@ try: import pandas # type: ignore -except ImportError: # pragma: NO COVER + + pandas_import_exception = None +except ImportError as exc: # pragma: NO COVER pandas = None - date_dtype_name = time_dtype_name = "" # Use '' rather than None because pytype + pandas_import_exception = exc else: import numpy - from db_dtypes import DateDtype, TimeDtype # type: ignore +try: + import db_dtypes # type: ignore + + date_dtype_name = db_dtypes.DateDtype.name + time_dtype_name = db_dtypes.TimeDtype.name + db_dtypes_import_exception = None +except ImportError as exc: # pragma: NO COVER + db_dtypes = None + db_dtypes_import_exception = exc + date_dtype_name = time_dtype_name = "" # Use '' rather than None because pytype - date_dtype_name = DateDtype.name - time_dtype_name = TimeDtype.name import pyarrow # type: ignore import pyarrow.parquet # type: ignore @@ -84,6 +93,9 @@ def _to_wkb(v): _MAX_QUEUE_SIZE_DEFAULT = object() # max queue size sentinel for BQ Storage downloads +_NO_PANDAS_ERROR = "Please install the 'pandas' package to use this function." +_NO_DB_TYPES_ERROR = "Please install the 'db-dtypes' package to use this function." + _PANDAS_DTYPE_TO_BQ = { "bool": "BOOLEAN", "datetime64[ns, UTC]": "TIMESTAMP", @@ -290,13 +302,13 @@ def types_mapper(arrow_data_type): not date_as_object and pyarrow.types.is_date(arrow_data_type) ): - return DateDtype() + return db_dtypes.DateDtype() elif pyarrow.types.is_integer(arrow_data_type): return pandas.Int64Dtype() elif pyarrow.types.is_time(arrow_data_type): - return TimeDtype() + return db_dtypes.TimeDtype() return types_mapper @@ -970,3 +982,10 @@ def dataframe_to_json_generator(dataframe): output[column] = value yield output + + +def verify_pandas_imports(): + if pandas is None: + raise ValueError(_NO_PANDAS_ERROR) from pandas_import_exception + if db_dtypes is None: + raise ValueError(_NO_DB_TYPES_ERROR) from db_dtypes_import_exception diff --git a/google/cloud/bigquery/table.py b/google/cloud/bigquery/table.py index ed4f214ce..5a4de6a01 100644 --- a/google/cloud/bigquery/table.py +++ b/google/cloud/bigquery/table.py @@ -28,8 +28,6 @@ import pandas # type: ignore except ImportError: # pragma: NO COVER pandas = None -else: - import db_dtypes # type: ignore # noqa import pyarrow # type: ignore @@ -69,10 +67,6 @@ from google.cloud.bigquery.dataset import DatasetReference -_NO_PANDAS_ERROR = ( - "The pandas library is not installed, please install " - "pandas to use the to_dataframe() function." -) _NO_GEOPANDAS_ERROR = ( "The geopandas library is not installed, please install " "geopandas to use the to_geodataframe() function." @@ -1818,8 +1812,8 @@ def to_dataframe_iterable( ValueError: If the :mod:`pandas` library cannot be imported. """ - if pandas is None: - raise ValueError(_NO_PANDAS_ERROR) + _pandas_helpers.verify_pandas_imports() + if dtypes is None: dtypes = {} @@ -1928,8 +1922,8 @@ def to_dataframe( :mod:`shapely` library cannot be imported. """ - if pandas is None: - raise ValueError(_NO_PANDAS_ERROR) + _pandas_helpers.verify_pandas_imports() + if geography_as_object and shapely is None: raise ValueError(_NO_SHAPELY_ERROR) @@ -2181,8 +2175,7 @@ def to_dataframe( Returns: pandas.DataFrame: An empty :class:`~pandas.DataFrame`. """ - if pandas is None: - raise ValueError(_NO_PANDAS_ERROR) + _pandas_helpers.verify_pandas_imports() return pandas.DataFrame() def to_geodataframe( @@ -2238,8 +2231,7 @@ def to_dataframe_iterable( ValueError: If the :mod:`pandas` library cannot be imported. """ - if pandas is None: - raise ValueError(_NO_PANDAS_ERROR) + _pandas_helpers.verify_pandas_imports() return iter((pandas.DataFrame(),)) def to_arrow_iterable( diff --git a/google/cloud/bigquery/version.py b/google/cloud/bigquery/version.py index d6f7def8c..ad3213664 100644 --- a/google/cloud/bigquery/version.py +++ b/google/cloud/bigquery/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "3.0.0" +__version__ = "3.0.1" diff --git a/samples/geography/requirements-test.txt b/samples/geography/requirements-test.txt index 4bd417eba..5e29de931 100644 --- a/samples/geography/requirements-test.txt +++ b/samples/geography/requirements-test.txt @@ -1,2 +1,2 @@ -pytest==7.0.1 +pytest==7.1.1 mock==4.0.3 diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index fed8be7f9..1b1b008e2 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -2,29 +2,29 @@ attrs==21.4.0 certifi==2021.10.8 cffi==1.15.0 charset-normalizer==2.0.12 -click==8.0.4 +click==8.1.0 click-plugins==1.1.1 cligj==0.7.2 dataclasses==0.8; python_version < '3.7' -db-dtypes==0.4.0 +db-dtypes==1.0.0 Fiona==1.8.21 geojson==2.5.0 geopandas==0.10.2 -google-api-core==2.6.1 -google-auth==2.6.0 -google-cloud-bigquery==2.34.2 -google-cloud-bigquery-storage==2.12.0 +google-api-core==2.7.1 +google-auth==2.6.2 +google-cloud-bigquery==3.0.0 +google-cloud-bigquery-storage==2.13.0 google-cloud-core==2.2.3 google-crc32c==1.3.0 -google-resumable-media==2.3.1 -googleapis-common-protos==1.55.0 +google-resumable-media==2.3.2 +googleapis-common-protos==1.56.0 grpcio==1.44.0 idna==3.3 libcst==0.4.1 munch==2.5.0 mypy-extensions==0.4.3 packaging==21.3 -pandas==1.3.5; python_version == '3.7' +pandas===1.3.5; python_version == '3.7' pandas==1.4.1; python_version >= '3.8' proto-plus==1.20.3 protobuf==3.19.4 @@ -34,7 +34,7 @@ pyasn1-modules==0.2.8 pycparser==2.21 pyparsing==3.0.7 python-dateutil==2.8.2 -pytz==2021.3 +pytz==2022.1 PyYAML==6.0 requests==2.27.1 rsa==4.8 @@ -42,4 +42,4 @@ Shapely==1.8.1.post1 six==1.16.0 typing-extensions==4.1.1 typing-inspect==0.7.1 -urllib3==1.26.8 +urllib3==1.26.9 diff --git a/samples/magics/requirements-test.txt b/samples/magics/requirements-test.txt index bafc3de2a..c5864d4f7 100644 --- a/samples/magics/requirements-test.txt +++ b/samples/magics/requirements-test.txt @@ -1,3 +1,3 @@ google-cloud-testutils==1.3.1 -pytest==7.0.1 +pytest==7.1.1 mock==4.0.3 diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt index 5c54ecd83..94ce22b00 100644 --- a/samples/magics/requirements.txt +++ b/samples/magics/requirements.txt @@ -1,13 +1,13 @@ -db-dtypes==0.4.0 -google-cloud-bigquery-storage==2.12.0 -google-auth-oauthlib==0.5.0 +db-dtypes==1.0.0 +google-cloud-bigquery-storage==2.13.0 +google-auth-oauthlib==0.5.1 grpcio==1.44.0 -ipython==7.31.1; python_version == '3.7' -ipython==8.0.1; python_version == '3.8' -ipython==8.1.1; python_version >= '3.9' +ipython===7.31.1; python_version == '3.7' +ipython===8.0.1; python_version == '3.8' +ipython==8.2.0; python_version >= '3.9' matplotlib==3.5.1 -pandas==1.3.5; python_version == '3.7' +pandas===1.3.5; python_version == '3.7' pandas==1.4.1; python_version >= '3.8' pyarrow==7.0.0 -pytz==2021.3 -typing-extensions==3.10.0.2 +pytz==2022.1 +typing-extensions==4.1.1 diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index bafc3de2a..c5864d4f7 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1,3 +1,3 @@ google-cloud-testutils==1.3.1 -pytest==7.0.1 +pytest==7.1.1 mock==4.0.3 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 5c54ecd83..94ce22b00 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,13 +1,13 @@ -db-dtypes==0.4.0 -google-cloud-bigquery-storage==2.12.0 -google-auth-oauthlib==0.5.0 +db-dtypes==1.0.0 +google-cloud-bigquery-storage==2.13.0 +google-auth-oauthlib==0.5.1 grpcio==1.44.0 -ipython==7.31.1; python_version == '3.7' -ipython==8.0.1; python_version == '3.8' -ipython==8.1.1; python_version >= '3.9' +ipython===7.31.1; python_version == '3.7' +ipython===8.0.1; python_version == '3.8' +ipython==8.2.0; python_version >= '3.9' matplotlib==3.5.1 -pandas==1.3.5; python_version == '3.7' +pandas===1.3.5; python_version == '3.7' pandas==1.4.1; python_version >= '3.8' pyarrow==7.0.0 -pytz==2021.3 -typing-extensions==3.10.0.2 +pytz==2022.1 +typing-extensions==4.1.1 diff --git a/setup.py b/setup.py index 62fb3bbb3..86eb2d41d 100644 --- a/setup.py +++ b/setup.py @@ -42,8 +42,8 @@ "google-cloud-core >= 1.4.1, <3.0.0dev", "google-resumable-media >= 0.6.0, < 3.0dev", "packaging >= 14.3", - "proto-plus >= 1.10.0", # For the legacy proto-based types. "protobuf >= 3.12.0", # For the legacy proto-based types. + "python-dateutil >= 2.7.2, <3.0dev", "pyarrow >= 3.0.0, < 8.0dev", "requests >= 2.18.0, < 3.0.0dev", ] diff --git a/tests/unit/test__pandas_helpers.py b/tests/unit/test__pandas_helpers.py index 5b2fadaf1..1a3f918eb 100644 --- a/tests/unit/test__pandas_helpers.py +++ b/tests/unit/test__pandas_helpers.py @@ -1751,3 +1751,16 @@ def test_bq_to_arrow_field_metadata(module_under_test, field_type, metadata): ).metadata == metadata ) + + +def test_verify_pandas_imports_no_pandas(module_under_test, monkeypatch): + monkeypatch.setattr(module_under_test, "pandas", None) + with pytest.raises(ValueError, match="Please install the 'pandas' package"): + module_under_test.verify_pandas_imports() + + +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") +def test_verify_pandas_imports_no_db_dtypes(module_under_test, monkeypatch): + monkeypatch.setattr(module_under_test, "db_dtypes", None) + with pytest.raises(ValueError, match="Please install the 'db-dtypes' package"): + module_under_test.verify_pandas_imports() diff --git a/tests/unit/test_table.py b/tests/unit/test_table.py index 5241230a4..66bc1d3db 100644 --- a/tests/unit/test_table.py +++ b/tests/unit/test_table.py @@ -1836,7 +1836,7 @@ def test_to_arrow_iterable(self): self.assertEqual(record_batch.num_rows, 0) self.assertEqual(record_batch.num_columns, 0) - @mock.patch("google.cloud.bigquery.table.pandas", new=None) + @mock.patch("google.cloud.bigquery._pandas_helpers.pandas", new=None) def test_to_dataframe_error_if_pandas_is_none(self): row_iterator = self._make_one() with self.assertRaises(ValueError): @@ -1849,7 +1849,7 @@ def test_to_dataframe(self): self.assertIsInstance(df, pandas.DataFrame) self.assertEqual(len(df), 0) # verify the number of rows - @mock.patch("google.cloud.bigquery.table.pandas", new=None) + @mock.patch("google.cloud.bigquery._pandas_helpers.pandas", new=None) def test_to_dataframe_iterable_error_if_pandas_is_none(self): row_iterator = self._make_one() with self.assertRaises(ValueError): @@ -2967,7 +2967,7 @@ def test_to_dataframe_iterable_w_bqstorage_max_results_warning(self): assert isinstance(dataframes[0], pandas.DataFrame) assert isinstance(dataframes[1], pandas.DataFrame) - @mock.patch("google.cloud.bigquery.table.pandas", new=None) + @mock.patch("google.cloud.bigquery._pandas_helpers.pandas", new=None) def test_to_dataframe_iterable_error_if_pandas_is_none(self): from google.cloud.bigquery.schema import SchemaField @@ -3339,7 +3339,7 @@ def test_to_dataframe_datetime_objects(self): self.assertEqual(df["ts"][0].date(), datetime.date(1336, 3, 23)) self.assertEqual(df["date"][0], datetime.date(1111, 1, 1)) - @mock.patch("google.cloud.bigquery.table.pandas", new=None) + @mock.patch("google.cloud.bigquery._pandas_helpers.pandas", new=None) def test_to_dataframe_error_if_pandas_is_none(self): from google.cloud.bigquery.schema import SchemaField