diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 32302e48..8912e9b5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks repos: diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e7f5cf2..3650dfa3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/pybigquery/#history +### [0.6.1](https://www.github.com/googleapis/python-bigquery-sqlalchemy/compare/v0.6.0...v0.6.1) (2021-04-12) + + +### Bug Fixes + +* use `project_id` property from service account credentials ([#120](https://www.github.com/googleapis/python-bigquery-sqlalchemy/issues/120)) ([ab2051d](https://www.github.com/googleapis/python-bigquery-sqlalchemy/commit/ab2051de3097adb68503c01a87f9a91092711d2a)) + ## [0.6.0](https://www.github.com/googleapis/python-bigquery-sqlalchemy/compare/v0.5.1...v0.6.0) (2021-04-06) diff --git a/docs/conf.py b/docs/conf.py index 00c625ac..d4eefe60 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,17 @@ # -*- coding: utf-8 -*- +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # pybigquery documentation build configuration file # diff --git a/pybigquery/_helpers.py b/pybigquery/_helpers.py index 35f7e4ab..fc48144c 100644 --- a/pybigquery/_helpers.py +++ b/pybigquery/_helpers.py @@ -38,13 +38,13 @@ def create_bigquery_client( credentials_path ) credentials = credentials.with_scopes(SCOPES) - default_project = credentials.project + default_project = credentials.project_id elif credentials_info: credentials = service_account.Credentials.from_service_account_info( credentials_info ) credentials = credentials.with_scopes(SCOPES) - default_project = credentials.project + default_project = credentials.project_id else: credentials, default_project = google.auth.default(scopes=SCOPES) diff --git a/setup.py b/setup.py index 153bf30e..2a26bf16 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ # Package metadata. name = "pybigquery" -version = "0.6.0" +version = "0.6.1" description = "SQLAlchemy dialect for BigQuery" # Should be one of: diff --git a/synth.metadata b/synth.metadata index 4cd6f08a..791f1e8f 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,14 +4,14 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/python-bigquery-sqlalchemy.git", - "sha": "b7c9d6f3151520a20239df197a151aa80c3ebdcd" + "sha": "101153a14706ae325893f3ddf6dbd934c4ef65a0" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "4501974ad08b5d693311457e2ea4ce845676e329" + "sha": "5b5bf6d519b2d658d9f2e483d9f6f3d0ba8ee6bc" } } ], diff --git a/tests/system/conftest.py b/tests/system/conftest.py index 376a164f..f16428c3 100644 --- a/tests/system/conftest.py +++ b/tests/system/conftest.py @@ -4,7 +4,9 @@ # license that can be found in the LICENSE file or at # https://opensource.org/licenses/MIT. +import datetime import pathlib +import random import pytest import google.api_core.exceptions @@ -16,6 +18,12 @@ DATA_DIR = pathlib.Path(__file__).parent / "data" +def temp_suffix(): + timestamp = datetime.datetime.utcnow().strftime("%y%m%d_%H%M%S") + random_string = hex(random.randrange(1000000))[2:] + return f"{timestamp}_{random_string}" + + def load_sample_data( full_table_id: str, bigquery_client: bigquery.Client, @@ -84,12 +92,36 @@ def bigquery_dataset( @pytest.fixture(scope="session", autouse=True) -def bigquery_empty_table(bigquery_dataset, bigquery_client, bigquery_schema): +def bigquery_dml_dataset(bigquery_client: bigquery.Client): + project_id = bigquery_client.project + dataset_id = "test_pybigquery_dml" + dataset = bigquery.Dataset(f"{project_id}.{dataset_id}") + # Add default table expiration in case cleanup fails. + dataset.default_table_expiration_ms = 1000 * int( + datetime.timedelta(days=1).total_seconds() + ) + dataset = bigquery_client.create_dataset(dataset, exists_ok=True) + return dataset_id + + +@pytest.fixture(scope="session", autouse=True) +def bigquery_empty_table( + bigquery_dataset: str, + bigquery_dml_dataset: str, + bigquery_client: bigquery.Client, + bigquery_schema: List[bigquery.SchemaField], +): project_id = bigquery_client.project - dataset_id = bigquery_dataset - table_id = f"{project_id}.{dataset_id}.sample_dml" + # Cleanup the sample_dml table, if it exists. + old_table_id = f"{project_id}.{bigquery_dataset}.sample_dml" + bigquery_client.delete_table(old_table_id, not_found_ok=True) + # Create new table in its own dataset. + dataset_id = bigquery_dml_dataset + table_id = f"{project_id}.{dataset_id}.sample_dml_{temp_suffix()}" empty_table = bigquery.Table(table_id, schema=bigquery_schema) - bigquery_client.create_table(empty_table, exists_ok=True) + bigquery_client.create_table(empty_table) + yield table_id + bigquery_client.delete_table(empty_table) @pytest.fixture(scope="session", autouse=True) diff --git a/tests/system/test_helpers.py b/tests/system/test_helpers.py new file mode 100644 index 00000000..18334631 --- /dev/null +++ b/tests/system/test_helpers.py @@ -0,0 +1,74 @@ +# Copyright 2021 The PyBigQuery Authors +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. + +import os +import json + +import pytest + + +@pytest.fixture(scope="session") +def module_under_test(): + from pybigquery import _helpers + + return _helpers + + +@pytest.fixture +def credentials_path(): + if "GOOGLE_APPLICATION_CREDENTIALS" not in os.environ: + pytest.skip("GOOGLE_APPLICATION_CREDENTIALS must be set") + return os.environ["GOOGLE_APPLICATION_CREDENTIALS"] + + +@pytest.fixture +def credentials_info(credentials_path): + with open(credentials_path) as credentials_file: + return json.load(credentials_file) + + +def test_create_bigquery_client_with_credentials_path( + module_under_test, credentials_path, credentials_info +): + bqclient = module_under_test.create_bigquery_client( + credentials_path=credentials_path + ) + assert bqclient.project == credentials_info["project_id"] + + +def test_create_bigquery_client_with_credentials_path_respects_project( + module_under_test, credentials_path +): + """Test that project_id is used, even when there is a default project. + + https://github.com/googleapis/python-bigquery-sqlalchemy/issues/48 + """ + bqclient = module_under_test.create_bigquery_client( + credentials_path=credentials_path, project_id="connection-url-project", + ) + assert bqclient.project == "connection-url-project" + + +def test_create_bigquery_client_with_credentials_info( + module_under_test, credentials_info +): + bqclient = module_under_test.create_bigquery_client( + credentials_info=credentials_info + ) + assert bqclient.project == credentials_info["project_id"] + + +def test_create_bigquery_client_with_credentials_info_respects_project( + module_under_test, credentials_info +): + """Test that project_id is used, even when there is a default project. + + https://github.com/googleapis/python-bigquery-sqlalchemy/issues/48 + """ + bqclient = module_under_test.create_bigquery_client( + credentials_info=credentials_info, project_id="connection-url-project", + ) + assert bqclient.project == "connection-url-project" diff --git a/tests/system/test_sqlalchemy_bigquery.py b/tests/system/test_sqlalchemy_bigquery.py index 5ce46dee..5a7842d5 100644 --- a/tests/system/test_sqlalchemy_bigquery.py +++ b/tests/system/test_sqlalchemy_bigquery.py @@ -174,8 +174,8 @@ def table_one_row(engine): @pytest.fixture(scope="session") -def table_dml(engine): - return Table("test_pybigquery.sample_dml", MetaData(bind=engine), autoload=True) +def table_dml(engine, bigquery_empty_table): + return Table(bigquery_empty_table, MetaData(bind=engine), autoload=True) @pytest.fixture(scope="session") @@ -355,13 +355,11 @@ def test_tables_list(engine, engine_using_test_dataset): tables = engine.table_names() assert "test_pybigquery.sample" in tables assert "test_pybigquery.sample_one_row" in tables - assert "test_pybigquery.sample_dml" in tables assert "test_pybigquery.sample_view" not in tables tables = engine_using_test_dataset.table_names() assert "sample" in tables assert "sample_one_row" in tables - assert "sample_dml" in tables assert "sample_view" not in tables @@ -520,7 +518,7 @@ def test_dml(engine, session, table_dml): {"string": "updated_row"}, synchronize_session=False ) updated_result = table_dml.select().execute().fetchone() - assert updated_result["test_pybigquery.sample_dml_string"] == "updated_row" + assert updated_result[table_dml.c.string] == "updated_row" # test delete session.query(table_dml).filter(table_dml.c.string == "updated_row").delete( @@ -576,16 +574,14 @@ def test_table_names_in_schema(inspector, inspector_using_test_dataset): tables = inspector.get_table_names("test_pybigquery") assert "test_pybigquery.sample" in tables assert "test_pybigquery.sample_one_row" in tables - assert "test_pybigquery.sample_dml" in tables assert "test_pybigquery.sample_view" not in tables - assert len(tables) == 3 + assert len(tables) == 2 tables = inspector_using_test_dataset.get_table_names() assert "sample" in tables assert "sample_one_row" in tables - assert "sample_dml" in tables assert "sample_view" not in tables - assert len(tables) == 3 + assert len(tables) == 2 def test_view_names(inspector, inspector_using_test_dataset): diff --git a/tests/unit/test_helpers.py b/tests/unit/test_helpers.py index 38be0453..1a3acc85 100644 --- a/tests/unit/test_helpers.py +++ b/tests/unit/test_helpers.py @@ -17,7 +17,7 @@ class AnonymousCredentialsWithProject(google.auth.credentials.AnonymousCredentia def __init__(self, project): super().__init__() - self.project = project + self.project_id = project def with_scopes(self, scopes): return self