Skip to content

feat: GCP access token support #459

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ Connection String Parameters

There are many situations where you can't call ``create_engine`` directly, such as when using tools like `Flask SQLAlchemy <http://flask-sqlalchemy.pocoo.org/2.3/>`_. For situations like these, or for situations where you want the ``Client`` to have a `default_query_job_config <https://googlecloudplatform.github.io/google-cloud-python/latest/bigquery/generated/google.cloud.bigquery.client.Client.html#google.cloud.bigquery.client.Client>`_, you can pass many arguments in the query of the connection string.

The ``credentials_path``, ``credentials_info``, ``credentials_base64``, ``location``, ``arraysize`` and ``list_tables_page_size`` parameters are used by this library, and the rest are used to create a `QueryJobConfig <https://googlecloudplatform.github.io/google-cloud-python/latest/bigquery/generated/google.cloud.bigquery.job.QueryJobConfig.html#google.cloud.bigquery.job.QueryJobConfig>`_
The ``credentials_path``, ``credentials_info``, ``credentials_base64``, ``credentials_access_token``, ``location``, ``arraysize`` and ``list_tables_page_size`` parameters are used by this library, and the rest are used to create a `QueryJobConfig <https://googlecloudplatform.github.io/google-cloud-python/latest/bigquery/generated/google.cloud.bigquery.job.QueryJobConfig.html#google.cloud.bigquery.job.QueryJobConfig>`_

Note that if you want to use query strings, it will be more reliable if you use three slashes, so ``'bigquery:///?a=b'`` will work reliably, but ``'bigquery://?a=b'`` might be interpreted as having a "database" of ``?a=b``, depending on the system being used to parse the connection string.

Expand Down Expand Up @@ -234,6 +234,15 @@ To create the base64 encoded string you can use the command line tool ``base64``

Alternatively, you can use an online generator like `www.base64encode.org <https://www.base64encode.org>_` to paste your credentials JSON file to be encoded.

Also, for authentication can be used GCP Access Token as credentials by passing ``credentials_access_token`` parameter.

.. code-block:: python

engine = create_engine('bigquery://', credentials_access_token='YM5/mbURNVpTzK2QC6LoHiaPQgszwchg4XdgcSNADPzYRIMeA3khUHTb30zkvV77kD3kCg5cgSn9buzX5dxJaUYCVwpjOfD/OvNqRTOJV2C')

To generate access token use `google.oauth2.credentials.UserAccessTokenCredentials <https://google-auth.readthedocs.io/en/stable/reference/google.oauth2.credentials.html#google.oauth2.credentials.UserAccessTokenCredentials>`_ function.
Keep in mind that Access Tokens have a maximum expiration time of 1 hour.

Creating tables
^^^^^^^^^^^^^^^

Expand Down
6 changes: 5 additions & 1 deletion sqlalchemy_bigquery/_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from google.api_core import client_info
import google.auth
from google.cloud import bigquery
from google.oauth2 import service_account
from google.oauth2 import service_account, credentials as oauth_credentials
import sqlalchemy
import base64
import json
Expand All @@ -33,6 +33,7 @@ def create_bigquery_client(
credentials_info=None,
credentials_path=None,
credentials_base64=None,
credentials_access_token=None,
default_query_job_config=None,
location=None,
project_id=None,
Expand All @@ -54,6 +55,9 @@ def create_bigquery_client(
)
credentials = credentials.with_scopes(SCOPES)
default_project = credentials.project_id
elif credentials_access_token:
credentials = oauth_credentials.Credentials(credentials_access_token)
_, default_project = google.auth.default(scopes=SCOPES)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is potentially a slow operation. I we might consider only doing this if project_id is None. Or maybe add a if default_project is None to the if project_id is None if statement below?

else:
credentials, default_project = google.auth.default(scopes=SCOPES)

Expand Down
3 changes: 3 additions & 0 deletions sqlalchemy_bigquery/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -759,6 +759,7 @@ def __init__(
location=None,
credentials_info=None,
credentials_base64=None,
credentials_access_token=None,
list_tables_page_size=1000,
*args,
**kwargs,
Expand All @@ -768,6 +769,7 @@ def __init__(
self.credentials_path = credentials_path
self.credentials_info = credentials_info
self.credentials_base64 = credentials_base64
self.credentials_access_token = credentials_access_token
self.location = location
self.dataset_id = None
self.list_tables_page_size = list_tables_page_size
Expand Down Expand Up @@ -816,6 +818,7 @@ def create_connect_args(self, url):
credentials_path=self.credentials_path,
credentials_info=self.credentials_info,
credentials_base64=self.credentials_base64,
credentials_access_token=self.credentials_access_token,
project_id=project_id,
location=self.location,
default_query_job_config=default_query_job_config,
Expand Down
30 changes: 30 additions & 0 deletions tests/system/test_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import base64
import os
import json
import google.auth

import pytest

Expand All @@ -25,6 +26,11 @@ def credentials_path():
return os.environ["GOOGLE_APPLICATION_CREDENTIALS"]


@pytest.fixture
def credentials_access_token():
return 'access_token'


@pytest.fixture
def credentials_info(credentials_path):
with open(credentials_path) as credentials_file:
Expand Down Expand Up @@ -83,6 +89,30 @@ def test_create_bigquery_client_with_credentials_info_respects_project(
assert bqclient.project == "connection-url-project"


def test_create_bigquery_client_with_credentials_access_token(
module_under_test, credentials_access_token
):
bqclient = module_under_test.create_bigquery_client(
credentials_access_token=credentials_access_token
)
_, default_project = google.auth.default()
assert bqclient.project == default_project


def test_create_bigquery_client_with_credentials_access_token_respects_project(
module_under_test, credentials_access_token
):
"""Test that project_id is used, even when there is a default project.

https://github.com/googleapis/python-bigquery-sqlalchemy/issues/48
"""
bqclient = module_under_test.create_bigquery_client(
credentials_access_token=credentials_access_token,
project_id="connection-url-project",
)
assert bqclient.project == "connection-url-project"


def test_create_bigquery_client_with_credentials_base64(
module_under_test, credentials_base64, credentials_info
):
Expand Down
29 changes: 29 additions & 0 deletions tests/unit/test_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@ def module_under_test():
return _helpers


@pytest.fixture
def credentials_access_token():
return 'access_token'


def test_create_bigquery_client_with_credentials_path(monkeypatch, module_under_test):
mock_service_account = mock.create_autospec(service_account.Credentials)
mock_service_account.from_service_account_file.return_value = (
Expand Down Expand Up @@ -108,6 +113,30 @@ def test_create_bigquery_client_with_credentials_info_respects_project(
assert bqclient.project == "connection-url-project"


def test_create_bigquery_client_with_credentials_access_token(
module_under_test, credentials_access_token
):
bqclient = module_under_test.create_bigquery_client(
credentials_access_token=credentials_access_token
)
_, default_project = google.auth.default()
assert bqclient.project == default_project


def test_create_bigquery_client_with_credentials_access_token_respects_project(
module_under_test, credentials_access_token
):
"""Test that project_id is used, even when there is a default project.

https://github.com/googleapis/python-bigquery-sqlalchemy/issues/48
"""
bqclient = module_under_test.create_bigquery_client(
credentials_access_token=credentials_access_token,
project_id="connection-url-project",
)
assert bqclient.project == "connection-url-project"


def test_create_bigquery_client_with_credentials_base64(monkeypatch, module_under_test):
mock_service_account = mock.create_autospec(service_account.Credentials)
mock_service_account.from_service_account_info.return_value = (
Expand Down