From 65e02bef6abd500b737a9ce5bb8e25eed323d72f Mon Sep 17 00:00:00 2001 From: Eli Bixby Date: Fri, 22 May 2015 10:31:09 -0700 Subject: [PATCH 1/4] Moved caching to product agnostic dir from bq --- bigquery/samples/async_query.py | 5 +- bigquery/samples/utils.py | 8 +++- datastore/ndb/README.md | 59 +++++++++++++++++++++-- discoverydoccaching/discovery_doc.py | 71 ++++++++++++++++++++++++++++ 4 files changed, 135 insertions(+), 8 deletions(-) create mode 100644 discoverydoccaching/discovery_doc.py diff --git a/bigquery/samples/async_query.py b/bigquery/samples/async_query.py index 50deb3a4731..eba8d28c609 100644 --- a/bigquery/samples/async_query.py +++ b/bigquery/samples/async_query.py @@ -74,9 +74,8 @@ def main(): query_string = raw_input("Enter the Bigquery SQL Query: ") batch = raw_input("Run query as batch (y/n)?: ") in ( 'True', 'true', 'y', 'Y', 'yes', 'Yes') - - num_retries = raw_input( - "Enter number of times to retry in case of 500 error: ") + num_retries = int(raw_input( + "Enter number of times to retry in case of 500 error: ")) interval = raw_input( "Enter how often to poll the query for completion (seconds): ") diff --git a/bigquery/samples/utils.py b/bigquery/samples/utils.py index d35d0c94713..a9d70e93a5b 100644 --- a/bigquery/samples/utils.py +++ b/bigquery/samples/utils.py @@ -15,8 +15,12 @@ # [START get_service] def get_service(): - from discovery_doc import build_and_update - return build_and_update('bigquery', 'v2') + from googleapiclient.discovery import build + from oauth2client.client import GoogleCredentials + credentials = GoogleCredentials.get_application_default() + if credentials.create_scoped_required(): + credentials = credentials.create_scoped('https://www.googleapis.com/auth/bigquery') + return build('bigquery','v2', credentials=GoogleCredentials.get_application_default()) # [END get_service] diff --git a/datastore/ndb/README.md b/datastore/ndb/README.md index 262b47ba03a..56a14bb3632 100644 --- a/datastore/ndb/README.md +++ b/datastore/ndb/README.md @@ -1,5 +1,58 @@ -appengine-ndb-snippets -====================== +## NDB Overview Sample -Sample code snippets for NDB. +This is a sample app for Google App Engine that exercises the [NDB Python API](https://cloud.google.com/appengine/docs/python/ndb/). +See our other [Google Cloud Platform github +repos](https://github.com/GoogleCloudPlatform) for sample applications and +scaffolding for other python frameworks and use cases. + +## Run Locally +1. Install the [Google Cloud SDK](https://cloud.google.com/sdk/), including the [gcloud tool](https://cloud.google.com/sdk/gcloud/), and [gcloud app component](https://cloud.google.com/sdk/gcloud-app). +2. Setup the gcloud tool. + + ``` + gcloud components update app + gcloud auth login + gcloud config set project + ``` + You don't need a valid app-id to run locally, but will need a valid id to deploy below. + +1. Clone this repo. + + ``` + git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git + cd python-docs-samples/datastore/ndb/ + ``` + +1. Run this project locally from the command line. + + ``` + gcloud preview app run ./ + ``` + +1. Visit the application at [http://localhost:8080](http://localhost:8080). + +## Deploying + +1. Use the [Cloud Developer Console](https://console.developer.google.com) to create a project/app id. (App id and project id are identical) +2. Configure gcloud with your app id. + + ``` + gcloud config set project + ``` +1. Use the [Admin Console](https://appengine.google.com) to view data, queues, and other App Engine specific administration tasks. +1. Use gcloud to deploy your app. + + ``` + gcloud preview app deploy ./ + ``` + +1. Congratulations! Your application is now live at your-app-id.appspot.com + +## Contributing changes + +* See [CONTRIBUTING.md](../../CONTRIBUTING.md) + +## Licensing + +* See [LICENSE](../../LICENSE) diff --git a/discoverydoccaching/discovery_doc.py b/discoverydoccaching/discovery_doc.py new file mode 100644 index 00000000000..006ff46903f --- /dev/null +++ b/discoverydoccaching/discovery_doc.py @@ -0,0 +1,71 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +A module that takes care of caching and updating discovery docs for +google-api-python-clients (until such a feature is integrated). +""" + +import json +import os +import time + +import httplib2 + +# [START build_and_update] + +RESOURCE_PATH = '..' # look for discovery docs in the parent folder +MAX_AGE = 86400 # update discovery docs older than a day +BIGQUERY_SCOPES = ['https://www.googleapis.com/auth/bigquery'] + + +def build_and_update(api, version): + from oauth2client.client import GoogleCredentials + from googleapiclient.discovery import build_from_document + + path = os.path.join(RESOURCE_PATH, '{}.{}'.format(api, version)) + try: + age = time.time() - os.path.getmtime(path) + if age > MAX_AGE: + _update_discovery_doc(api, version, path) + except os.error: + _update_discovery_doc(api, version, path) + + credentials = GoogleCredentials.get_application_default() + if credentials.create_scoped_required(): + credentials = credentials.create_scoped(BIGQUERY_SCOPES) + with open(path, 'r') as discovery_doc: + return build_from_document(discovery_doc.read(), + http=httplib2.Http(), + credentials=credentials) + + +def _update_discovery_doc(api, version, path): + from apiclient.discovery import DISCOVERY_URI + from apiclient.errors import HttpError + from apiclient.errors import InvalidJsonError + import uritemplate + + requested_url = uritemplate.expand(DISCOVERY_URI, + {'api': api, 'apiVersion': version}) + resp, content = httplib2.Http().request(requested_url) + if resp.status >= 400: + raise HttpError(resp, content, uri=requested_url) + try: + with open(path, 'w') as discovery_doc: + discovery_json = json.loads(content) + json.dump(discovery_json, discovery_doc) + except ValueError: + raise InvalidJsonError( + 'Bad JSON: %s from %s.' % (content, requested_url)) +# [END build_and_update] From 87631e0bcaf4ee1dac4358d8e03b9eb6910f587d Mon Sep 17 00:00:00 2001 From: Eli Bixby Date: Fri, 22 May 2015 10:31:42 -0700 Subject: [PATCH 2/4] removed caching from bigquery --- bigquery/samples/discovery_doc.py | 71 ------------------------------- 1 file changed, 71 deletions(-) delete mode 100644 bigquery/samples/discovery_doc.py diff --git a/bigquery/samples/discovery_doc.py b/bigquery/samples/discovery_doc.py deleted file mode 100644 index 006ff46903f..00000000000 --- a/bigquery/samples/discovery_doc.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2015, Google, Inc. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -""" -A module that takes care of caching and updating discovery docs for -google-api-python-clients (until such a feature is integrated). -""" - -import json -import os -import time - -import httplib2 - -# [START build_and_update] - -RESOURCE_PATH = '..' # look for discovery docs in the parent folder -MAX_AGE = 86400 # update discovery docs older than a day -BIGQUERY_SCOPES = ['https://www.googleapis.com/auth/bigquery'] - - -def build_and_update(api, version): - from oauth2client.client import GoogleCredentials - from googleapiclient.discovery import build_from_document - - path = os.path.join(RESOURCE_PATH, '{}.{}'.format(api, version)) - try: - age = time.time() - os.path.getmtime(path) - if age > MAX_AGE: - _update_discovery_doc(api, version, path) - except os.error: - _update_discovery_doc(api, version, path) - - credentials = GoogleCredentials.get_application_default() - if credentials.create_scoped_required(): - credentials = credentials.create_scoped(BIGQUERY_SCOPES) - with open(path, 'r') as discovery_doc: - return build_from_document(discovery_doc.read(), - http=httplib2.Http(), - credentials=credentials) - - -def _update_discovery_doc(api, version, path): - from apiclient.discovery import DISCOVERY_URI - from apiclient.errors import HttpError - from apiclient.errors import InvalidJsonError - import uritemplate - - requested_url = uritemplate.expand(DISCOVERY_URI, - {'api': api, 'apiVersion': version}) - resp, content = httplib2.Http().request(requested_url) - if resp.status >= 400: - raise HttpError(resp, content, uri=requested_url) - try: - with open(path, 'w') as discovery_doc: - discovery_json = json.loads(content) - json.dump(discovery_json, discovery_doc) - except ValueError: - raise InvalidJsonError( - 'Bad JSON: %s from %s.' % (content, requested_url)) -# [END build_and_update] From 8d7a1ca3942364a7dcb470033cfa9231112e2976 Mon Sep 17 00:00:00 2001 From: Eli Bixby Date: Fri, 22 May 2015 10:55:28 -0700 Subject: [PATCH 3/4] Added scopes as arg to service builder --- discoverydoccaching/discovery_doc.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/discoverydoccaching/discovery_doc.py b/discoverydoccaching/discovery_doc.py index 006ff46903f..980deab6fc4 100644 --- a/discoverydoccaching/discovery_doc.py +++ b/discoverydoccaching/discovery_doc.py @@ -26,10 +26,8 @@ RESOURCE_PATH = '..' # look for discovery docs in the parent folder MAX_AGE = 86400 # update discovery docs older than a day -BIGQUERY_SCOPES = ['https://www.googleapis.com/auth/bigquery'] - -def build_and_update(api, version): +def build_and_update(api, version, scopes=None): from oauth2client.client import GoogleCredentials from googleapiclient.discovery import build_from_document @@ -42,8 +40,8 @@ def build_and_update(api, version): _update_discovery_doc(api, version, path) credentials = GoogleCredentials.get_application_default() - if credentials.create_scoped_required(): - credentials = credentials.create_scoped(BIGQUERY_SCOPES) + if not scopes is None and credentials.create_scoped_required(): + credentials = credentials.create_scoped(scopes) with open(path, 'r') as discovery_doc: return build_from_document(discovery_doc.read(), http=httplib2.Http(), From 6dbfe79cf2c778397e940ecda7763f3c70040365 Mon Sep 17 00:00:00 2001 From: Eli Bixby Date: Fri, 22 May 2015 11:29:54 -0700 Subject: [PATCH 4/4] Fixed nit --- discoverydoccaching/discovery_doc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discoverydoccaching/discovery_doc.py b/discoverydoccaching/discovery_doc.py index 980deab6fc4..2b3c1509e56 100644 --- a/discoverydoccaching/discovery_doc.py +++ b/discoverydoccaching/discovery_doc.py @@ -40,7 +40,7 @@ def build_and_update(api, version, scopes=None): _update_discovery_doc(api, version, path) credentials = GoogleCredentials.get_application_default() - if not scopes is None and credentials.create_scoped_required(): + if scopes is not None and credentials.create_scoped_required(): credentials = credentials.create_scoped(scopes) with open(path, 'r') as discovery_doc: return build_from_document(discovery_doc.read(),