From fe89e2ff4dbe2fb9a02ae4df86d0a3f1bfb3593d Mon Sep 17 00:00:00 2001 From: Ryan Matsumoto Date: Thu, 9 Feb 2017 17:12:48 -0800 Subject: [PATCH 1/5] Added AppEngine Storage Client sample, renamed subfolders --- .../storage/{ => api-client}/README.md | 0 .../storage/{ => api-client}/app.yaml | 0 .../{ => api-client}/appengine_config.py | 0 .../standard/storage/{ => api-client}/main.py | 0 .../storage/{ => api-client}/main_test.py | 0 .../storage/{ => api-client}/requirements.txt | 0 .../storage/appengine-client/__init__.py | 0 .../storage/appengine-client/app.yaml | 12 ++ .../appengine-client/appengine_config.py | 4 + .../standard/storage/appengine-client/main.py | 186 ++++++++++++++++++ .../storage/appengine-client/main_test.py | 27 +++ .../storage/appengine-client/requirements.txt | 1 + 12 files changed, 230 insertions(+) rename appengine/standard/storage/{ => api-client}/README.md (100%) rename appengine/standard/storage/{ => api-client}/app.yaml (100%) rename appengine/standard/storage/{ => api-client}/appengine_config.py (100%) rename appengine/standard/storage/{ => api-client}/main.py (100%) rename appengine/standard/storage/{ => api-client}/main_test.py (100%) rename appengine/standard/storage/{ => api-client}/requirements.txt (100%) create mode 100644 appengine/standard/storage/appengine-client/__init__.py create mode 100644 appengine/standard/storage/appengine-client/app.yaml create mode 100644 appengine/standard/storage/appengine-client/appengine_config.py create mode 100644 appengine/standard/storage/appengine-client/main.py create mode 100644 appengine/standard/storage/appengine-client/main_test.py create mode 100644 appengine/standard/storage/appengine-client/requirements.txt diff --git a/appengine/standard/storage/README.md b/appengine/standard/storage/api-client/README.md similarity index 100% rename from appengine/standard/storage/README.md rename to appengine/standard/storage/api-client/README.md diff --git a/appengine/standard/storage/app.yaml b/appengine/standard/storage/api-client/app.yaml similarity index 100% rename from appengine/standard/storage/app.yaml rename to appengine/standard/storage/api-client/app.yaml diff --git a/appengine/standard/storage/appengine_config.py b/appengine/standard/storage/api-client/appengine_config.py similarity index 100% rename from appengine/standard/storage/appengine_config.py rename to appengine/standard/storage/api-client/appengine_config.py diff --git a/appengine/standard/storage/main.py b/appengine/standard/storage/api-client/main.py similarity index 100% rename from appengine/standard/storage/main.py rename to appengine/standard/storage/api-client/main.py diff --git a/appengine/standard/storage/main_test.py b/appengine/standard/storage/api-client/main_test.py similarity index 100% rename from appengine/standard/storage/main_test.py rename to appengine/standard/storage/api-client/main_test.py diff --git a/appengine/standard/storage/requirements.txt b/appengine/standard/storage/api-client/requirements.txt similarity index 100% rename from appengine/standard/storage/requirements.txt rename to appengine/standard/storage/api-client/requirements.txt diff --git a/appengine/standard/storage/appengine-client/__init__.py b/appengine/standard/storage/appengine-client/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/appengine/standard/storage/appengine-client/app.yaml b/appengine/standard/storage/appengine-client/app.yaml new file mode 100644 index 00000000000..3ec099ad09c --- /dev/null +++ b/appengine/standard/storage/appengine-client/app.yaml @@ -0,0 +1,12 @@ +runtime: python27 +api_version: 1 +threadsafe: yes + +env_variables: + +handlers: +- url: /blobstore.* + script: blobstore.app + +- url: /.* + script: main.app diff --git a/appengine/standard/storage/appengine-client/appengine_config.py b/appengine/standard/storage/appengine-client/appengine_config.py new file mode 100644 index 00000000000..4fdc5d2b60f --- /dev/null +++ b/appengine/standard/storage/appengine-client/appengine_config.py @@ -0,0 +1,4 @@ +from google.appengine.ext import vendor + +# Add any libraries installed in the "lib" folder. +vendor.add('lib') diff --git a/appengine/standard/storage/appengine-client/main.py b/appengine/standard/storage/appengine-client/main.py new file mode 100644 index 00000000000..d15393f67ea --- /dev/null +++ b/appengine/standard/storage/appengine-client/main.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python + +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# [START sample] +"""A sample app that uses GCS client to operate on bucket and file.""" + +# [START imports] +import logging +import os + +import cloudstorage as gcs +from google.appengine.api import app_identity + +import webapp2 + +# [END imports] + +# [START retries] +my_default_retry_params = gcs.RetryParams( + initial_delay=0.2, max_delay=5.0, backoff_factor=2, max_retry_period=15) +gcs.set_default_retry_params(my_default_retry_params) +# [END retries] + + +class MainPage(webapp2.RequestHandler): + """Main page for GCS demo application.""" + +# [START get_default_bucket] + def get(self): + bucket_name = os.environ.get( + 'BUCKET_NAME', app_identity.get_default_gcs_bucket_name()) + + self.response.headers['Content-Type'] = 'text/plain' + self.response.write( + 'Demo GCS Application running from Version: ' + + os.environ['CURRENT_VERSION_ID'] + '\n') + self.response.write('Using bucket name: ' + bucket_name + '\n\n') +# [END get_default_bucket] + + bucket = '/' + bucket_name + filename = bucket + '/demo-testfile' + self.tmp_filenames_to_clean_up = [] + + try: + self.create_file(filename) + self.response.write('\n\n') + + self.read_file(filename) + self.response.write('\n\n') + + self.stat_file(filename) + self.response.write('\n\n') + + self.create_files_for_list_bucket(bucket) + self.response.write('\n\n') + + self.list_bucket(bucket) + self.response.write('\n\n') + + self.list_bucket_directory_mode(bucket) + self.response.write('\n\n') + + except Exception, e: + logging.exception(e) + self.delete_files() + self.response.write( + '\n\nThere was an error running the demo!Please check the logs' + ' for more details.\n') + + else: + self.delete_files() + self.response.write('\n\nThe demo ran successfully!\n') + +# [START write] + def create_file(self, filename): + """Create a file. + + The retry_params specified in the open call will override the default + retry params for this particular file handle. + + Args: + filename: filename. + """ + self.response.write('Creating file %s\n' % filename) + + write_retry_params = gcs.RetryParams(backoff_factor=1.1) + gcs_file = gcs.open( + filename, 'w', content_type='text/plain', options={ + 'x-goog-meta-foo': 'foo', 'x-goog-meta-bar': 'bar'}, + retry_params=write_retry_params) + gcs_file.write('abcde\n') + gcs_file.write('f'*1024*4 + '\n') + gcs_file.close() + self.tmp_filenames_to_clean_up.append(filename) +# [END write] + +# [START read] + def read_file(self, filename): + self.response.write( + 'Abbreviated file content (first line and last 1K):\n') + + gcs_file = gcs.open(filename) + self.response.write(gcs_file.readline()) + gcs_file.seek(-1024, os.SEEK_END) + self.response.write(gcs_file.read()) + gcs_file.close() +# [END read] + + def stat_file(self, filename): + self.response.write('File stat:\n') + + stat = gcs.stat(filename) + self.response.write(repr(stat)) + + def create_files_for_list_bucket(self, bucket): + self.response.write('Creating more files for listbucket...\n') + filenames = [bucket + n for n in [ + '/foo1', '/foo2', '/bar', '/bar/1', '/bar/2', '/boo/']] + for f in filenames: + self.create_file(f) + +# [START list_bucket] + def list_bucket(self, bucket): + """Create several files and paginate through them. + + Production apps should set page_size to a practical value. + + Args: + bucket: bucket. + """ + self.response.write('Listbucket result:\n') + + page_size = 1 + stats = gcs.listbucket(bucket + '/foo', max_keys=page_size) + while True: + count = 0 + for stat in stats: + count += 1 + self.response.write(repr(stat)) + self.response.write('\n') + + if count != page_size or count == 0: + break + stats = gcs.listbucket( + bucket + '/foo', max_keys=page_size, marker=stat.filename) +# [END list_bucket] + + def list_bucket_directory_mode(self, bucket): + self.response.write('Listbucket directory mode result:\n') + for stat in gcs.listbucket(bucket + '/b', delimiter='/'): + self.response.write('%r' % stat) + self.response.write('\n') + if stat.is_dir: + for subdir_file in gcs.listbucket( + stat.filename, delimiter='/'): + self.response.write(' %r' % subdir_file) + self.response.write('\n') + +# [START delete_files] + def delete_files(self): + self.response.write('Deleting files...\n') + for filename in self.tmp_filenames_to_clean_up: + self.response.write('Deleting file %s\n' % filename) + try: + gcs.delete(filename) + except gcs.NotFoundError: + pass +# [END delete_files] + + +app = webapp2.WSGIApplication( + [('/', MainPage)], debug=True) +# [END sample] diff --git a/appengine/standard/storage/appengine-client/main_test.py b/appengine/standard/storage/appengine-client/main_test.py new file mode 100644 index 00000000000..57db55cde73 --- /dev/null +++ b/appengine/standard/storage/appengine-client/main_test.py @@ -0,0 +1,27 @@ +# Copyright 2017 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import webtest + +import main + + +def test_get(testbed, cloud_config): + main.BUCKET_NAME = cloud_config.project + app = webtest.TestApp(main.app) + + response = app.get('/') + + assert response.status_int == 200 + assert 'The demo ran successfully!' in response.body diff --git a/appengine/standard/storage/appengine-client/requirements.txt b/appengine/standard/storage/appengine-client/requirements.txt new file mode 100644 index 00000000000..f2ec35f05f9 --- /dev/null +++ b/appengine/standard/storage/appengine-client/requirements.txt @@ -0,0 +1 @@ +GoogleAppEngineCloudStorageClient==1.9.22.1 From 603df4bfb7ae3497ea0279a37ce3476de94b5351 Mon Sep 17 00:00:00 2001 From: Ryan Matsumoto Date: Fri, 10 Feb 2017 11:31:07 -0800 Subject: [PATCH 2/5] Fixed code review issues --- .../standard/storage/appengine-client/main.py | 113 ++++++++---------- 1 file changed, 47 insertions(+), 66 deletions(-) diff --git a/appengine/standard/storage/appengine-client/main.py b/appengine/standard/storage/appengine-client/main.py index d15393f67ea..87ca21501b1 100644 --- a/appengine/standard/storage/appengine-client/main.py +++ b/appengine/standard/storage/appengine-client/main.py @@ -18,10 +18,9 @@ """A sample app that uses GCS client to operate on bucket and file.""" # [START imports] -import logging import os -import cloudstorage as gcs +import cloudstorage from google.appengine.api import app_identity import webapp2 @@ -29,9 +28,10 @@ # [END imports] # [START retries] -my_default_retry_params = gcs.RetryParams( - initial_delay=0.2, max_delay=5.0, backoff_factor=2, max_retry_period=15) -gcs.set_default_retry_params(my_default_retry_params) +cloudstorage.set_default_retry_params( + cloudstorage.RetryParams( + initial_delay=0.2, max_delay=5.0, backoff_factor=2, max_retry_period=15 + )) # [END retries] @@ -45,65 +45,51 @@ def get(self): self.response.headers['Content-Type'] = 'text/plain' self.response.write( - 'Demo GCS Application running from Version: ' + - os.environ['CURRENT_VERSION_ID'] + '\n') - self.response.write('Using bucket name: ' + bucket_name + '\n\n') + 'Demo GCS Application running from Version: {}\n'.format( + os.environ['CURRENT_VERSION_ID'])) + self.response.write('Using bucket name: \n\n'.format(bucket_name)) # [END get_default_bucket] bucket = '/' + bucket_name filename = bucket + '/demo-testfile' self.tmp_filenames_to_clean_up = [] - try: - self.create_file(filename) - self.response.write('\n\n') + self.create_file(filename) + self.response.write('\n\n') - self.read_file(filename) - self.response.write('\n\n') + self.read_file(filename) + self.response.write('\n\n') - self.stat_file(filename) - self.response.write('\n\n') + self.stat_file(filename) + self.response.write('\n\n') - self.create_files_for_list_bucket(bucket) - self.response.write('\n\n') + self.create_files_for_list_bucket(bucket) + self.response.write('\n\n') - self.list_bucket(bucket) - self.response.write('\n\n') + self.list_bucket(bucket) + self.response.write('\n\n') - self.list_bucket_directory_mode(bucket) - self.response.write('\n\n') + self.list_bucket_directory_mode(bucket) + self.response.write('\n\n') - except Exception, e: - logging.exception(e) - self.delete_files() - self.response.write( - '\n\nThere was an error running the demo!Please check the logs' - ' for more details.\n') - - else: - self.delete_files() - self.response.write('\n\nThe demo ran successfully!\n') + self.delete_files() + self.response.write('\n\nThe demo ran successfully!\n') # [START write] def create_file(self, filename): - """Create a file. - - The retry_params specified in the open call will override the default - retry params for this particular file handle. + """Create a file.""" - Args: - filename: filename. - """ - self.response.write('Creating file %s\n' % filename) + self.response.write('Creating file {}\n'.format(filename)) - write_retry_params = gcs.RetryParams(backoff_factor=1.1) - gcs_file = gcs.open( + # The retry_params specified in the open call will override the default + # retry params for this particular file handle. + write_retry_params = cloudstorage.RetryParams(backoff_factor=1.1) + with cloudstorage.open( filename, 'w', content_type='text/plain', options={ 'x-goog-meta-foo': 'foo', 'x-goog-meta-bar': 'bar'}, - retry_params=write_retry_params) - gcs_file.write('abcde\n') - gcs_file.write('f'*1024*4 + '\n') - gcs_file.close() + retry_params=write_retry_params) as cloudstorage_file: + cloudstorage_file.write('abcde\n') + cloudstorage_file.write('f'*1024*4 + '\n') self.tmp_filenames_to_clean_up.append(filename) # [END write] @@ -112,17 +98,16 @@ def read_file(self, filename): self.response.write( 'Abbreviated file content (first line and last 1K):\n') - gcs_file = gcs.open(filename) - self.response.write(gcs_file.readline()) - gcs_file.seek(-1024, os.SEEK_END) - self.response.write(gcs_file.read()) - gcs_file.close() + with cloudstorage.open(filename) as cloudstorage_file: + self.response.write(cloudstorage_file.readline()) + cloudstorage_file.seek(-1024, os.SEEK_END) + self.response.write(cloudstorage_file.read()) # [END read] def stat_file(self, filename): self.response.write('File stat:\n') - stat = gcs.stat(filename) + stat = cloudstorage.stat(filename) self.response.write(repr(stat)) def create_files_for_list_bucket(self, bucket): @@ -134,17 +119,13 @@ def create_files_for_list_bucket(self, bucket): # [START list_bucket] def list_bucket(self, bucket): - """Create several files and paginate through them. - - Production apps should set page_size to a practical value. + """Create several files and paginate through them.""" - Args: - bucket: bucket. - """ self.response.write('Listbucket result:\n') + # Production apps should set page_size to a practical value. page_size = 1 - stats = gcs.listbucket(bucket + '/foo', max_keys=page_size) + stats = cloudstorage.listbucket(bucket + '/foo', max_keys=page_size) while True: count = 0 for stat in stats: @@ -154,29 +135,29 @@ def list_bucket(self, bucket): if count != page_size or count == 0: break - stats = gcs.listbucket( + stats = cloudstorage.listbucket( bucket + '/foo', max_keys=page_size, marker=stat.filename) # [END list_bucket] def list_bucket_directory_mode(self, bucket): self.response.write('Listbucket directory mode result:\n') - for stat in gcs.listbucket(bucket + '/b', delimiter='/'): - self.response.write('%r' % stat) + for stat in cloudstorage.listbucket(bucket + '/b', delimiter='/'): + self.response.write(stat) self.response.write('\n') if stat.is_dir: - for subdir_file in gcs.listbucket( + for subdir_file in cloudstorage.listbucket( stat.filename, delimiter='/'): - self.response.write(' %r' % subdir_file) + self.response.write(' {}'.format(subdir_file)) self.response.write('\n') # [START delete_files] def delete_files(self): self.response.write('Deleting files...\n') for filename in self.tmp_filenames_to_clean_up: - self.response.write('Deleting file %s\n' % filename) + self.response.write('Deleting file {}\n'.format(filename)) try: - gcs.delete(filename) - except gcs.NotFoundError: + cloudstorage.delete(filename) + except cloudstorage.NotFoundError: pass # [END delete_files] From 6e8807a225b74343a93a99e47925bb4ff4e038d7 Mon Sep 17 00:00:00 2001 From: Ryan Matsumoto Date: Fri, 10 Feb 2017 16:35:12 -0800 Subject: [PATCH 3/5] Updated storage api-client code to upload/delete objects - now testing works --- appengine/standard/storage/api-client/main.py | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/appengine/standard/storage/api-client/main.py b/appengine/standard/storage/api-client/main.py index e952878ced8..a747d47f68f 100644 --- a/appengine/standard/storage/api-client/main.py +++ b/appengine/standard/storage/api-client/main.py @@ -23,28 +23,53 @@ """ import json +import StringIO from googleapiclient import discovery +from googleapiclient import http from oauth2client.client import GoogleCredentials import webapp2 # The bucket that will be used to list objects. -BUCKET_NAME = '' +BUCKET_NAME = 'unique-apricot-158222' + +# The filename for an object to be uploaded. +FILE_NAME = 'storage-api-client-sample-file.txt' credentials = GoogleCredentials.get_application_default() storage = discovery.build('storage', 'v1', credentials=credentials) class MainPage(webapp2.RequestHandler): + def upload_object(self, bucket, filename): + body = { + 'name': FILE_NAME, + } + req = storage.objects().insert( + bucket=bucket, body=body, media_body=http.MediaIoBaseUpload( + filename, 'application/octet-stream')) + resp = req.execute() + return resp + + def delete_object(self, bucket, filename): + req = storage.objects().delete(bucket=bucket, object=filename) + resp = req.execute() + return resp + def get(self): - response = storage.objects().list(bucket=BUCKET_NAME).execute() + string_io_file = StringIO.StringIO() + string_io_file.write('Hello World!') + self.upload_object(BUCKET_NAME, string_io_file) + response = storage.objects().list(bucket=BUCKET_NAME).execute() self.response.write( '

Objects.list raw response:

' '
{}
'.format( json.dumps(response, sort_keys=True, indent=2))) + self.delete_object(BUCKET_NAME, FILE_NAME) + app = webapp2.WSGIApplication([ ('/', MainPage) From 4e409433ec7307e28c17c2c7895f4bad9b39b452 Mon Sep 17 00:00:00 2001 From: Ryan Matsumoto Date: Fri, 10 Feb 2017 16:38:18 -0800 Subject: [PATCH 4/5] Changed BUCKET_NAME back to --- appengine/standard/storage/api-client/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appengine/standard/storage/api-client/main.py b/appengine/standard/storage/api-client/main.py index a747d47f68f..56589c41501 100644 --- a/appengine/standard/storage/api-client/main.py +++ b/appengine/standard/storage/api-client/main.py @@ -32,7 +32,7 @@ # The bucket that will be used to list objects. -BUCKET_NAME = 'unique-apricot-158222' +BUCKET_NAME = '' # The filename for an object to be uploaded. FILE_NAME = 'storage-api-client-sample-file.txt' From 897dd07d77030161ced80b323af8d114f338794d Mon Sep 17 00:00:00 2001 From: Ryan Matsumoto Date: Mon, 13 Feb 2017 13:30:58 -0800 Subject: [PATCH 5/5] Fixed code review issues --- appengine/standard/storage/api-client/main.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/appengine/standard/storage/api-client/main.py b/appengine/standard/storage/api-client/main.py index 56589c41501..3bc15d97862 100644 --- a/appengine/standard/storage/api-client/main.py +++ b/appengine/standard/storage/api-client/main.py @@ -34,21 +34,18 @@ # The bucket that will be used to list objects. BUCKET_NAME = '' -# The filename for an object to be uploaded. -FILE_NAME = 'storage-api-client-sample-file.txt' - credentials = GoogleCredentials.get_application_default() storage = discovery.build('storage', 'v1', credentials=credentials) class MainPage(webapp2.RequestHandler): - def upload_object(self, bucket, filename): + def upload_object(self, bucket, file_object): body = { - 'name': FILE_NAME, + 'name': 'storage-api-client-sample-file.txt', } req = storage.objects().insert( bucket=bucket, body=body, media_body=http.MediaIoBaseUpload( - filename, 'application/octet-stream')) + file_object, 'application/octet-stream')) resp = req.execute() return resp @@ -58,8 +55,7 @@ def delete_object(self, bucket, filename): return resp def get(self): - string_io_file = StringIO.StringIO() - string_io_file.write('Hello World!') + string_io_file = StringIO.StringIO('Hello World!') self.upload_object(BUCKET_NAME, string_io_file) response = storage.objects().list(bucket=BUCKET_NAME).execute() @@ -68,7 +64,7 @@ def get(self): '
{}
'.format( json.dumps(response, sort_keys=True, indent=2))) - self.delete_object(BUCKET_NAME, FILE_NAME) + self.delete_object(BUCKET_NAME, 'storage-api-client-sample-file.txt') app = webapp2.WSGIApplication([