From 69b19e2653106e5f49d7e460cc034ed9857d4800 Mon Sep 17 00:00:00 2001 From: Shun Fan Date: Tue, 21 Jul 2015 22:16:15 -0700 Subject: [PATCH 1/3] Add storagetransfer samples --- storage/storage_transfer/README.md | 49 +++++++++++++ storage/storage_transfer/aws_request.py | 68 +++++++++++++++++++ storage/storage_transfer/create_client.py | 27 ++++++++ storage/storage_transfer/nearline_request.py | 64 +++++++++++++++++ storage/storage_transfer/test_aws_request.py | 39 +++++++++++ .../storage_transfer/test_create_client.py | 44 ++++++++++++ .../storage_transfer/test_nearline_request.py | 39 +++++++++++ .../storage_transfer/test_transfer_check.py | 31 +++++++++ storage/storage_transfer/transfer_check.py | 44 ++++++++++++ 9 files changed, 405 insertions(+) create mode 100644 storage/storage_transfer/README.md create mode 100644 storage/storage_transfer/aws_request.py create mode 100644 storage/storage_transfer/create_client.py create mode 100644 storage/storage_transfer/nearline_request.py create mode 100644 storage/storage_transfer/test_aws_request.py create mode 100644 storage/storage_transfer/test_create_client.py create mode 100644 storage/storage_transfer/test_nearline_request.py create mode 100644 storage/storage_transfer/test_transfer_check.py create mode 100644 storage/storage_transfer/transfer_check.py diff --git a/storage/storage_transfer/README.md b/storage/storage_transfer/README.md new file mode 100644 index 00000000000..43b5aa03d4b --- /dev/null +++ b/storage/storage_transfer/README.md @@ -0,0 +1,49 @@ +# Transfer Service sample using Python + +This app creates two types of transfers using the Transfer Service tool. + +## Prerequisites + +1. Set up a project on Google Developers Console. + 1. Go to the [Developers Console](https://cloud.google.com/console) and create or select your project. + You will need the project ID later. +1. Add the Storage Transfer service account, cloud-mobility@system.gserviceaccount.com as an + editor of your project. +1. Set up gcloud for application default credentials. + 1. `gcloud components update` + 1. `gcloud auth login` + 1. `gcloud config set project PROJECT_ID` + 1. `export GOOGLE_APPLICATION_CREDENTIALS=PATH/TO/CREDENTIALS.json` +1. Install [Google API Client Library for Python](https://developers.google.com/api-client-library/python/start/installation). + +## Transfer from Amazon S3 to Google Cloud Storage + +Creating a one-time transfer from Amazon S3 to Google Cloud Storage. +1. Set up data sink. + 1. Go to the Developers Console and create a bucket under Cloud Storage > Storage Browser. +1. Set up data source. + 1. Go to AWS Management Console and create a bucket. + 1. Under Security Credentials, create an IAM User with access to the bucket. + 1. Create an Access Key for the user. Note the Access Key ID and Secret Access Key. +1. In aws_request.py, fill in the Transfer Job JSON template with relevant values. +1. Run with `python aws_request.py` + 1. Note the job ID in the returned Transfer Job. + +## Transfer data from a standard Cloud Storage bucket to a Cloud Storage Nearline bucket + +Creating a daily transfer from a standard Cloud Storage bucket to a Cloud Storage Nearline +bucket for files untouched for 30 days. +1. Set up data sink. + 1. Go to the Developers Console and create a bucket under Cloud Storage > Storage Browser. + 1. Select Nearline for Storage Class. +1. Set up data source. + 1. Go to the Developers Console and create a bucket under Cloud Storage > Storage Browser. +1. In nearline_request.py, fill in the Transfer Job JSON template with relevant values. +1. Run with `python nearline_request.py` + 1. Note the job ID in the returned Transfer Job. + +## Checking the status of a transfer + +1. In transfer_check.py, fill in the Transfer Job JSON template with relevant values. + Use the Job Name you recorded earlier. +1. Run with `python transfer_check.py` diff --git a/storage/storage_transfer/aws_request.py b/storage/storage_transfer/aws_request.py new file mode 100644 index 00000000000..84714de4793 --- /dev/null +++ b/storage/storage_transfer/aws_request.py @@ -0,0 +1,68 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import json +import logging + +import create_client + + +def main(): + """Create a one-off transfer from Amazon S3 to GCS.""" + logging.getLogger().setLevel(logging.DEBUG) + transfer_service_client = create_client.create_transfer_client() + + # Edit this template with desired parameters. + # Specify times below using US Pacific Time Zone. + transfer_job = ''' + { + "description": "YOUR DESCRIPTION", + "status": "ENABLED", + "projectId": "YOUR_PROJECT_ID", + "schedule": { + "scheduleStartDate": { + "day": 1, + "month": 1, + "year": 2015 + }, + "scheduleEndDate": { + "day": 1, + "month": 1, + "year": 2015 + }, + "startTimeOfDay": { + "hours": 0, + "minutes": 0 + } + }, + "transferSpec": { + "awsS3DataSource": { + "bucketName": "YOUR_SOURCE_BUCKET", + "awsAccessKey": { + "accessKeyId": "YOUR_ACCESS_KEY_ID", + "secretAccessKey": "YOUR_SECRET_ACCESS_KEY" + } + }, + "gcsDataSink": { + "bucketName": "YOUR_SINK_BUCKET" + } + } + } + ''' + + result = transfer_service_client.transferJobs().create(body=json.loads( + transfer_job)).execute() + logging.info('Returned transferJob: %s', json.dumps(result, indent=4)) + +if __name__ == '__main__': + main() diff --git a/storage/storage_transfer/create_client.py b/storage/storage_transfer/create_client.py new file mode 100644 index 00000000000..c5708950918 --- /dev/null +++ b/storage/storage_transfer/create_client.py @@ -0,0 +1,27 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import logging + +from apiclient import discovery +from oauth2client.client import GoogleCredentials + +CLOUD_SCOPES = 'https://www.googleapis.com/auth/cloud-platform' + + +def create_transfer_client(): + """Create a transfer client.""" + + logging.getLogger().setLevel(logging.DEBUG) + credentials = GoogleCredentials.get_application_default() + return discovery.build('storagetransfer', 'v1', credentials=credentials) diff --git a/storage/storage_transfer/nearline_request.py b/storage/storage_transfer/nearline_request.py new file mode 100644 index 00000000000..2e16b52252e --- /dev/null +++ b/storage/storage_transfer/nearline_request.py @@ -0,0 +1,64 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import json +import logging + +import create_client + + +def main(): + """Transfer from standard Cloud Storage to Cloud Storage Nearline.""" + logging.getLogger().setLevel(logging.DEBUG) + transfer_service_client = create_client.create_transfer_client() + + # Edit this template with desired parameters. + # Specify times below using US Pacific Time Zone. + transfer_job = ''' + { + "description": "YOUR DESCRIPTION", + "status": "ENABLED", + "projectId": "YOUR_PROJECT_ID", + "schedule": { + "scheduleStartDate": { + "day": 1, + "month": 1, + "year": 2015 + }, + "startTimeOfDay": { + "hours": 1, + "minutes": 1 + } + }, + "transferSpec": { + "gcsDataSource": { + "bucketName": "YOUR_SOURCE_BUCKET" + }, + "gcsDataSink": { + "bucketName": "YOUR_SINK_BUCKET" + }, + "objectConditions": { + "minTimeElapsedSinceLastModification": "2592000s" + }, + "transferOptions": { + "deleteObjectsFromSourceAfterTransfer": true + } + } + } + ''' + result = transfer_service_client.transferJobs().create(body=json.loads( + transfer_job)).execute() + logging.info('Returned transferJob: %s', json.dumps(result, indent=4)) + +if __name__ == '__main__': + main() diff --git a/storage/storage_transfer/test_aws_request.py b/storage/storage_transfer/test_aws_request.py new file mode 100644 index 00000000000..f37a7c512ab --- /dev/null +++ b/storage/storage_transfer/test_aws_request.py @@ -0,0 +1,39 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import unittest + +from aws_request import main + +from mock import Mock +from mock import patch + + +class AwsRequestTestCase(unittest.TestCase): + """A test case for creating a TransferJob from AWS S3.""" + + def setUp(self): + patcher1 = patch( + 'storage.storage_transfer.aws_request.create_client') + self.mock_create_client = patcher1.start() + self.addCleanup(patcher1.stop) + self.mock_client = Mock(spec=['transferJobs']) + self.mock_create_client.create_transfer_client.return_value = \ + self.mock_client + + def test_create_aws_request(self): + execute = self.mock_client.transferJobs.return_value.create.return_value \ + .execute + execute.return_value = "" + main() + execute.assert_called_with() diff --git a/storage/storage_transfer/test_create_client.py b/storage/storage_transfer/test_create_client.py new file mode 100644 index 00000000000..d20b579db9d --- /dev/null +++ b/storage/storage_transfer/test_create_client.py @@ -0,0 +1,44 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import unittest + +from mock import Mock +from mock import patch +from oauth2client.client import GoogleCredentials + +import create_client + + +class CheckCreateClientTestCase(unittest.TestCase): + """A test case for client creation.""" + + def setUp(self): + patcher1 = patch( + 'storage.storage_transfer.create_client.GoogleCredentials') + patcher2 = patch( + 'storage.storage_transfer.create_client.discovery.build') + self.mock_google_credentials = patcher1.start() + self.mock_discovery = patcher2.start() + self.addCleanup(patcher1.stop) + self.addCleanup(patcher2.stop) + + self.mock_credentials = Mock(spec=GoogleCredentials) + self.mock_google_credentials.get_application_default.return_value = \ + self.mock_credentials + + def test_create_client(self): + create_client.create_transfer_client() + self.mock_discovery.assert_called_with( + 'storagetransfer', 'v1', + credentials=self.mock_credentials) diff --git a/storage/storage_transfer/test_nearline_request.py b/storage/storage_transfer/test_nearline_request.py new file mode 100644 index 00000000000..f70b45e5d5d --- /dev/null +++ b/storage/storage_transfer/test_nearline_request.py @@ -0,0 +1,39 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import unittest + +from mock import Mock +from mock import patch + +from nearline_request import main + + +class NearlineRequestTestCase(unittest.TestCase): + """A test case for creating a TransferJob to Nearline for old files.""" + + def setUp(self): + patcher1 = patch( + 'storage.storage_transfer.nearline_request.create_client') + self.mock_create_client = patcher1.start() + self.addCleanup(patcher1.stop) + self.mock_client = Mock(spec=['transferJobs']) + self.mock_create_client.create_transfer_client.return_value = \ + self.mock_client + + def test_create_nearline_request(self): + execute = self.mock_client.transferJobs.return_value.create.return_value \ + .execute + execute.return_value = "" + main() + execute.assert_called_with() diff --git a/storage/storage_transfer/test_transfer_check.py b/storage/storage_transfer/test_transfer_check.py new file mode 100644 index 00000000000..cd3c6886979 --- /dev/null +++ b/storage/storage_transfer/test_transfer_check.py @@ -0,0 +1,31 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import unittest + +from mock import Mock + +from transfer_check import check_operation + + +class CheckTransferTestCase(unittest.TestCase): + """A test case for querying transfer job completion.""" + + def test_check_operation(self): + mock_client = Mock(spec=['transferOperations']) + execute = mock_client.transferOperations.return_value.list.return_value \ + .execute + project_id = "" + job_name = "" + check_operation(mock_client, project_id, job_name) + execute.assert_called_with() diff --git a/storage/storage_transfer/transfer_check.py b/storage/storage_transfer/transfer_check.py new file mode 100644 index 00000000000..5b1029819bb --- /dev/null +++ b/storage/storage_transfer/transfer_check.py @@ -0,0 +1,44 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import json +import logging + +import create_client + +# Edit these values with desired parameters. +PROJECT_ID = 'YOUR_PROJECT_ID' +JOB_NAME = 'YOUR_JOB_NAME' + + +def check_operation(transfer_service_client, project_id, job_name): + """Review the transfer operations associated with a transfer job.""" + filterString = ( + '{{"project_id": "{project_id}", ' + '"job_names": ["{job_name}"]}}').format( + project_id=project_id, job_name=job_name) + return transfer_service_client.transferOperations().list( + name="transferOperations", + filter=filterString).execute() + + +def main(): + logging.getLogger().setLevel(logging.DEBUG) + transfer_service_client = create_client.create_transfer_client() + + result = check_operation(transfer_service_client, PROJECT_ID, JOB_NAME) + logging.info('Result of transferOperations/list: %s', + json.dumps(result, indent=4, sort_keys=True)) + +if __name__ == '__main__': + main() From 89a1bd1b80e3f0eaa4c031c1251e932aa8c7a1c2 Mon Sep 17 00:00:00 2001 From: Shun Fan Date: Tue, 21 Jul 2015 22:44:55 -0700 Subject: [PATCH 2/3] Fix pep8 error --- storage/storage_transfer/test_create_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/storage_transfer/test_create_client.py b/storage/storage_transfer/test_create_client.py index d20b579db9d..c74cde2e16e 100644 --- a/storage/storage_transfer/test_create_client.py +++ b/storage/storage_transfer/test_create_client.py @@ -13,12 +13,12 @@ # import unittest +import create_client + from mock import Mock from mock import patch from oauth2client.client import GoogleCredentials -import create_client - class CheckCreateClientTestCase(unittest.TestCase): """A test case for client creation.""" From ecbae42755fab9d20bae45bf436c5fdd5f2d86cb Mon Sep 17 00:00:00 2001 From: Shun Fan Date: Wed, 22 Jul 2015 01:13:31 -0700 Subject: [PATCH 3/3] Update README.md with intructions on setting environment variable --- storage/storage_transfer/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/storage/storage_transfer/README.md b/storage/storage_transfer/README.md index 43b5aa03d4b..2c3f7762ff3 100644 --- a/storage/storage_transfer/README.md +++ b/storage/storage_transfer/README.md @@ -7,13 +7,15 @@ This app creates two types of transfers using the Transfer Service tool. 1. Set up a project on Google Developers Console. 1. Go to the [Developers Console](https://cloud.google.com/console) and create or select your project. You will need the project ID later. +1. Within Developers Console, select APIs & auth > Credentials. + 1. Add a new JSON key credential for a service account. + 1. Set the environment variable GOOGLE_APPLICATION_CREDENTIALS to point to your JSON key. 1. Add the Storage Transfer service account, cloud-mobility@system.gserviceaccount.com as an editor of your project. 1. Set up gcloud for application default credentials. 1. `gcloud components update` 1. `gcloud auth login` 1. `gcloud config set project PROJECT_ID` - 1. `export GOOGLE_APPLICATION_CREDENTIALS=PATH/TO/CREDENTIALS.json` 1. Install [Google API Client Library for Python](https://developers.google.com/api-client-library/python/start/installation). ## Transfer from Amazon S3 to Google Cloud Storage