Skip to content

Commit 69b19e2

Browse files
author
Shun Fan
committed
Add storagetransfer samples
1 parent d2818ba commit 69b19e2

File tree

9 files changed

+405
-0
lines changed

9 files changed

+405
-0
lines changed

‎storage/storage_transfer/README.md

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Transfer Service sample using Python
2+
3+
This app creates two types of transfers using the Transfer Service tool.
4+
5+
## Prerequisites
6+
7+
1. Set up a project on Google Developers Console.
8+
1. Go to the [Developers Console](https://cloud.google.com/console) and create or select your project.
9+
You will need the project ID later.
10+
1. Add the Storage Transfer service account, cloud-mobility@system.gserviceaccount.com as an
11+
editor of your project.
12+
1. Set up gcloud for application default credentials.
13+
1. `gcloud components update`
14+
1. `gcloud auth login`
15+
1. `gcloud config set project PROJECT_ID`
16+
1. `export GOOGLE_APPLICATION_CREDENTIALS=PATH/TO/CREDENTIALS.json`
17+
1. Install [Google API Client Library for Python](https://developers.google.com/api-client-library/python/start/installation).
18+
19+
## Transfer from Amazon S3 to Google Cloud Storage
20+
21+
Creating a one-time transfer from Amazon S3 to Google Cloud Storage.
22+
1. Set up data sink.
23+
1. Go to the Developers Console and create a bucket under Cloud Storage > Storage Browser.
24+
1. Set up data source.
25+
1. Go to AWS Management Console and create a bucket.
26+
1. Under Security Credentials, create an IAM User with access to the bucket.
27+
1. Create an Access Key for the user. Note the Access Key ID and Secret Access Key.
28+
1. In aws_request.py, fill in the Transfer Job JSON template with relevant values.
29+
1. Run with `python aws_request.py`
30+
1. Note the job ID in the returned Transfer Job.
31+
32+
## Transfer data from a standard Cloud Storage bucket to a Cloud Storage Nearline bucket
33+
34+
Creating a daily transfer from a standard Cloud Storage bucket to a Cloud Storage Nearline
35+
bucket for files untouched for 30 days.
36+
1. Set up data sink.
37+
1. Go to the Developers Console and create a bucket under Cloud Storage > Storage Browser.
38+
1. Select Nearline for Storage Class.
39+
1. Set up data source.
40+
1. Go to the Developers Console and create a bucket under Cloud Storage > Storage Browser.
41+
1. In nearline_request.py, fill in the Transfer Job JSON template with relevant values.
42+
1. Run with `python nearline_request.py`
43+
1. Note the job ID in the returned Transfer Job.
44+
45+
## Checking the status of a transfer
46+
47+
1. In transfer_check.py, fill in the Transfer Job JSON template with relevant values.
48+
Use the Job Name you recorded earlier.
49+
1. Run with `python transfer_check.py`

‎storage/storage_transfer/aws_request.py

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# Copyright 2015, Google, Inc.
2+
# Licensed under the Apache License, Version 2.0 (the "License");
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an "AS IS" BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
#
14+
import json
15+
import logging
16+
17+
import create_client
18+
19+
20+
def main():
21+
"""Create a one-off transfer from Amazon S3 to GCS."""
22+
logging.getLogger().setLevel(logging.DEBUG)
23+
transfer_service_client = create_client.create_transfer_client()
24+
25+
# Edit this template with desired parameters.
26+
# Specify times below using US Pacific Time Zone.
27+
transfer_job = '''
28+
{
29+
"description": "YOUR DESCRIPTION",
30+
"status": "ENABLED",
31+
"projectId": "YOUR_PROJECT_ID",
32+
"schedule": {
33+
"scheduleStartDate": {
34+
"day": 1,
35+
"month": 1,
36+
"year": 2015
37+
},
38+
"scheduleEndDate": {
39+
"day": 1,
40+
"month": 1,
41+
"year": 2015
42+
},
43+
"startTimeOfDay": {
44+
"hours": 0,
45+
"minutes": 0
46+
}
47+
},
48+
"transferSpec": {
49+
"awsS3DataSource": {
50+
"bucketName": "YOUR_SOURCE_BUCKET",
51+
"awsAccessKey": {
52+
"accessKeyId": "YOUR_ACCESS_KEY_ID",
53+
"secretAccessKey": "YOUR_SECRET_ACCESS_KEY"
54+
}
55+
},
56+
"gcsDataSink": {
57+
"bucketName": "YOUR_SINK_BUCKET"
58+
}
59+
}
60+
}
61+
'''
62+
63+
result = transfer_service_client.transferJobs().create(body=json.loads(
64+
transfer_job)).execute()
65+
logging.info('Returned transferJob: %s', json.dumps(result, indent=4))
66+
67+
if __name__ == '__main__':
68+
main()

‎storage/storage_transfer/create_client.py

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Copyright 2015, Google, Inc.
2+
# Licensed under the Apache License, Version 2.0 (the "License");
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an "AS IS" BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
#
14+
import logging
15+
16+
from apiclient import discovery
17+
from oauth2client.client import GoogleCredentials
18+
19+
CLOUD_SCOPES = 'https://www.googleapis.com/auth/cloud-platform'
20+
21+
22+
def create_transfer_client():
23+
"""Create a transfer client."""
24+
25+
logging.getLogger().setLevel(logging.DEBUG)
26+
credentials = GoogleCredentials.get_application_default()
27+
return discovery.build('storagetransfer', 'v1', credentials=credentials)

‎storage/storage_transfer/nearline_request.py

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# Copyright 2015, Google, Inc.
2+
# Licensed under the Apache License, Version 2.0 (the "License");
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an "AS IS" BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
#
14+
import json
15+
import logging
16+
17+
import create_client
18+
19+
20+
def main():
21+
"""Transfer from standard Cloud Storage to Cloud Storage Nearline."""
22+
logging.getLogger().setLevel(logging.DEBUG)
23+
transfer_service_client = create_client.create_transfer_client()
24+
25+
# Edit this template with desired parameters.
26+
# Specify times below using US Pacific Time Zone.
27+
transfer_job = '''
28+
{
29+
"description": "YOUR DESCRIPTION",
30+
"status": "ENABLED",
31+
"projectId": "YOUR_PROJECT_ID",
32+
"schedule": {
33+
"scheduleStartDate": {
34+
"day": 1,
35+
"month": 1,
36+
"year": 2015
37+
},
38+
"startTimeOfDay": {
39+
"hours": 1,
40+
"minutes": 1
41+
}
42+
},
43+
"transferSpec": {
44+
"gcsDataSource": {
45+
"bucketName": "YOUR_SOURCE_BUCKET"
46+
},
47+
"gcsDataSink": {
48+
"bucketName": "YOUR_SINK_BUCKET"
49+
},
50+
"objectConditions": {
51+
"minTimeElapsedSinceLastModification": "2592000s"
52+
},
53+
"transferOptions": {
54+
"deleteObjectsFromSourceAfterTransfer": true
55+
}
56+
}
57+
}
58+
'''
59+
result = transfer_service_client.transferJobs().create(body=json.loads(
60+
transfer_job)).execute()
61+
logging.info('Returned transferJob: %s', json.dumps(result, indent=4))
62+
63+
if __name__ == '__main__':
64+
main()

‎storage/storage_transfer/test_aws_request.py

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Copyright 2015, Google, Inc.
2+
# Licensed under the Apache License, Version 2.0 (the "License");
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an "AS IS" BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
#
14+
import unittest
15+
16+
from aws_request import main
17+
18+
from mock import Mock
19+
from mock import patch
20+
21+
22+
class AwsRequestTestCase(unittest.TestCase):
23+
"""A test case for creating a TransferJob from AWS S3."""
24+
25+
def setUp(self):
26+
patcher1 = patch(
27+
'storage.storage_transfer.aws_request.create_client')
28+
self.mock_create_client = patcher1.start()
29+
self.addCleanup(patcher1.stop)
30+
self.mock_client = Mock(spec=['transferJobs'])
31+
self.mock_create_client.create_transfer_client.return_value = \
32+
self.mock_client
33+
34+
def test_create_aws_request(self):
35+
execute = self.mock_client.transferJobs.return_value.create.return_value \
36+
.execute
37+
execute.return_value = ""
38+
main()
39+
execute.assert_called_with()

‎storage/storage_transfer/test_create_client.py

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# Copyright 2015, Google, Inc.
2+
# Licensed under the Apache License, Version 2.0 (the "License");
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an "AS IS" BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
#
14+
import unittest
15+
16+
from mock import Mock
17+
from mock import patch
18+
from oauth2client.client import GoogleCredentials
19+
20+
import create_client
21+
22+
23+
class CheckCreateClientTestCase(unittest.TestCase):
24+
"""A test case for client creation."""
25+
26+
def setUp(self):
27+
patcher1 = patch(
28+
'storage.storage_transfer.create_client.GoogleCredentials')
29+
patcher2 = patch(
30+
'storage.storage_transfer.create_client.discovery.build')
31+
self.mock_google_credentials = patcher1.start()
32+
self.mock_discovery = patcher2.start()
33+
self.addCleanup(patcher1.stop)
34+
self.addCleanup(patcher2.stop)
35+
36+
self.mock_credentials = Mock(spec=GoogleCredentials)
37+
self.mock_google_credentials.get_application_default.return_value = \
38+
self.mock_credentials
39+
40+
def test_create_client(self):
41+
create_client.create_transfer_client()
42+
self.mock_discovery.assert_called_with(
43+
'storagetransfer', 'v1',
44+
credentials=self.mock_credentials)

‎storage/storage_transfer/test_nearline_request.py

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Copyright 2015, Google, Inc.
2+
# Licensed under the Apache License, Version 2.0 (the "License");
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an "AS IS" BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
#
14+
import unittest
15+
16+
from mock import Mock
17+
from mock import patch
18+
19+
from nearline_request import main
20+
21+
22+
class NearlineRequestTestCase(unittest.TestCase):
23+
"""A test case for creating a TransferJob to Nearline for old files."""
24+
25+
def setUp(self):
26+
patcher1 = patch(
27+
'storage.storage_transfer.nearline_request.create_client')
28+
self.mock_create_client = patcher1.start()
29+
self.addCleanup(patcher1.stop)
30+
self.mock_client = Mock(spec=['transferJobs'])
31+
self.mock_create_client.create_transfer_client.return_value = \
32+
self.mock_client
33+
34+
def test_create_nearline_request(self):
35+
execute = self.mock_client.transferJobs.return_value.create.return_value \
36+
.execute
37+
execute.return_value = ""
38+
main()
39+
execute.assert_called_with()

‎storage/storage_transfer/test_transfer_check.py

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Copyright 2015, Google, Inc.
2+
# Licensed under the Apache License, Version 2.0 (the "License");
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
#
6+
# http://www.apache.org/licenses/LICENSE-2.0
7+
#
8+
# Unless required by applicable law or agreed to in writing, software
9+
# distributed under the License is distributed on an "AS IS" BASIS,
10+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
# See the License for the specific language governing permissions and
12+
# limitations under the License.
13+
#
14+
import unittest
15+
16+
from mock import Mock
17+
18+
from transfer_check import check_operation
19+
20+
21+
class CheckTransferTestCase(unittest.TestCase):
22+
"""A test case for querying transfer job completion."""
23+
24+
def test_check_operation(self):
25+
mock_client = Mock(spec=['transferOperations'])
26+
execute = mock_client.transferOperations.return_value.list.return_value \
27+
.execute
28+
project_id = ""
29+
job_name = ""
30+
check_operation(mock_client, project_id, job_name)
31+
execute.assert_called_with()

0 commit comments

Comments
 (0)