Skip to content

Commit c4542a5

Browse files
author
Shun Fan
committed
Add storagetransfer samples
1 parent 105bcfd commit c4542a5

File tree

16 files changed

+1244
-20
lines changed

16 files changed

+1244
-20
lines changed

‎cloud-storage/storage-transfer/README.md

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# Transfer Service sample using Java
2+
3+
This app creates two types of transfers using the Transfer Service tool.
4+
5+
## Prerequisites
6+
7+
1. Set up a project on Google Developers Console.
8+
1. Go to the [Developers Console](https://cloud.google.com/console) and create or select your project.
9+
You will need the project ID later.
10+
1. Add the Storage Transfer service account, cloud-mobility@system.gserviceaccount.com as an
11+
editor of your project.
12+
1. Set up gcloud for application default credentials.
13+
1. `gcloud components update`
14+
1. `gcloud auth login`
15+
1. `gcloud config set project PROJECT_ID`
16+
1. `export GOOGLE_APPLICATION_CREDENTIALS=PATH/TO/CREDENTIALS.json`
17+
1. Install jar
18+
`mvn install:install-file -Dfile=libstoragetransfer-v1-java-public.jar \`
19+
`-DgroupId=com.google.storagetransfer.samples -DartifactId=libstoragetransfer -Dversion=1 -Dpackaging=jar`
20+
21+
## Transfer from Amazon S3 to Google Cloud Storage
22+
23+
Creating a one-time transfer from Amazon S3 to Google Cloud Storage.
24+
1. Set up data sink.
25+
1. Go to the Developers Console and create a bucket under Cloud Storage > Storage Browser.
26+
1. Set up data source.
27+
1. Go to AWS Management Console and create a bucket.
28+
1. Under Security Credentials, create an IAM User with access to the bucket.
29+
1. Create an Access Key for the user. Note the Access Key ID and Secret Access Key.
30+
1. In AwsRequester.java, fill in the user-provided constants.
31+
1. Run with `mvn compile` and
32+
`mvn exec:java -Dexec.mainClass="com.google.cloud.storage.storagetransfer.samples.AwsRequester"`
33+
1. Note the job ID in the returned Transfer Job.
34+
35+
## Transfer data from a standard Cloud Storage bucket to a Cloud Storage Nearline bucket
36+
37+
Creating a daily transfer from a standard Cloud Storage bucket to a Cloud Storage Nearline
38+
bucket for files untouched for 30 days.
39+
1. Set up data sink.
40+
1. Go to the Developers Console and create a bucket under Cloud Storage > Storage Browser.
41+
1. Select Nearline for Storage Class.
42+
1. Set up data source.
43+
1. Go to the Developers Console and create a bucket under Cloud Storage > Storage Browser.
44+
1. In NearlineRequester.java, fill in the user-provided constants.
45+
1. Run with `mvn compile` and
46+
`mvn exec:java -Dexec.mainClass="com.google.cloud.storage.storagetransfer.samples.NearlineRequester"`
47+
1. Note the job ID in the returned Transfer Job.
48+
49+
## Checking the status of a transfer
50+
51+
1. In RequestChecker.java, fill in the user-provided constants. Use the Job Name you recorded earlier.
52+
1. Run with `mvn compile` and
53+
`mvn exec:java -Dexec.mainClass="com.google.cloud.storage.storagetransfer.samples.RequestChecker"`

‎cloud-storage/storage-transfer/pom.xml

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Copyright (c) 2015 Google Inc. All Rights Reserved.
4+
~
5+
~ Licensed under the Apache License, Version 2.0 (the "License"); you
6+
~ may not use this file except in compliance with the License. You may
7+
~ obtain a copy of the License at
8+
~
9+
~ http://www.apache.org/licenses/LICENSE-2.0
10+
~
11+
~ Unless required by applicable law or agreed to in writing, software
12+
~ distributed under the License is distributed on an "AS IS" BASIS,
13+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
14+
~ implied. See the License for the specific language governing
15+
~ permissions and limitations under the License.
16+
-->
17+
18+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
19+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
20+
<modelVersion>4.0.0</modelVersion>
21+
<parent>
22+
<artifactId>doc-samples</artifactId>
23+
<groupId>com.google.cloud</groupId>
24+
<version>1.0.0</version>
25+
<relativePath>../..</relativePath>
26+
</parent>
27+
28+
<groupId>com.google.storagetransfer.samples</groupId>
29+
<artifactId>transfersample</artifactId>
30+
<version>0.1</version>
31+
<packaging>jar</packaging>
32+
33+
<name>transfersample</name>
34+
<url>http://maven.apache.org</url>
35+
36+
<properties>
37+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
38+
<powermock.version>1.6.2</powermock.version>
39+
</properties>
40+
41+
<dependencies>
42+
<dependency>
43+
<groupId>com.google.apis</groupId>
44+
<artifactId>google-api-services-storagetransfer</artifactId>
45+
<version>v1-rev1-1.20.0</version>
46+
</dependency>
47+
48+
<!-- Test Dependencies -->
49+
<dependency>
50+
<groupId>org.powermock</groupId>
51+
<artifactId>powermock-module-junit4</artifactId>
52+
<version>${powermock.version}</version>
53+
<scope>test</scope>
54+
</dependency>
55+
<dependency>
56+
<groupId>org.powermock</groupId>
57+
<artifactId>powermock-api-mockito</artifactId>
58+
<version>${powermock.version}</version>
59+
<scope>test</scope>
60+
</dependency>
61+
</dependencies>
62+
<build>
63+
<plugins>
64+
<plugin>
65+
<groupId>org.apache.maven.plugins</groupId>
66+
<artifactId>maven-compiler-plugin</artifactId>
67+
<version>3.1</version>
68+
<configuration>
69+
<source>1.7</source>
70+
<target>1.7</target>
71+
</configuration>
72+
</plugin>
73+
</plugins>
74+
</build>
75+
</project>

‎cloud-storage/storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/AwsRequester.java

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
/**
2+
* Copyright 2015 Google Inc. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage.storagetransfer.samples;
18+
19+
import com.google.api.services.storagetransfer.Storagetransfer;
20+
import com.google.api.services.storagetransfer.model.AwsAccessKey;
21+
import com.google.api.services.storagetransfer.model.AwsS3Data;
22+
import com.google.api.services.storagetransfer.model.Date;
23+
import com.google.api.services.storagetransfer.model.GcsData;
24+
import com.google.api.services.storagetransfer.model.Schedule;
25+
import com.google.api.services.storagetransfer.model.TimeOfDay;
26+
import com.google.api.services.storagetransfer.model.TransferJob;
27+
import com.google.api.services.storagetransfer.model.TransferSpec;
28+
29+
import java.io.IOException;
30+
import java.util.logging.Logger;
31+
32+
/**
33+
* Creates a one-off transfer job from Amazon S3 to Google Cloud Storage.
34+
*
35+
*/
36+
public final class AwsRequester {
37+
38+
private static final String JOB_DESC = "YOUR DESCRIPTION";
39+
private static final String PROJECT_ID = "YOUR_PROJECT_ID";
40+
private static final String AWS_SOURCE_NAME = "YOUR SOURCE BUCKET";
41+
private static final String AWS_ACCESS_KEY_ID = "YOUR_ACCESS_KEY_ID";
42+
private static final String AWS_SECRET_ACCESS_KEY = "YOUR_SECRET_ACCESS_KEY";
43+
private static final String GCS_SINK_NAME = "YOUR_SINK_BUCKET";
44+
private static final String START_DATE = "YYYY-MM-DD";
45+
private static final String START_TIME = "HH:MM:SS";
46+
47+
private static final Logger LOG = Logger.getLogger(AwsRequester.class.getName());
48+
49+
/**
50+
* Creates and executes a request for a TransferJob from Amazon S3 to Cloud Storage.
51+
*
52+
* @return the response TransferJob if the request is successful
53+
* @throws InstantiationException
54+
* if instantiation fails when building the TransferJob
55+
* @throws IllegalAccessException
56+
* if an illegal access occurs when building the TransferJob
57+
* @throws IOException
58+
* if the client failed to complete the request
59+
*/
60+
public static TransferJob createAwsTransferJob() throws InstantiationException,
61+
IllegalAccessException, IOException {
62+
Date date = TransferJobUtils.createDate(START_DATE);
63+
TimeOfDay time = TransferJobUtils.createTimeOfDay(START_TIME);
64+
TransferJob transferJob = TransferJob.class
65+
.newInstance()
66+
.setDescription(JOB_DESC)
67+
.setProjectId(PROJECT_ID)
68+
.setTransferSpec(
69+
TransferSpec.class
70+
.newInstance()
71+
.setAwsS3DataSource(
72+
AwsS3Data.class
73+
.newInstance()
74+
.setBucketName(AWS_SOURCE_NAME)
75+
.setAwsAccessKey(
76+
AwsAccessKey.class.newInstance().setAccessKeyId(AWS_ACCESS_KEY_ID)
77+
.setSecretAccessKey(AWS_SECRET_ACCESS_KEY)))
78+
.setGcsDataSink(GcsData.class.newInstance().setBucketName(GCS_SINK_NAME)))
79+
.setSchedule(
80+
Schedule.class.newInstance().setScheduleStartDate(date).setScheduleEndDate(date)
81+
.setStartTimeOfDay(time)).setStatus("ENABLED");
82+
83+
Storagetransfer client = TransferClientCreator.createStorageTransferClient();
84+
return client.transferJobs().create(transferJob).execute();
85+
}
86+
87+
/**
88+
* Output the contents of a successfully created TransferJob.
89+
*
90+
* @param args
91+
* arguments from the command line
92+
*/
93+
public static void main(String[] args) {
94+
try {
95+
TransferJob responseT = createAwsTransferJob();
96+
LOG.info("Return transferJob: " + responseT.toPrettyString());
97+
} catch (Exception e) {
98+
e.printStackTrace();
99+
}
100+
}
101+
}

‎cloud-storage/storage-transfer/src/main/java/com/google/cloud/storage/storagetransfer/samples/NearlineRequester.java

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
/**
2+
* Copyright 2015 Google Inc. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.storage.storagetransfer.samples;
18+
19+
import com.google.api.services.storagetransfer.Storagetransfer;
20+
import com.google.api.services.storagetransfer.model.Date;
21+
import com.google.api.services.storagetransfer.model.GcsData;
22+
import com.google.api.services.storagetransfer.model.ObjectConditions;
23+
import com.google.api.services.storagetransfer.model.Schedule;
24+
import com.google.api.services.storagetransfer.model.TimeOfDay;
25+
import com.google.api.services.storagetransfer.model.TransferJob;
26+
import com.google.api.services.storagetransfer.model.TransferOptions;
27+
import com.google.api.services.storagetransfer.model.TransferSpec;
28+
29+
import java.io.IOException;
30+
import java.util.logging.Logger;
31+
32+
/**
33+
* Creates a daily transfer from a standard Cloud Storage bucket to a Cloud Storage Nearline
34+
* bucket for files untouched for 30 days.
35+
*
36+
*/
37+
public final class NearlineRequester {
38+
39+
private static final String JOB_DESC = "YOUR DESCRIPTION";
40+
private static final String PROJECT_ID = "YOUR_PROJECT_ID";
41+
private static final String GCS_SOURCE_NAME = "YOUR_SOURCE_BUCKET";
42+
private static final String NEARLINE_SINK_NAME = "YOUR_SINK_BUCKET";
43+
private static final String START_DATE = "YYYY-MM-DD";
44+
private static final String START_TIME = "HH:MM:SS";
45+
46+
private static final Logger LOG = Logger.getLogger(AwsRequester.class.getName());
47+
48+
/**
49+
* Creates and executes a request for a TransferJob to Cloud Storage Nearline.
50+
*
51+
* @return the response TransferJob if the request is successful
52+
* @throws InstantiationException
53+
* if instantiation fails when building the TransferJob
54+
* @throws IllegalAccessException
55+
* if an illegal access occurs when building the TransferJob
56+
* @throws IOException
57+
* if the client failed to complete the request
58+
*/
59+
public static TransferJob createNearlineTransferJob() throws InstantiationException,
60+
IllegalAccessException, IOException {
61+
Date date = TransferJobUtils.createDate(START_DATE);
62+
TimeOfDay time = TransferJobUtils.createTimeOfDay(START_TIME);
63+
TransferJob transferJob = TransferJob.class
64+
.newInstance()
65+
.setDescription(JOB_DESC)
66+
.setProjectId(PROJECT_ID)
67+
.setTransferSpec(
68+
TransferSpec.class
69+
.newInstance()
70+
.setGcsDataSource(GcsData.class.newInstance().setBucketName(GCS_SOURCE_NAME))
71+
.setGcsDataSink(GcsData.class.newInstance().setBucketName(NEARLINE_SINK_NAME))
72+
.setObjectConditions(
73+
ObjectConditions.class.newInstance().setMinTimeElapsedSinceLastModification("2592000s"))
74+
.setTransferOptions(
75+
TransferOptions.class.newInstance().setDeleteObjectsFromSourceAfterTransfer(true)))
76+
.setSchedule(Schedule.class.newInstance().setScheduleStartDate(date)
77+
.setStartTimeOfDay(time))
78+
.setStatus("ENABLED");
79+
80+
Storagetransfer client = TransferClientCreator.createStorageTransferClient();
81+
return client.transferJobs().create(transferJob).execute();
82+
}
83+
84+
/**
85+
* Output the contents of a successfully created TransferJob.
86+
*
87+
* @param args
88+
* arguments from the command line
89+
*/
90+
public static void main(String[] args) {
91+
try {
92+
TransferJob responseT = createNearlineTransferJob();
93+
LOG.info("Return transferJob: " + responseT.toPrettyString());
94+
} catch (Exception e) {
95+
e.printStackTrace();
96+
}
97+
}
98+
}

0 commit comments

Comments
 (0)