Skip to content

Commit 246d11e

Browse files
suztomosuztomo
authored
feat: migrating bigquerydatatransfer snippet (GoogleCloudPlatform#8441)
* feat: migrating bigquerydatatransfer snippet Migrating the snippet files from https://github.com/googleapis/java-bigquerydatatransfer/tree/f0f22ce/samples/snippets. * adding environment variable file
1 parent ea70078 commit 246d11e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+3626
-1
lines changed

‎.kokoro/tests/run_tests.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,8 @@ if [[ "$SCRIPT_DEBUG" != "true" ]]; then
8181
"java-cloud-sql-samples-secrets.txt" \
8282
"java-iam-samples-secrets.txt" \
8383
"java-scc-samples-secrets.txt" \
84-
"java-bigqueryconnection-samples-secrets.txt")
84+
"java-bigqueryconnection-samples-secrets.txt" \
85+
"java-bigquerydatatransfer-samples-secrets.txt")
8586

8687
# create secret dir
8788
mkdir -p "${KOKORO_GFILE_DIR}/secrets"

‎bigquery/bigquerydatatransfer/snippets/pom.xml

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
<?xml version='1.0' encoding='UTF-8'?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
<groupId>com.example.bigquery</groupId>
5+
<artifactId>bigquerydatatransfer-snippets</artifactId>
6+
<packaging>jar</packaging>
7+
<name>Google Cloud BigQuery Data Transfer Service Snippets</name>
8+
9+
<!--
10+
The parent pom defines common style checks and testing strategies for our samples.
11+
Removing or replacing it should not affect the execution of the samples in anyway.
12+
-->
13+
<parent>
14+
<groupId>com.google.cloud.samples</groupId>
15+
<artifactId>shared-configuration</artifactId>
16+
<version>1.2.0</version>
17+
</parent>
18+
19+
<properties>
20+
<maven.compiler.target>1.8</maven.compiler.target>
21+
<maven.compiler.source>1.8</maven.compiler.source>
22+
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
23+
</properties>
24+
25+
26+
<!-- [START bigquerydatatransfer_install_with_bom] -->
27+
<dependencyManagement>
28+
<dependencies>
29+
<dependency>
30+
<groupId>com.google.cloud</groupId>
31+
<artifactId>libraries-bom</artifactId>
32+
<version>26.18.0</version>
33+
<type>pom</type>
34+
<scope>import</scope>
35+
</dependency>
36+
</dependencies>
37+
</dependencyManagement>
38+
39+
<dependencies>
40+
<dependency>
41+
<groupId>com.google.cloud</groupId>
42+
<artifactId>google-cloud-bigquerydatatransfer</artifactId>
43+
</dependency>
44+
<!-- [END bigquerydatatransfer_install_with_bom] -->
45+
46+
<dependency>
47+
<groupId>com.google.protobuf</groupId>
48+
<artifactId>protobuf-java-util</artifactId>
49+
</dependency>
50+
51+
<!-- Test dependencies -->
52+
<dependency>
53+
<groupId>junit</groupId>
54+
<artifactId>junit</artifactId>
55+
<version>4.13.2</version>
56+
<scope>test</scope>
57+
</dependency>
58+
<dependency>
59+
<groupId>com.google.truth</groupId>
60+
<artifactId>truth</artifactId>
61+
<version>1.1.3</version>
62+
<scope>test</scope>
63+
</dependency>
64+
<dependency>
65+
<groupId>com.google.cloud</groupId>
66+
<artifactId>google-cloud-bigquery</artifactId>
67+
<scope>test</scope>
68+
</dependency>
69+
<dependency>
70+
<groupId>com.google.cloud</groupId>
71+
<artifactId>google-cloud-pubsub</artifactId>
72+
<scope>test</scope>
73+
</dependency>
74+
</dependencies>
75+
</project>

‎bigquery/bigquerydatatransfer/snippets/src/main/java/com/example/bigquerydatatransfer/CopyDataset.java

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example.bigquerydatatransfer;
18+
19+
// [START bigquerydatatransfer_copy_dataset]
20+
import com.google.api.gax.rpc.ApiException;
21+
import com.google.cloud.bigquery.datatransfer.v1.CreateTransferConfigRequest;
22+
import com.google.cloud.bigquery.datatransfer.v1.DataTransferServiceClient;
23+
import com.google.cloud.bigquery.datatransfer.v1.ProjectName;
24+
import com.google.cloud.bigquery.datatransfer.v1.TransferConfig;
25+
import com.google.protobuf.Struct;
26+
import com.google.protobuf.Value;
27+
import java.io.IOException;
28+
import java.util.HashMap;
29+
import java.util.Map;
30+
31+
// Sample to copy dataset from another gcp project
32+
public class CopyDataset {
33+
34+
public static void main(String[] args) throws IOException {
35+
// TODO(developer): Replace these variables before running the sample.
36+
final String destinationProjectId = "MY_DESTINATION_PROJECT_ID";
37+
final String destinationDatasetId = "MY_DESTINATION_DATASET_ID";
38+
final String sourceProjectId = "MY_SOURCE_PROJECT_ID";
39+
final String sourceDatasetId = "MY_SOURCE_DATASET_ID";
40+
Map<String, Value> params = new HashMap<>();
41+
params.put("source_project_id", Value.newBuilder().setStringValue(sourceProjectId).build());
42+
params.put("source_dataset_id", Value.newBuilder().setStringValue(sourceDatasetId).build());
43+
TransferConfig transferConfig =
44+
TransferConfig.newBuilder()
45+
.setDestinationDatasetId(destinationDatasetId)
46+
.setDisplayName("Your Dataset Copy Name")
47+
.setDataSourceId("cross_region_copy")
48+
.setParams(Struct.newBuilder().putAllFields(params).build())
49+
.setSchedule("every 24 hours")
50+
.build();
51+
copyDataset(destinationProjectId, transferConfig);
52+
}
53+
54+
public static void copyDataset(String projectId, TransferConfig transferConfig)
55+
throws IOException {
56+
try (DataTransferServiceClient dataTransferServiceClient = DataTransferServiceClient.create()) {
57+
ProjectName parent = ProjectName.of(projectId);
58+
CreateTransferConfigRequest request =
59+
CreateTransferConfigRequest.newBuilder()
60+
.setParent(parent.toString())
61+
.setTransferConfig(transferConfig)
62+
.build();
63+
TransferConfig config = dataTransferServiceClient.createTransferConfig(request);
64+
System.out.println("Copy dataset created successfully :" + config.getName());
65+
} catch (ApiException ex) {
66+
System.out.print("Copy dataset was not created." + ex.toString());
67+
}
68+
}
69+
}
70+
// [END bigquerydatatransfer_copy_dataset]

‎bigquery/bigquerydatatransfer/snippets/src/main/java/com/example/bigquerydatatransfer/CreateAdManagerTransfer.java

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example.bigquerydatatransfer;
18+
19+
// [START bigquerydatatransfer_create_admanager_transfer]
20+
import com.google.api.gax.rpc.ApiException;
21+
import com.google.cloud.bigquery.datatransfer.v1.CreateTransferConfigRequest;
22+
import com.google.cloud.bigquery.datatransfer.v1.DataTransferServiceClient;
23+
import com.google.cloud.bigquery.datatransfer.v1.ProjectName;
24+
import com.google.cloud.bigquery.datatransfer.v1.TransferConfig;
25+
import com.google.protobuf.Struct;
26+
import com.google.protobuf.Value;
27+
import java.io.IOException;
28+
import java.util.HashMap;
29+
import java.util.Map;
30+
31+
// Sample to create a ad manager(formerly DFP) transfer config
32+
public class CreateAdManagerTransfer {
33+
34+
public static void main(String[] args) throws IOException {
35+
// TODO(developer): Replace these variables before running the sample.
36+
final String projectId = "MY_PROJECT_ID";
37+
String datasetId = "MY_DATASET_ID";
38+
String bucket = "gs://cloud-sample-data";
39+
// the network_code can only be digits with length 1 to 15
40+
String networkCode = "12345678";
41+
Map<String, Value> params = new HashMap<>();
42+
params.put("bucket", Value.newBuilder().setStringValue(bucket).build());
43+
params.put("network_code", Value.newBuilder().setStringValue(networkCode).build());
44+
TransferConfig transferConfig =
45+
TransferConfig.newBuilder()
46+
.setDestinationDatasetId(datasetId)
47+
.setDisplayName("Your Ad Manager Config Name")
48+
.setDataSourceId("dfp_dt")
49+
.setParams(Struct.newBuilder().putAllFields(params).build())
50+
.build();
51+
createAdManagerTransfer(projectId, transferConfig);
52+
}
53+
54+
public static void createAdManagerTransfer(String projectId, TransferConfig transferConfig)
55+
throws IOException {
56+
try (DataTransferServiceClient client = DataTransferServiceClient.create()) {
57+
ProjectName parent = ProjectName.of(projectId);
58+
CreateTransferConfigRequest request =
59+
CreateTransferConfigRequest.newBuilder()
60+
.setParent(parent.toString())
61+
.setTransferConfig(transferConfig)
62+
.build();
63+
TransferConfig config = client.createTransferConfig(request);
64+
System.out.println("Ad manager transfer created successfully :" + config.getName());
65+
} catch (ApiException ex) {
66+
System.out.print("Ad manager transfer was not created." + ex.toString());
67+
}
68+
}
69+
}
70+
// [END bigquerydatatransfer_create_admanager_transfer]

‎bigquery/bigquerydatatransfer/snippets/src/main/java/com/example/bigquerydatatransfer/CreateAdsTransfer.java

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example.bigquerydatatransfer;
18+
19+
// [START bigquerydatatransfer_create_ads_transfer]
20+
import com.google.api.gax.rpc.ApiException;
21+
import com.google.cloud.bigquery.datatransfer.v1.CreateTransferConfigRequest;
22+
import com.google.cloud.bigquery.datatransfer.v1.DataTransferServiceClient;
23+
import com.google.cloud.bigquery.datatransfer.v1.ProjectName;
24+
import com.google.cloud.bigquery.datatransfer.v1.TransferConfig;
25+
import com.google.protobuf.Struct;
26+
import com.google.protobuf.Value;
27+
import java.io.IOException;
28+
import java.util.HashMap;
29+
import java.util.Map;
30+
31+
// Sample to create ads(formerly AdWords) transfer config
32+
public class CreateAdsTransfer {
33+
34+
public static void main(String[] args) throws IOException {
35+
// TODO(developer): Replace these variables before running the sample.
36+
final String projectId = "MY_PROJECT_ID";
37+
String datasetId = "MY_DATASET_ID";
38+
// the customer_id only allows digits and hyphen ('-').
39+
String customerId = "012-345-6789";
40+
String refreshWindow = "100";
41+
Map<String, Value> params = new HashMap<>();
42+
params.put("customer_id", Value.newBuilder().setStringValue(customerId).build());
43+
params.put("refreshWindow", Value.newBuilder().setStringValue(refreshWindow).build());
44+
TransferConfig transferConfig =
45+
TransferConfig.newBuilder()
46+
.setDestinationDatasetId(datasetId)
47+
.setDisplayName("Your Ads Transfer Config Name")
48+
.setDataSourceId("adwords")
49+
.setParams(Struct.newBuilder().putAllFields(params).build())
50+
.build();
51+
createAdsTransfer(projectId, transferConfig);
52+
}
53+
54+
public static void createAdsTransfer(String projectId, TransferConfig transferConfig)
55+
throws IOException {
56+
try (DataTransferServiceClient client = DataTransferServiceClient.create()) {
57+
ProjectName parent = ProjectName.of(projectId);
58+
CreateTransferConfigRequest request =
59+
CreateTransferConfigRequest.newBuilder()
60+
.setParent(parent.toString())
61+
.setTransferConfig(transferConfig)
62+
.build();
63+
TransferConfig config = client.createTransferConfig(request);
64+
System.out.println("Ads transfer created successfully :" + config.getName());
65+
} catch (ApiException ex) {
66+
System.out.print("Ads transfer was not created." + ex.toString());
67+
}
68+
}
69+
}
70+
// [END bigquerydatatransfer_create_ads_transfer]

‎bigquery/bigquerydatatransfer/snippets/src/main/java/com/example/bigquerydatatransfer/CreateAmazonS3Transfer.java

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.example.bigquerydatatransfer;
18+
19+
// [START bigquerydatatransfer_create_amazons3_transfer]
20+
import com.google.api.gax.rpc.ApiException;
21+
import com.google.cloud.bigquery.datatransfer.v1.CreateTransferConfigRequest;
22+
import com.google.cloud.bigquery.datatransfer.v1.DataTransferServiceClient;
23+
import com.google.cloud.bigquery.datatransfer.v1.ProjectName;
24+
import com.google.cloud.bigquery.datatransfer.v1.TransferConfig;
25+
import com.google.protobuf.Struct;
26+
import com.google.protobuf.Value;
27+
import java.io.IOException;
28+
import java.util.HashMap;
29+
import java.util.Map;
30+
31+
// Sample to create amazon s3 transfer config.
32+
public class CreateAmazonS3Transfer {
33+
34+
public static void main(String[] args) throws IOException {
35+
// TODO(developer): Replace these variables before running the sample.
36+
final String projectId = "MY_PROJECT_ID";
37+
String datasetId = "MY_DATASET_ID";
38+
String tableId = "MY_TABLE_ID";
39+
// Amazon S3 Bucket Uri with read role permission
40+
String sourceUri = "s3://your-bucket-name/*";
41+
String awsAccessKeyId = "MY_AWS_ACCESS_KEY_ID";
42+
String awsSecretAccessId = "AWS_SECRET_ACCESS_ID";
43+
String sourceFormat = "CSV";
44+
String fieldDelimiter = ",";
45+
String skipLeadingRows = "1";
46+
Map<String, Value> params = new HashMap<>();
47+
params.put(
48+
"destination_table_name_template", Value.newBuilder().setStringValue(tableId).build());
49+
params.put("data_path", Value.newBuilder().setStringValue(sourceUri).build());
50+
params.put("access_key_id", Value.newBuilder().setStringValue(awsAccessKeyId).build());
51+
params.put("secret_access_key", Value.newBuilder().setStringValue(awsSecretAccessId).build());
52+
params.put("source_format", Value.newBuilder().setStringValue(sourceFormat).build());
53+
params.put("field_delimiter", Value.newBuilder().setStringValue(fieldDelimiter).build());
54+
params.put("skip_leading_rows", Value.newBuilder().setStringValue(skipLeadingRows).build());
55+
TransferConfig transferConfig =
56+
TransferConfig.newBuilder()
57+
.setDestinationDatasetId(datasetId)
58+
.setDisplayName("Your Aws S3 Config Name")
59+
.setDataSourceId("amazon_s3")
60+
.setParams(Struct.newBuilder().putAllFields(params).build())
61+
.setSchedule("every 24 hours")
62+
.build();
63+
createAmazonS3Transfer(projectId, transferConfig);
64+
}
65+
66+
public static void createAmazonS3Transfer(String projectId, TransferConfig transferConfig)
67+
throws IOException {
68+
try (DataTransferServiceClient client = DataTransferServiceClient.create()) {
69+
ProjectName parent = ProjectName.of(projectId);
70+
CreateTransferConfigRequest request =
71+
CreateTransferConfigRequest.newBuilder()
72+
.setParent(parent.toString())
73+
.setTransferConfig(transferConfig)
74+
.build();
75+
TransferConfig config = client.createTransferConfig(request);
76+
System.out.println("Amazon s3 transfer created successfully :" + config.getName());
77+
} catch (ApiException ex) {
78+
System.out.print("Amazon s3 transfer was not created." + ex.toString());
79+
}
80+
}
81+
}
82+
// [END bigquerydatatransfer_create_amazons3_transfer]

0 commit comments

Comments
 (0)