Skip to content

Commit a1c0d83

Browse files
anguillanneufanguillanneuf
authored
feat: add gradle build scripts to pubsub and pubsublite tutorials for beam (GoogleCloudPlatform#6586)
* feat: add gradle build script to pubsublite dataflow tutorial * add build.gradle to Pub/Sub Dataflow tutorial * address david's comments
1 parent 091a376 commit a1c0d83

File tree

6 files changed

+156
-5
lines changed

6 files changed

+156
-5
lines changed

‎pubsub/streaming-analytics/README.md

+13
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,19 @@ The following example will run a streaming pipeline. It will read messages from
104104
+ `--runner [optional]`: specifies the runner to run the pipeline, defaults to `DirectRunner`
105105
+ `--windowSize [optional]`: specifies the window size in minutes, defaults to 1
106106

107+
Gradle:
108+
```bash
109+
gradle execute -Dexec.args="\
110+
--project=$PROJECT_NAME \
111+
--region=$REGION \
112+
--inputTopic=projects/$PROJECT_NAME/topics/cron-topic \
113+
--output=gs://$BUCKET_NAME/samples/output \
114+
--gcpTempLocation=gs://$BUCKET_NAME/temp \
115+
--runner=DataflowRunner \
116+
--windowSize=2"
117+
```
118+
119+
Maven:
107120
```bash
108121
mvn compile exec:java \
109122
-Dexec.mainClass=com.examples.pubsub.streaming.PubSubToGcs \

‎pubsub/streaming-analytics/build.gradle

+60
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
/*
2+
* Copyright 2021 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
plugins {
18+
id 'java'
19+
}
20+
21+
repositories {
22+
mavenCentral()
23+
maven {
24+
url = uri('https://repository.apache.org/content/repositories/snapshots/')
25+
}
26+
27+
maven {
28+
url = uri('https://packages.confluent.io/maven/')
29+
}
30+
31+
maven {
32+
url = uri('https://repo.maven.apache.org/maven2/')
33+
}
34+
}
35+
36+
def beamVersion = '2.34.0'
37+
def slf4jVersion = '1.7.32'
38+
dependencies {
39+
implementation 'com.github.spotbugs:spotbugs-annotations:4.5.0'
40+
implementation "org.apache.beam:beam-sdks-java-core:${beamVersion}"
41+
implementation "org.apache.beam:beam-sdks-java-io-google-cloud-platform:${beamVersion}"
42+
implementation "org.apache.beam:beam-examples-java:${beamVersion}"
43+
implementation "org.slf4j:slf4j-api:${slf4jVersion}"
44+
implementation "org.slf4j:slf4j-jdk14:${slf4jVersion}"
45+
runtimeOnly "org.apache.beam:beam-runners-direct-java:${beamVersion}"
46+
runtimeOnly "org.apache.beam:beam-runners-google-cloud-dataflow-java:${beamVersion}"
47+
testImplementation 'com.google.cloud:google-cloud-core:2.3.2'
48+
}
49+
50+
group = 'com.example'
51+
version = '1.0.0-SNAPSHOT'
52+
description = 'pubsub-streaming'
53+
54+
task execute (type:JavaExec) {
55+
mainClass = 'com.examples.pubsub.streaming.PubSubToGcs'
56+
description('Run the Beam pipeline with gradle execute -Dexec.args=...')
57+
classpath = sourceSets.main.runtimeClasspath
58+
systemProperties System.getProperties()
59+
args System.getProperty("exec.args", "").split()
60+
}

‎pubsub/streaming-analytics/pom.xml

+1-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121

2222
<groupId>com.example</groupId>
2323
<artifactId>pubsub-streaming</artifactId>
24-
<version>1.0</version>
24+
<version>1.0.0-SNAPSHOT</version>
2525

2626
<parent>
2727
<groupId>com.google.cloud.samples</groupId>

‎pubsublite/streaming-analytics/README.md

+13
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,19 @@ The following example runs a streaming pipeline. Choose `DirectRunner` to test i
7373
+ `--region [optional]`: the Dataflow region, optional if using `DirectRunner`
7474
+ `--tempLocation`: a Cloud Storage location for temporary files, optional if using `DirectRunner`
7575

76+
Gradle:
77+
```sh
78+
gradle execute -Dexec.args="\
79+
--subscription=projects/$PROJECT_ID/locations/$LITE_LOCATION/subscriptions/$SUBSCRIPTION \
80+
--output=gs://$BUCKET/samples/output \
81+
--windowSize=1 \
82+
--runner=DataflowRunner \
83+
--project=$PROJECT_ID \
84+
--region=$DATAFLOW_REGION \
85+
--tempLocation=gs://$BUCKET/temp"
86+
```
87+
88+
Maven:
7689
```sh
7790
mvn compile exec:java \
7891
-Dexec.mainClass=examples.PubsubliteToGcs \

‎pubsublite/streaming-analytics/build.gradle

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* Copyright 2021 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
plugins {
18+
id 'java'
19+
}
20+
21+
repositories {
22+
mavenLocal()
23+
maven {
24+
url = uri('https://repository.apache.org/content/repositories/snapshots/')
25+
}
26+
27+
maven {
28+
url = uri('https://packages.confluent.io/maven/')
29+
}
30+
31+
maven {
32+
url = uri('https://repo.maven.apache.org/maven2/')
33+
}
34+
}
35+
36+
def beamVersion = '2.34.0'
37+
def slf4jVersion = '1.7.32'
38+
dependencies {
39+
implementation 'com.github.spotbugs:spotbugs-annotations:4.5.0'
40+
implementation "org.slf4j:slf4j-api:${slf4jVersion}"
41+
implementation "org.slf4j:slf4j-jdk14:${slf4jVersion}"
42+
implementation "org.apache.beam:beam-sdks-java-core:${beamVersion}"
43+
implementation "org.apache.beam:beam-sdks-java-io-google-cloud-platform:${beamVersion}"
44+
implementation "org.apache.beam:beam-examples-java:${beamVersion}"
45+
runtimeOnly "org.apache.beam:beam-runners-direct-java:${beamVersion}"
46+
runtimeOnly "org.apache.beam:beam-runners-google-cloud-dataflow-java:${beamVersion}"
47+
testImplementation 'com.google.api-client:google-api-client:1.32.2'
48+
testImplementation 'com.google.apis:google-api-services-dataflow:v1b3-rev20210825-1.32.1'
49+
testImplementation 'com.google.cloud:google-cloud-core:2.3.2'
50+
testImplementation 'com.google.cloud:google-cloud-storage:2.1.9'
51+
testImplementation 'com.google.truth:truth:1.1.3'
52+
testImplementation 'org.hamcrest:hamcrest-all:1.3'
53+
}
54+
55+
group = 'com.example'
56+
version = '1.0.0-SNAPSHOT'
57+
description = 'pubsublite-streaming'
58+
59+
task execute (type:JavaExec) {
60+
mainClass = 'examples.PubsubliteToGcs'
61+
description('Run the Beam pipeline with gradle execute -Dexec.args=...')
62+
classpath = sourceSets.main.runtimeClasspath
63+
systemProperties System.getProperties()
64+
args System.getProperty("exec.args", "").split()
65+
}

‎pubsublite/streaming-analytics/pom.xml

+4-4
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121

2222
<groupId>com.example</groupId>
2323
<artifactId>pubsublite-streaming</artifactId>
24-
<version>1.0</version>
24+
<version>1.0.0-SNAPSHOT</version>
2525

2626
<parent>
2727
<groupId>com.google.cloud.samples</groupId>
@@ -150,13 +150,13 @@
150150
<groupId>com.google.api-client</groupId>
151151
<artifactId>google-api-client</artifactId>
152152
<version>1.32.2</version>
153-
<scope>runtime</scope>
153+
<scope>test</scope>
154154
</dependency>
155155
<dependency>
156156
<groupId>com.google.apis</groupId>
157157
<artifactId>google-api-services-dataflow</artifactId>
158-
<version>v1b3-rev20210408-1.31.0</version>
159-
<scope>runtime</scope>
158+
<version>v1b3-rev20210825-1.32.1</version>
159+
<scope>test</scope>
160160
</dependency>
161161
</dependencies>
162162

0 commit comments

Comments
 (0)