Skip to content

Commit 702e9cf

Browse files
billyjacobsonbillyjacobson
authored
(docs) Bigtable test cleanup and additional workload generator test (GoogleCloudPlatform#7190)
* Add test to validate deployed template Cleanup non-canonical tests * Fix lint * Lint
1 parent 4bbc47c commit 702e9cf

File tree

9 files changed

+77
-27
lines changed

9 files changed

+77
-27
lines changed

‎bigtable/beam/workload-generator/README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ If you would like to modify this and run it yourself you can use these commands:
9090
9191
```
9292
mvn compile exec:java -Dexec.mainClass=WorkloadGenerator \
93-
"-Dexec.args=--bigtableInstanceId=$INSTANCE_ID =--bigtableTableId=$TABLE_ID \
93+
"-Dexec.args=--bigtableInstanceId=$INSTANCE_ID --bigtableTableId=$TABLE_ID \
9494
--runner=dataflow --project=$GOOGLE_CLOUD_PROJECT \
9595
--region=$REGION" \
9696
--workloadRate=$WORKLOAD_RATE

‎bigtable/beam/workload-generator/pom.xml

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,12 @@
8787
<groupId>org.apache.beam</groupId>
8888
<artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
8989
<version>${apache_beam.version}</version>
90+
<exclusions>
91+
<exclusion>
92+
<groupId>io.grpc</groupId>
93+
<artifactId>grpc-netty</artifactId>
94+
</exclusion>
95+
</exclusions>
9096
</dependency>
9197
<dependency>
9298
<groupId>org.apache.beam</groupId>
@@ -98,15 +104,21 @@
98104
<dependency>
99105
<groupId>com.google.guava</groupId>
100106
<artifactId>guava</artifactId>
101-
<version>31.0.1-jre</version>
107+
<version>31.1-jre</version>
102108
</dependency>
103109

104110
<dependency>
105111
<groupId>com.google.cloud.bigtable</groupId>
106112
<artifactId>bigtable-hbase-beam</artifactId>
107-
<version>2.2.0</version>
113+
<version>2.4.0</version>
108114
</dependency>
109115

116+
<dependency>
117+
<groupId>com.google.cloud</groupId>
118+
<artifactId>google-cloud-dataflow</artifactId>
119+
<version>0.6.0</version>
120+
<scope>test</scope>
121+
</dependency>
110122
<dependency>
111123
<groupId>junit</groupId>
112124
<artifactId>junit</artifactId>

‎bigtable/beam/workload-generator/src/main/java/bigtable/WorkloadGenerator.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ static PipelineResult generateWorkload(BigtableWorkloadOptions options) {
6868
ScheduledThreadPoolExecutor exec = new ScheduledThreadPoolExecutor(1);
6969
exec.schedule(() -> {
7070
try {
71+
System.out.println("Cancelling job.");
7172
cancelJob(options, (DataflowPipelineJob) pipelineResult);
7273
} catch (IOException e) {
7374
e.printStackTrace();

‎bigtable/beam/workload-generator/src/test/java/bigtable/WorkloadGeneratorTest.java

Lines changed: 41 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,10 @@
3131
import com.google.bigtable.repackaged.com.google.protobuf.util.Timestamps;
3232
import com.google.cloud.bigtable.beam.CloudBigtableTableConfiguration;
3333
import com.google.cloud.bigtable.hbase.BigtableConfiguration;
34+
import com.google.dataflow.v1beta3.FlexTemplatesServiceClient;
35+
import com.google.dataflow.v1beta3.LaunchFlexTemplateParameter;
36+
import com.google.dataflow.v1beta3.LaunchFlexTemplateRequest;
37+
import com.google.dataflow.v1beta3.LaunchFlexTemplateResponse;
3438
import java.io.ByteArrayOutputStream;
3539
import java.io.IOException;
3640
import java.io.PrintStream;
@@ -132,7 +136,7 @@ public void testGenerateWorkload() {
132136
p.run().waitUntilFinish();
133137

134138
String output = bout.toString();
135-
assertThat(output.contains("Connected to table"));
139+
assertThat(output).contains("Connected to table");
136140
}
137141

138142
@Test
@@ -181,13 +185,46 @@ public void testPipeline() throws IOException, InterruptedException {
181185
startRequestCount = ts.getPoints(0).getValue().getInt64Value();
182186
endRequestCount = ts.getPoints(ts.getPointsCount() - 1).getValue().getInt64Value();
183187
}
184-
assertThat(endRequestCount - startRequestCount > rate);
188+
assertThat(endRequestCount - startRequestCount > rate).isTrue();
185189

186-
// Stop the running job.
190+
// Ensure the job is stopped after duration.
187191
String jobId = ((DataflowPipelineJob) pipelineResult).getJobId();
188192
DataflowClient client = DataflowClient.create(options);
189193
Job job = client.getJob(jobId);
190194

191-
assertThat(job.getCurrentState().equals("JOB_STATE_CANCELLED"));
195+
assertThat(job.getCurrentState()).matches("JOB_STATE_CANCELLED");
196+
}
197+
198+
@Test
199+
public void testDeployedPipeline() throws IOException, InterruptedException {
200+
FlexTemplatesServiceClient flexTemplatesServiceClient =
201+
FlexTemplatesServiceClient.create();
202+
LaunchFlexTemplateRequest request =
203+
LaunchFlexTemplateRequest.newBuilder()
204+
.setProjectId(projectId)
205+
.setLaunchParameter(
206+
LaunchFlexTemplateParameter.newBuilder()
207+
.setContainerSpecGcsPath(
208+
"gs://cloud-bigtable-dataflow-templates/generate-workload.json")
209+
.setJobName("generate-workload" + UUID.randomUUID().toString().substring(0, 20))
210+
.putParameters("bigtableInstanceId", instanceId)
211+
.putParameters("bigtableTableId", TABLE_ID)
212+
.build())
213+
.build();
214+
215+
LaunchFlexTemplateResponse response = flexTemplatesServiceClient.launchFlexTemplate(request);
216+
217+
String jobId = response.getJob().getId();
218+
BigtableWorkloadOptions options = PipelineOptionsFactory.create()
219+
.as(BigtableWorkloadOptions.class);
220+
DataflowClient client = DataflowClient.create(options);
221+
222+
Thread.sleep(3 * 60 * 1000);
223+
Job job = client.getJob(jobId);
224+
assertThat(job.getCurrentState()).matches("JOB_STATE_RUNNING");
225+
226+
// Cancel job manually because test job never ends.
227+
job.setRequestedState("JOB_STATE_CANCELLED");
228+
client.updateJob(jobId, job);
192229
}
193230
}

‎bigtable/cassandra-migration-codelab/pom.xml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,5 +53,11 @@
5353
<version>4.13.2</version>
5454
<scope>test</scope>
5555
</dependency>
56+
<dependency>
57+
<groupId>com.google.truth</groupId>
58+
<artifactId>truth</artifactId>
59+
<version>1.1.3</version>
60+
<scope>test</scope>
61+
</dependency>
5662
</dependencies>
5763
</project>

‎bigtable/cassandra-migration-codelab/src/test/java/CassandraMigrationCodelabTest.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
* limitations under the License.
1515
*/
1616

17-
import static org.hamcrest.MatcherAssert.assertThat;
17+
import static com.google.common.truth.Truth.assertThat;
1818
import static org.junit.Assert.assertNotNull;
1919

2020
import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient;
@@ -23,7 +23,6 @@
2323
import java.io.ByteArrayOutputStream;
2424
import java.io.PrintStream;
2525
import java.util.UUID;
26-
import org.hamcrest.CoreMatchers;
2726
import org.junit.Test;
2827

2928
public class CassandraMigrationCodelabTest {
@@ -66,7 +65,7 @@ public void testRunDoesNotFail() throws Exception {
6665
cassandraMigrationCodelab.run();
6766

6867
String output = bout.toString();
69-
assertThat(output, CoreMatchers.not(CoreMatchers.containsString("Error during")));
68+
assertThat(output).doesNotContainMatch("Error during");
7069

7170
adminClient.deleteTable(TABLE_ID);
7271
}

‎bigtable/hbase/snippets/src/test/java/com/example/bigtable/ConfigureConnectionPoolTest.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,11 @@
1616

1717
package com.example.bigtable;
1818

19+
import static com.google.common.truth.Truth.assertThat;
1920
import static org.junit.Assert.assertNotNull;
20-
import static org.junit.Assert.assertThat;
2121

2222
import java.io.ByteArrayOutputStream;
2323
import java.io.PrintStream;
24-
import org.hamcrest.CoreMatchers;
2524
import org.junit.After;
2625
import org.junit.Before;
2726
import org.junit.BeforeClass;
@@ -65,6 +64,6 @@ public void testConfigureConnectionPool() {
6564
ConfigureConnectionPool.configureConnectionPool(projectId, instanceId);
6665

6766
String output = bout.toString();
68-
assertThat(output, CoreMatchers.containsString("Connected with pool size of 10"));
67+
assertThat(output).contains("Connected with pool size of 10");
6968
}
7069
}

‎bigtable/hbase/snippets/src/test/java/com/example/bigtable/WritesTest.java

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@
1616

1717
package com.example.bigtable;
1818

19+
import static com.google.common.truth.Truth.assertThat;
1920
import static org.junit.Assert.assertNotNull;
20-
import static org.junit.Assert.assertThat;
2121

2222
import com.google.cloud.bigtable.hbase.BigtableConfiguration;
2323
import java.io.ByteArrayOutputStream;
@@ -30,7 +30,6 @@
3030
import org.apache.hadoop.hbase.client.Connection;
3131
import org.apache.hadoop.hbase.client.Table;
3232
import org.apache.hadoop.hbase.util.Bytes;
33-
import org.hamcrest.CoreMatchers;
3433
import org.junit.AfterClass;
3534
import org.junit.Before;
3635
import org.junit.BeforeClass;
@@ -93,30 +92,30 @@ public void test1_WriteSimple() {
9392
WriteSimple.writeSimple(projectId, instanceId, TABLE_ID);
9493

9594
String output = bout.toString();
96-
assertThat(output, CoreMatchers.containsString("Successfully wrote row"));
95+
assertThat(output).contains("Successfully wrote row");
9796
}
9897

9998
@Test
10099
public void test2_WriteBatch() {
101100
WriteBatch.writeBatch(projectId, instanceId, TABLE_ID);
102101

103102
String output = bout.toString();
104-
assertThat(output, CoreMatchers.containsString("Successfully wrote 2 rows"));
103+
assertThat(output).contains("Successfully wrote 2 rows");
105104
}
106105

107106
@Test
108107
public void test3_WriteConditionally() {
109108
WriteConditionally.writeConditionally(projectId, instanceId, TABLE_ID);
110109

111110
String output = bout.toString();
112-
assertThat(output, CoreMatchers.containsString("Successfully updated row's os_name"));
111+
assertThat(output).contains("Successfully updated row's os_name");
113112
}
114113

115114
@Test
116115
public void test4_WriteIncrement() {
117116
WriteIncrement.writeIncrement(projectId, instanceId, TABLE_ID);
118117

119118
String output = bout.toString();
120-
assertThat(output, CoreMatchers.containsString("Successfully updated row"));
119+
assertThat(output).contains("Successfully updated row");
121120
}
122121
}

‎bigtable/memorystore/src/test/java/MemcachedTest.java

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,9 @@
1515
*/
1616

1717

18+
import static com.google.common.truth.Truth.assertThat;
1819
import static java.lang.Thread.sleep;
1920
import static org.junit.Assert.assertNotNull;
20-
import static org.junit.Assert.assertThat;
2121

2222
import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient;
2323
import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest;
@@ -26,8 +26,6 @@
2626
import java.io.ByteArrayOutputStream;
2727
import java.io.PrintStream;
2828
import java.util.UUID;
29-
import org.hamcrest.CoreMatchers;
30-
import org.hamcrest.Matcher;
3129
import org.junit.AfterClass;
3230
import org.junit.Before;
3331
import org.junit.BeforeClass;
@@ -130,17 +128,16 @@ public void testMemcached() throws InterruptedException {
130128
Memcached.main(null);
131129

132130
String output = bout.toString();
133-
assertThat(output, CoreMatchers.containsString("Value fetched from Bigtable: PQ2A.190405.003"));
131+
assertThat(output).contains("Value fetched from Bigtable: PQ2A.190405.003");
134132

135133
// retry (due to occasional flakiness) if we didn't yet get the result in the cache
136134
int retryCount = 0;
137-
Matcher<String> foundInCache =
138-
CoreMatchers.containsString("Value fetched from cache: PQ2A.190405.003");
139-
while (retryCount < 5 && !foundInCache.matches(output)) {
135+
String foundInCache = "Value fetched from cache: PQ2A.190405.003";
136+
while (retryCount < 5 && !output.contains(foundInCache)) {
140137
Memcached.main(null);
141138
output = bout.toString();
142139
retryCount++;
143140
}
144-
assertThat(output, foundInCache);
141+
assertThat(output).contains(foundInCache);
145142
}
146143
}

0 commit comments

Comments
 (0)