Skip to content

Commit 2cb07bb

Browse files
billyjacobsonbillyjacobson
committed
more configs
1 parent d7c554f commit 2cb07bb

File tree

1 file changed

+38
-40
lines changed

1 file changed

+38
-40
lines changed

‎bigtable/beam/keyviz-art/src/main/java/keyviz/FSReadData2.java

+38-40
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import com.google.cloud.datastore.StructuredQuery.CompositeFilter;
1717
import com.google.cloud.datastore.StructuredQuery.PropertyFilter;
1818
import java.util.ArrayList;
19+
import java.util.Arrays;
1920
import java.util.Date;
2021
import java.util.List;
2122
import java.util.Random;
@@ -47,7 +48,7 @@ public static void main(String[] args) {
4748

4849
Pipeline p = Pipeline.create(options);
4950
// Initiates a new pipeline every second
50-
p.apply(GenerateSequence.from(0).withRate(1, new Duration(1000)))
51+
p.apply(GenerateSequence.from(0).withRate(1, new Duration(2000)))
5152
.apply(ParDo.of(new ReadFromTableFn()));
5253

5354
p.run();
@@ -63,16 +64,16 @@ public static class ReadFromTableFn extends DoFn<Long, Void> {
6364
protected Datastore datastore;
6465
private KeyFactory keyFactory;
6566
// private final String kind = "BillyDF500reversed1mswithP";
66-
private final String kind = "BillyDFrw1s500rowsWithFetch";
67+
private final String kind = "BillyDF_WRFP_2s500krows";
6768

68-
private static final int RANDOM_ID_BOUND = 500;
69+
private static final int RANDOM_ID_BOUND = 500000;
6970
Random rand = new Random();
7071

7172
public ReadFromTableFn() {
7273
super();
74+
keys = new String[RANDOM_ID_BOUND];
7375
// downloadImageData(readDataOptions.getFilePath());
74-
// generateRowkeys(getNumRows(readDataOptions));
75-
// generateRowkeys(1000);
76+
generateRowkeys();
7677
}
7778

7879
protected synchronized Datastore getDatastore() {
@@ -100,26 +101,27 @@ public void processElement(@Element Long count, OutputReceiver<Void> out) {
100101
int c = 0;
101102

102103
if (count == 1) {
103-
for (int i = 0; i < maxInput; i++) {
104+
for (int i = 0; i < keys.length; i++) {
104105

105-
String paddedRowkey = String.format(numberFormat, i);
106-
String reversedRowkey = new StringBuilder(paddedRowkey).reverse().toString();
107-
int rowNumber = i / rowHeight;
108-
109-
float p = (float) rowNumber / numRows;
110-
System.out.printf("index: %d rowNumber %d prob %f count %d", i, rowNumber, p, count);
106+
// String paddedRowkey = String.format(numberFormat, i);
107+
// String reversedRowkey = new StringBuilder(paddedRowkey).reverse().toString();
108+
// int rowNumber = i / rowHeight;
109+
//
110+
// float p = (float) rowNumber / numRows;
111+
// System.out.printf("index: %d rowNumber %d prob %f count %d", i, rowNumber, p, count);
111112

112113
FullEntity<com.google.cloud.datastore.Key> fullEntity =
113-
com.google.cloud.datastore.Entity.newBuilder(keyFactory.newKey(reversedRowkey))
114+
com.google.cloud.datastore.Entity.newBuilder(keyFactory.newKey(keys[i]))
114115
.set("description", getRandomStringValue())
115116
.build();
116117
entities.add(fullEntity);
117118
// logger.info("writing entity" + fullEntity.getKey());
118119
c++;
119120

121+
// ds.put(fullEntity);
122+
120123
// 500 per write limit
121124
if (i % 500 == 0) {
122-
System.out.printf("500 mod 0");
123125
ds.put(entities.toArray(new FullEntity<?>[0]));
124126
entities = new ArrayList<>();
125127
}
@@ -138,8 +140,8 @@ public void processElement(@Element Long count, OutputReceiver<Void> out) {
138140
// } else {
139141
// query.setFilter(evenRangeFilter);
140142
// }
141-
QueryResults<Entity> entityQueryResults = ds.run(query.build());
142-
c = 0;
143+
// QueryResults<Entity> entityQueryResults = ds.run(query.build());
144+
// c = 0;
143145
// while (entityQueryResults.hasNext()) {
144146
// Entity entity = entityQueryResults.next();
145147
// System.out.println(entity);
@@ -159,31 +161,13 @@ public void processElement(@Element Long count, OutputReceiver<Void> out) {
159161
c++;
160162
}
161163
}
162-
datastore.fetch();
163-
System.out.println(c + " entities fetched");
164+
List<Entity> fetchedEntities = datastore.fetch();
165+
for (Entity e : fetchedEntities) {
166+
// System.out.println(e);
167+
c++;
168+
}
164169

165-
// String kind = "Billy10ms1krs";
166-
// int maxInput = 1000;
167-
// int maxLength = ("" + maxInput).length();
168-
// // Make each number the same length by padding with 0s
169-
// String numberFormat = "%0" + maxLength + "d";
170-
//
171-
// for (int i = 0; i < maxInput; i++) {
172-
// byte[] b = new byte[(int) 1000];
173-
// new Random().nextBytes(b);
174-
//
175-
// String paddedRowkey = String.format(numberFormat, i);
176-
// String reversedRowkey = new StringBuilder(paddedRowkey).reverse().toString();
177-
//
178-
// Entity entity = Entity.newBuilder()
179-
// .setKey(makeKey(kind, "testing" + reversedRowkey))
180-
// .putProperties(
181-
// "long",
182-
// makeValue(b.toString()).setExcludeFromIndexes(true).build()
183-
// ).build();
184-
//
185-
// out.output(entity);
186-
// }
170+
System.out.println(c + " entities fetched");
187171
}
188172

189173
private StringValue getRandomStringValue() {
@@ -201,6 +185,20 @@ private StringValue getRandomStringValue() {
201185
private int randomId(Random rand) {
202186
return rand.nextInt(RANDOM_ID_BOUND) + 1;
203187
}
188+
189+
private void generateRowkeys() {
190+
int maxLength = ("" + RANDOM_ID_BOUND).length();
191+
// Make each number the same length by padding with 0s
192+
String numberFormat = "%0" + maxLength + "d";
193+
194+
for (int i = 0; i < RANDOM_ID_BOUND; i++) {
195+
String paddedRowkey = String.format(numberFormat, i);
196+
String reversedRowkey = new StringBuilder(paddedRowkey).reverse().toString();
197+
keys[i] = "" + reversedRowkey;
198+
}
199+
Arrays.sort(keys);
200+
}
201+
204202
}
205203

206204
public interface ReadDataOptions extends BigtableOptions {

0 commit comments

Comments
 (0)