Skip to content

Commit df6cf91

Browse files
eliranmoyaleliranmoyal
committed
Merge branch 'master' into elastic2.0
Conflicts: pom.xml src/main/java/org/nlpcn/es4sql/query/maker/AggMaker.java
2 parents de72b2f + 502c012 commit df6cf91

File tree

7 files changed

+116
-40
lines changed

7 files changed

+116
-40
lines changed

‎src/main/java/org/nlpcn/es4sql/domain/MethodField.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,4 +53,14 @@ public void setOption(String option) {
5353
this.option = option;
5454
}
5555

56+
@Override
57+
public boolean isNested() {
58+
return this.getParamsAsMap().containsKey("nested");
59+
}
60+
61+
@Override
62+
public String getNestedPath() {
63+
if(!this.isNested()) return null;
64+
return this.getParamsAsMap().get("nested").toString();
65+
}
5666
}

‎src/main/java/org/nlpcn/es4sql/parse/FieldMaker.java

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -146,11 +146,19 @@ private static MethodField makeMethodField(String name, List<SQLExpr> arguments,
146146
}
147147
} else if(object instanceof SQLMethodInvokeExpr) {
148148
SQLMethodInvokeExpr mExpr = (SQLMethodInvokeExpr) object;
149-
if(mExpr.getMethodName().toLowerCase().equals("script")){
149+
String methodName = mExpr.getMethodName().toLowerCase();
150+
if(methodName.equals("script")){
150151
KVValue script = new KVValue("script", makeMethodField(mExpr.getMethodName(), mExpr.getParameters(), null, alias));
151152
paramers.add(script);
152153
}
153-
else throw new SqlParseException("only support script as nested functions");
154+
else if(methodName.equals("nested")){
155+
NestedType nestedType = new NestedType();
156+
if(!nestedType.tryFillFromExpr(object)){
157+
throw new SqlParseException("failed parsing nested expr " + object);
158+
}
159+
paramers.add(new KVValue("nested",nestedType));
160+
}
161+
else throw new SqlParseException("only support script/nested as inner functions");
154162
}else {
155163
paramers.add(new KVValue(Util.expr2Object(object)));
156164
}

‎src/main/java/org/nlpcn/es4sql/query/AggregationQueryAction.java

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ public SqlElasticSearchRequestBuilder explain() throws SqlParseException {
5959
}
6060

6161
if(field.isNested()){
62-
NestedBuilder nestedBuilder = AggregationBuilders.nested(field.getName() + "@NESTED")
62+
NestedBuilder nestedBuilder = AggregationBuilders.nested(getNestedAggName(field))
6363
.path(field.getNestedPath());
6464
if(insertFilterIfExistsAfter(lastAgg, groupBy, nestedBuilder,1)){
6565
groupBy.remove(1);
@@ -73,7 +73,6 @@ public SqlElasticSearchRequestBuilder explain() throws SqlParseException {
7373
request.addAggregation(lastAgg);
7474
}
7575

76-
7776
for (int i = 1; i < groupBy.size(); i++) {
7877
field = groupBy.get(i);
7978
AggregationBuilder<?> subAgg = aggMaker.makeGroupAgg(field);
@@ -83,7 +82,7 @@ public SqlElasticSearchRequestBuilder explain() throws SqlParseException {
8382

8483
if(field.isNested()){
8584

86-
NestedBuilder nestedBuilder = AggregationBuilders.nested(field.getName() + "@NESTED")
85+
NestedBuilder nestedBuilder = AggregationBuilders.nested(getNestedAggName(field))
8786
.path(field.getNestedPath());
8887
if(insertFilterIfExistsAfter(subAgg, groupBy, nestedBuilder,i+1)){
8988
groupBy.remove(i+1);
@@ -144,6 +143,18 @@ public SqlElasticSearchRequestBuilder explain() throws SqlParseException {
144143
return sqlElasticRequestBuilder;
145144
}
146145

146+
private String getNestedAggName(Field field) {
147+
String prefix;
148+
if(field instanceof MethodField){
149+
prefix = field.getNestedPath();
150+
}
151+
else {
152+
prefix = field.getName();
153+
}
154+
return prefix + "@NESTED";
155+
}
156+
157+
147158
private boolean insertFilterIfExistsAfter(AggregationBuilder<?> agg, List<Field> groupBy, NestedBuilder builder, int nextPosition) throws SqlParseException {
148159
if(groupBy.size() <= nextPosition) return false;
149160
Field filterFieldCandidate = groupBy.get(nextPosition);

‎src/main/java/org/nlpcn/es4sql/query/maker/AggMaker.java

Lines changed: 34 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
import org.elasticsearch.search.aggregations.bucket.range.RangeBuilder;
2222
import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeBuilder;
2323
import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder;
24-
import org.elasticsearch.search.aggregations.metrics.MetricsAggregationBuilder;
2524
import org.elasticsearch.search.aggregations.metrics.ValuesSourceMetricsAggregationBuilder;
2625
import org.elasticsearch.search.aggregations.metrics.scripted.ScriptedMetricBuilder;
2726
import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsBuilder;
@@ -32,6 +31,7 @@
3231
import org.nlpcn.es4sql.domain.MethodField;
3332
import org.nlpcn.es4sql.domain.Where;
3433
import org.nlpcn.es4sql.exception.SqlParseException;
34+
import org.nlpcn.es4sql.parse.NestedType;
3535

3636
public class AggMaker {
3737

@@ -77,32 +77,25 @@ public AbstractAggregationBuilder makeFieldAgg(MethodField field, AbstractAggreg
7777
switch (field.getName().toUpperCase()) {
7878
case "SUM":
7979
builder = AggregationBuilders.sum(field.getAlias());
80-
addFieldOrScriptToAggregation(field, builder);
81-
return builder;
80+
return addFieldToAgg(field, builder);
8281
case "MAX":
8382
builder = AggregationBuilders.max(field.getAlias());
84-
addFieldOrScriptToAggregation(field, builder);
85-
return builder;
83+
return addFieldToAgg(field, builder);
8684
case "MIN":
8785
builder = AggregationBuilders.min(field.getAlias());
88-
addFieldOrScriptToAggregation(field, builder);
89-
return builder;
86+
return addFieldToAgg(field, builder);
9087
case "AVG":
9188
builder = AggregationBuilders.avg(field.getAlias());
92-
addFieldOrScriptToAggregation(field, builder);
93-
return builder;
89+
return addFieldToAgg(field, builder);
9490
case "STATS":
9591
builder = AggregationBuilders.stats(field.getAlias());
96-
addFieldOrScriptToAggregation(field, builder);
97-
return builder;
92+
return addFieldToAgg(field, builder);
9893
case "EXTENDED_STATS":
9994
builder = AggregationBuilders.extendedStats(field.getAlias());
100-
addFieldOrScriptToAggregation(field, builder);
101-
return builder;
95+
return addFieldToAgg(field, builder);
10296
case "PERCENTILES":
10397
builder = AggregationBuilders.percentiles(field.getAlias());
104-
addFieldOrScriptToAggregation(field, builder);
105-
return builder;
98+
return addFieldToAgg(field, builder);
10699
case "TOPHITS":
107100
return makeTopHitsAgg(field);
108101
case "SCRIPTED_METRIC":
@@ -120,15 +113,20 @@ private String fixAlias(String alias) {
120113
return alias.replaceAll("\\[","(").replaceAll("\\]",")");
121114
}
122115

123-
private void addFieldOrScriptToAggregation(MethodField field, ValuesSourceMetricsAggregationBuilder builder) {
116+
private AbstractAggregationBuilder addFieldToAgg(MethodField field, ValuesSourceMetricsAggregationBuilder builder) {
124117
KVValue kvValue = field.getParams().get(0);
125-
if(kvValue.key==null || !kvValue.key.equals("script") )
126-
builder.field(kvValue.toString());
127-
else
118+
if (kvValue.key != null && kvValue.key.equals("script")){
119+
return builder.script(new Script(((MethodField) kvValue.value).getParams().get(1).toString()));
120+
}
121+
else if (kvValue.key != null && kvValue.key.equals("nested"))
128122
{
129-
//todo: support different lang script
130-
builder.script(new Script(((MethodField)kvValue.value).getParams().get(1).toString()));
123+
NestedType nestedType = (NestedType) kvValue.value;
124+
builder.field(nestedType.field);
125+
return AggregationBuilders.nested(nestedType.field+"@NESTED").path(nestedType.path).subAggregation(builder);
126+
131127
}
128+
129+
return builder.field(kvValue.toString());
132130
}
133131

134132
private AggregationBuilder<?> makeRangeGroup(MethodField field) throws SqlParseException {
@@ -178,39 +176,40 @@ private AbstractAggregationBuilder scriptedMetric(MethodField field) throws SqlP
178176
scriptedMetricBuilder.mapScript(new Script(paramValue));
179177
break;
180178
case "map_script_id":
181-
scriptedMetricBuilder.mapScript(new Script(paramValue, ScriptService.ScriptType.INDEXED,null,null));
179+
scriptedMetricBuilder.mapScript(new Script(paramValue, ScriptService.ScriptType.INDEXED, null, null));
182180
break;
183181
case "map_script_file":
184-
scriptedMetricBuilder.mapScript(new Script(paramValue, ScriptService.ScriptType.FILE,null,null));
182+
scriptedMetricBuilder.mapScript(new Script(paramValue, ScriptService.ScriptType.FILE, null, null));
185183
break;
186184
case "init_script":
187185
scriptedMetricBuilder.initScript(new Script(paramValue));
188186
break;
189187
case "init_script_id":
190-
scriptedMetricBuilder.initScript(new Script(paramValue, ScriptService.ScriptType.INDEXED,null,null));
188+
scriptedMetricBuilder.initScript(new Script(paramValue, ScriptService.ScriptType.INDEXED, null, null));
191189
break;
192190
case "init_script_file":
193-
scriptedMetricBuilder.initScript(new Script(paramValue, ScriptService.ScriptType.FILE,null,null));
191+
scriptedMetricBuilder.initScript(new Script(paramValue, ScriptService.ScriptType.FILE, null, null));
194192
break;
195193
case "combine_script":
196194
scriptedMetricBuilder.combineScript(new Script(paramValue));
197195
break;
198196
case "combine_script_id":
199-
scriptedMetricBuilder.combineScript(new Script(paramValue, ScriptService.ScriptType.INDEXED,null,null));
197+
scriptedMetricBuilder.combineScript(new Script(paramValue, ScriptService.ScriptType.INDEXED, null, null));
200198
break;
201199
case "combine_script_file":
202-
scriptedMetricBuilder.combineScript(new Script(paramValue, ScriptService.ScriptType.FILE,null,null));
200+
scriptedMetricBuilder.combineScript(new Script(paramValue, ScriptService.ScriptType.FILE, null, null));
203201
break;
204202
case "reduce_script":
205203
scriptedMetricBuilder.reduceScript(new Script(paramValue, ScriptService.ScriptType.INLINE,null,reduceScriptAdditionalParams));
206204
break;
207205
case "reduce_script_id":
208-
scriptedMetricBuilder.reduceScript(new Script(paramValue, ScriptService.ScriptType.INDEXED,null,reduceScriptAdditionalParams));
206+
scriptedMetricBuilder.reduceScript(new Script(paramValue, ScriptService.ScriptType.INDEXED, null, reduceScriptAdditionalParams));
209207
break;
210208
case "reduce_script_file":
211-
scriptedMetricBuilder.reduceScript(new Script(paramValue, ScriptService.ScriptType.FILE,null,reduceScriptAdditionalParams));
209+
scriptedMetricBuilder.reduceScript(new Script(paramValue, ScriptService.ScriptType.FILE, null, reduceScriptAdditionalParams));
212210
break;
213211
case "alias":
212+
case "nested":
214213
break;
215214
default:
216215
throw new SqlParseException("scripted_metric err or not define field " + param.getKey());
@@ -244,6 +243,7 @@ private AggregationBuilder<?> geohashGrid(MethodField field) throws SqlParseExce
244243
geoHashGrid.shardSize(Integer.parseInt(value));
245244
break;
246245
case "alias":
246+
case "nested":
247247
break;
248248
default:
249249
throw new SqlParseException("geohash grid err or not define field " + kv.toString());
@@ -274,7 +274,7 @@ private ValuesSourceAggregationBuilder<?> dateRange(MethodField field) {
274274
} else if ("to".equals(kv.key)) {
275275
dateRange.addUnboundedTo(kv.value);
276276
continue;
277-
} else if ("alias".equals(kv.key)){
277+
} else if ("alias".equals(kv.key) || "nested".equals(kv.key)){
278278
continue;
279279
} else {
280280
ranges.add(value);
@@ -316,6 +316,7 @@ private DateHistogramBuilder dateHistogram(MethodField field) throws SqlParseExc
316316
break;
317317

318318
case "alias":
319+
case "nested":
319320
break;
320321
default:
321322
throw new SqlParseException("date range err or not define field " + kv.toString());
@@ -355,6 +356,7 @@ private HistogramBuilder histogram(MethodField field) throws SqlParseException {
355356
histogram.extendedBounds(Long.valueOf(bounds[0]), Long.valueOf(bounds[1]));
356357
break;
357358
case "alias":
359+
case "nested":
358360
break;
359361
case "order":
360362
Histogram.Order order = null;
@@ -448,7 +450,8 @@ private AbstractAggregationBuilder makeTopHitsAgg(MethodField field) {
448450
topHits.setSize((int) kv.value);
449451
break;
450452
case "alias":
451-
break;
453+
case "nested":
454+
break;
452455
default:
453456
topHits.addSort(kv.key, SortOrder.valueOf(kv.value.toString().toUpperCase()));
454457
break;

‎src/test/java/org/nlpcn/es4sql/AggregationTest.java

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter;
1212
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGrid;
1313
import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGrid;
14+
import org.elasticsearch.search.aggregations.bucket.histogram.Histogram;
1415
import org.elasticsearch.search.aggregations.bucket.nested.InternalNested;
1516
import org.elasticsearch.search.aggregations.bucket.terms.Terms;
1617
import org.elasticsearch.search.aggregations.metrics.avg.Avg;
@@ -473,6 +474,46 @@ public void groupByOnNestedFieldWithFilterTest() throws Exception {
473474
}
474475
}
475476

477+
@Test
478+
public void minOnNestedField() throws Exception {
479+
Aggregations result = query(String.format("SELECT min(nested(message.dayOfWeek)) as minDays FROM %s/nestedType", TEST_INDEX));
480+
InternalNested nested = result.get("message.dayOfWeek@NESTED");
481+
Min mins = nested.getAggregations().get("minDays");
482+
Assert.assertEquals(1.0,mins.getValue(),0.0001);
483+
484+
}
485+
486+
@Test
487+
public void sumOnNestedField() throws Exception {
488+
Aggregations result = query(String.format("SELECT sum(nested(message.dayOfWeek)) as sumDays FROM %s/nestedType", TEST_INDEX));
489+
InternalNested nested = result.get("message.dayOfWeek@NESTED");
490+
Sum sum = nested.getAggregations().get("sumDays");
491+
Assert.assertEquals(13.0,sum.getValue(),0.0001);
492+
493+
}
494+
495+
@Test
496+
public void histogramOnNestedField() throws Exception {
497+
Aggregations result = query(String.format("select count(*) from %s/nestedType group by histogram('field'='message.dayOfWeek','nested'='message','interval'='2' , 'alias' = 'someAlias' )", TEST_INDEX));
498+
InternalNested nested = result.get("message@NESTED");
499+
Histogram histogram = nested.getAggregations().get("someAlias");
500+
for(Histogram.Bucket bucket : histogram.getBuckets()){
501+
long count = ((ValueCount) bucket.getAggregations().get("COUNT(*)")).getValue();
502+
String key = bucket.getKey().toString();
503+
if(key.equals("0") || key.equals("4")){
504+
Assert.assertEquals(2,count);
505+
}
506+
else if (key.equals("2")){
507+
Assert.assertEquals(1,count);
508+
}
509+
else{
510+
Assert.assertTrue("only 0 2 4 keys are allowed got:" + key,false);
511+
}
512+
}
513+
514+
515+
}
516+
476517

477518

478519
}

‎src/test/java/org/nlpcn/es4sql/MainTestSuite.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,9 @@ private static void prepareNestedTypeIndex() {
101101
" \"author\": {\n" +
102102
" \"type\": \"string\",\n" +
103103
" \"index\": \"not_analyzed\"\n" +
104+
" },\n" +
105+
" \"dayOfWeek\": {\n" +
106+
" \"type\": \"long\"\n" +
104107
" }\n" +
105108
" }\n" +
106109
" },\n" +

‎src/test/resources/nested_objects.json

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
{"index":{"_type": "nestedType", "_id":"1"}}
2-
{"message":{"info":"a","author":"e"},"myNum":1}
2+
{"message":{"info":"a","author":"e","dayOfWeek":1},"myNum":1}
33
{"index":{"_type": "nestedType", "_id":"2"}}
4-
{"message":{"info":"b","author":"f"},"myNum":2}
4+
{"message":{"info":"b","author":"f","dayOfWeek":2},"myNum":2}
55
{"index":{"_type": "nestedType", "_id":"3"}}
6-
{"message":{"info":"c","author":"g"},"myNum":3}
6+
{"message":{"info":"c","author":"g","dayOfWeek":1},"myNum":3}
77
{"index":{"_type": "nestedType", "_id":"4"}}
8-
{"message":[{"info":"c","author":"h"},{"info":"a","author":"i"}],"myNum":4}
8+
{"message":[{"info":"c","author":"h","dayOfWeek":4},{"info":"a","author":"i","dayOfWeek":5}],"myNum":4}

0 commit comments

Comments
 (0)