Skip to content

Commit f1f3060

Browse files
eliranmoyaleliranmoyal
committed
Merge commit '18f7dd58c045d7c378def207ad6a171e9c18136d' into elastic2.0
2 parents c2e04dd + 18f7dd5 commit f1f3060

File tree

7 files changed

+246
-19
lines changed

7 files changed

+246
-19
lines changed

‎src/_site/query.js

Lines changed: 46 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -157,13 +157,30 @@ function addFieldsToRow (row,hit) {
157157

158158

159159

160+
161+
function removeNestedAndFilters (aggs) {
162+
for(field in aggs)
163+
{
164+
if (field.endsWith("@NESTED") || field.endsWith("@FILTER")){
165+
delete aggs[field]["doc_count"];
166+
delete aggs[field]["key"];
167+
leftField = Object.keys(aggs[field])[0];
168+
aggs[leftField] = aggs[field][leftField];
169+
delete aggs[field];
170+
removeNestedAndFilters(aggs);
171+
}
172+
if(typeof(aggs[field])=="object"){
173+
removeNestedAndFilters(aggs[field]);
174+
}
175+
}
176+
}
160177
/* AggregationQueryResultHandler object
161178
Handle the query result,
162179
in case of Aggregation query
163180
(SQL group by)
164181
*/
165182
var AggregationQueryResultHandler = function(data) {
166-
183+
removeNestedAndFilters(data.aggregations);
167184
function getRows(bucketName, bucket, additionalColumns) {
168185
var rows = []
169186

@@ -200,7 +217,13 @@ var AggregationQueryResultHandler = function(data) {
200217
}
201218

202219
for(var field in bucket) {
220+
203221
var bucketValue = bucket[field]
222+
if(bucketValue.buckets != undefined ){
223+
var newRows = getRows(subBucketName, bucketValue, newAdditionalColumns);
224+
$.merge(rows, newRows);
225+
continue;
226+
}
204227
if(bucketValue.value != undefined) {
205228
if("value_as_string" in bucket[field]){
206229
obj[field] = bucketValue["value_as_string"]
@@ -210,8 +233,17 @@ var AggregationQueryResultHandler = function(data) {
210233
}
211234
}
212235
else {
213-
if(typeof(bucket[field])=="object"){
214-
fillFieldsForSpecificAggregation(obj,bucketValue,field);
236+
if(typeof(bucketValue)=="object"){
237+
/*subBuckets = getSubBuckets(bucketValue);
238+
if(subBuckets.length >0){
239+
var newRows = getRows(subBucketName, {"buckets":subBuckets}, newAdditionalColumns);
240+
$.merge(rows, newRows);
241+
continue;
242+
}*/
243+
244+
245+
fillFieldsForSpecificAggregation(obj,bucketValue,field);
246+
215247
}
216248
}
217249
}
@@ -223,7 +255,8 @@ var AggregationQueryResultHandler = function(data) {
223255

224256

225257
function fillFieldsForSpecificAggregation(obj,value,field)
226-
{
258+
{
259+
227260
for(key in value){
228261
if(key == "values"){
229262
fillFieldsForSpecificAggregation(obj,value[key],field);
@@ -244,6 +277,15 @@ var AggregationQueryResultHandler = function(data) {
244277
subBuckets.push({"bucketName": field, "bucket": buckets[i]})
245278
}
246279
}
280+
else {
281+
innerAgg = bucket[field];
282+
for(var innerField in innerAgg){
283+
if(typeof(innerAgg[innerField])=="object"){
284+
innerBuckets = getSubBuckets(innerAgg[innerField]);
285+
$.merge(subBuckets,innerBuckets);
286+
}
287+
}
288+
}
247289
}
248290

249291
return subBuckets

‎src/main/java/org/nlpcn/es4sql/parse/FieldMaker.java

Lines changed: 46 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
import org.nlpcn.es4sql.domain.Field;
1010
import org.nlpcn.es4sql.domain.KVValue;
1111
import org.nlpcn.es4sql.domain.MethodField;
12+
import org.nlpcn.es4sql.domain.Where;
1213
import org.nlpcn.es4sql.exception.SqlParseException;
1314
import com.alibaba.druid.sql.ast.*;
1415
/**
@@ -30,11 +31,17 @@ public static Field makeField(SQLExpr expr, String alias,String tableAlias) thro
3031
} else if (expr instanceof SQLAllColumnExpr) {
3132
} else if (expr instanceof SQLMethodInvokeExpr) {
3233
SQLMethodInvokeExpr mExpr = (SQLMethodInvokeExpr) expr;
33-
NestedType nestedType = new NestedType();
34-
if(nestedType.tryFillFromExpr(mExpr)){
35-
return handleIdentifier(nestedType,alias,tableAlias);
34+
String methodName = mExpr.getMethodName();
35+
if(methodName.toLowerCase().equals("nested")){
36+
NestedType nestedType = new NestedType();
37+
if(nestedType.tryFillFromExpr(mExpr)){
38+
return handleIdentifier(nestedType,alias,tableAlias);
39+
}
40+
}
41+
else if (methodName.toLowerCase().equals("filter")){
42+
return makeFilterMethodField(mExpr,alias);
3643
}
37-
return makeMethodField(mExpr.getMethodName(), mExpr.getParameters(), null, alias);
44+
return makeMethodField(methodName, mExpr.getParameters(), null, alias);
3845
} else if (expr instanceof SQLAggregateExpr) {
3946
SQLAggregateExpr sExpr = (SQLAggregateExpr) expr;
4047
return makeMethodField(sExpr.getMethodName(), sExpr.getArguments(), sExpr.getOption(), alias);
@@ -44,6 +51,40 @@ public static Field makeField(SQLExpr expr, String alias,String tableAlias) thro
4451
return null;
4552
}
4653

54+
private static Field makeFilterMethodField(SQLMethodInvokeExpr filterMethod,String alias) throws SqlParseException {
55+
List<SQLExpr> parameters = filterMethod.getParameters();
56+
int parametersSize = parameters.size();
57+
if(parametersSize != 1 && parametersSize !=2){
58+
throw new SqlParseException("filter group by field should only have one or 2 parameters filter(Expr) or filter(name,Expr)");
59+
}
60+
String filterAlias = filterMethod.getMethodName();
61+
SQLExpr exprToCheck = null;
62+
if(parametersSize == 1){
63+
exprToCheck = parameters.get(0);
64+
filterAlias = "filter(" + exprToCheck.toString().replaceAll("\n"," ") +")";
65+
}
66+
if(parametersSize == 2){
67+
//todo: function extendedToString - if sqlString remove ''
68+
filterAlias = extendedToString(parameters.get(0));
69+
exprToCheck = parameters.get(1);
70+
}
71+
Where where = Where.newInstance();
72+
new SqlParser().parseWhere(exprToCheck,where);
73+
if(where.getWheres().size() == 0)
74+
throw new SqlParseException("unable to parse filter where.");
75+
List<KVValue> methodParameters = new ArrayList<>();
76+
methodParameters.add(new KVValue("where",where));
77+
methodParameters.add(new KVValue("alias",filterAlias+"@FILTER"));
78+
return new MethodField("filter", methodParameters, null, alias);
79+
}
80+
81+
private static String extendedToString(SQLExpr sqlExpr) {
82+
if(sqlExpr instanceof SQLTextLiteralExpr){
83+
return ((SQLTextLiteralExpr) sqlExpr).getText();
84+
}
85+
return sqlExpr.toString();
86+
}
87+
4788
private static Field handleIdentifier(NestedType nestedType, String alias, String tableAlias) {
4889
Field field = handleIdentifier(new SQLIdentifierExpr(nestedType.field), alias, tableAlias);
4990
field.setNested(true);
@@ -98,7 +139,7 @@ private static MethodField makeMethodField(String name, List<SQLExpr> arguments,
98139
List<KVValue> paramers = new LinkedList<>();
99140
for (SQLExpr object : arguments) {
100141
if (object instanceof SQLBinaryOpExpr) {
101-
//todo: if operator is different from equal ....
142+
102143
SQLBinaryOpExpr binaryOpExpr = (SQLBinaryOpExpr) object;
103144
if(!binaryOpExpr.getOperator().getName().equals("=")){
104145
paramers.add(new KVValue("script", makeScriptMethodField(binaryOpExpr,null)));

‎src/main/java/org/nlpcn/es4sql/parse/SqlParser.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ private boolean isAllowedMethodOnConditionLeft(SQLMethodInvokeExpr method) {
9292
return method.getMethodName().toLowerCase().equals("nested");
9393
}
9494

95-
private void parseWhere(SQLExpr expr, Where where) throws SqlParseException {
95+
public void parseWhere(SQLExpr expr, Where where) throws SqlParseException {
9696
if (expr instanceof SQLBinaryOpExpr && !isCond((SQLBinaryOpExpr) expr)) {
9797
SQLBinaryOpExpr bExpr = (SQLBinaryOpExpr) expr;
9898
routeCond(bExpr, bExpr.getLeft(), where);
@@ -282,7 +282,7 @@ private void findGroupBy(MySqlSelectQueryBlock query, Select select) throws SqlP
282282
sqlExpr = sqlSelectGroupByExpr.getExpr();
283283
}
284284

285-
if ((sqlExpr instanceof SQLParensIdentifierExpr || !(sqlExpr instanceof SQLIdentifierExpr)) && !standardGroupBys.isEmpty()) {
285+
if ((sqlExpr instanceof SQLParensIdentifierExpr || !(sqlExpr instanceof SQLIdentifierExpr|| sqlExpr instanceof SQLMethodInvokeExpr)) && !standardGroupBys.isEmpty()) {
286286
// flush the standard group bys
287287
select.addGroupBy(convertExprsToFields(standardGroupBys));
288288
standardGroupBys = new ArrayList<>();

‎src/main/java/org/nlpcn/es4sql/query/AggregationQueryAction.java

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,14 @@ public SqlElasticSearchRequestBuilder explain() throws SqlParseException {
5959
}
6060

6161
if(field.isNested()){
62-
NestedBuilder nestedBuilder = AggregationBuilders.nested(field.getName() + "Nested")
63-
.path(field.getNestedPath())
64-
.subAggregation(lastAgg);
62+
NestedBuilder nestedBuilder = AggregationBuilders.nested(field.getName() + "@NESTED")
63+
.path(field.getNestedPath());
64+
if(insertFilterIfExistsAfter(lastAgg, groupBy, nestedBuilder,1)){
65+
groupBy.remove(1);
66+
}
67+
else {
68+
nestedBuilder.subAggregation(lastAgg);
69+
}
6570
request.addAggregation(nestedBuilder);
6671
}
6772
else {
@@ -77,9 +82,16 @@ public SqlElasticSearchRequestBuilder explain() throws SqlParseException {
7782
}
7883

7984
if(field.isNested()){
80-
NestedBuilder nestedBuilder = AggregationBuilders.nested(field.getName() + "Nested")
81-
.path(field.getNestedPath())
82-
.subAggregation(subAgg);
85+
86+
NestedBuilder nestedBuilder = AggregationBuilders.nested(field.getName() + "@NESTED")
87+
.path(field.getNestedPath());
88+
if(insertFilterIfExistsAfter(subAgg, groupBy, nestedBuilder,i+1)){
89+
groupBy.remove(i+1);
90+
i++;
91+
}
92+
else {
93+
nestedBuilder.subAggregation(subAgg);
94+
}
8395
lastAgg.subAggregation(nestedBuilder);
8496

8597
}
@@ -132,6 +144,16 @@ public SqlElasticSearchRequestBuilder explain() throws SqlParseException {
132144
return sqlElasticRequestBuilder;
133145
}
134146

147+
private boolean insertFilterIfExistsAfter(AggregationBuilder<?> agg, List<Field> groupBy, NestedBuilder builder, int nextPosition) throws SqlParseException {
148+
if(groupBy.size() <= nextPosition) return false;
149+
Field filterFieldCandidate = groupBy.get(nextPosition);
150+
if(! (filterFieldCandidate instanceof MethodField)) return false;
151+
MethodField methodField = (MethodField) filterFieldCandidate;
152+
if(!methodField.getName().toLowerCase().equals("filter")) return false;
153+
builder.subAggregation(aggMaker.makeGroupAgg(filterFieldCandidate).subAggregation(agg));
154+
return true;
155+
}
156+
135157
private AggregationBuilder<?> updateAggIfNested(AggregationBuilder<?> lastAgg, Field field) {
136158
if(field.isNested()){
137159
lastAgg = AggregationBuilders.nested(field.getName() + "Nested")

‎src/main/java/org/nlpcn/es4sql/query/maker/AggMaker.java

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.nlpcn.es4sql.domain.Field;
3131
import org.nlpcn.es4sql.domain.KVValue;
3232
import org.nlpcn.es4sql.domain.MethodField;
33+
import org.nlpcn.es4sql.domain.Where;
3334
import org.nlpcn.es4sql.exception.SqlParseException;
3435

3536
public class AggMaker {
@@ -45,7 +46,15 @@ public class AggMaker {
4546
*/
4647
public AggregationBuilder<?> makeGroupAgg(Field field) throws SqlParseException {
4748
if (field instanceof MethodField) {
48-
return makeRangeGroup((MethodField) field);
49+
50+
MethodField methodField = (MethodField) field;
51+
if(methodField.getName().equals("filter")){
52+
Map<String, Object> paramsAsMap = methodField.getParamsAsMap();
53+
Where where = (Where) paramsAsMap.get("where");
54+
return AggregationBuilders.filter(paramsAsMap.get("alias").toString()).
55+
filter(FilterMaker.explan(where));
56+
}
57+
return makeRangeGroup(methodField);
4958
} else {
5059
TermsBuilder termsBuilder = AggregationBuilders.terms(field.getName()).field(field.getName());
5160
groupMap.put(field.getName(), new KVValue("KEY", termsBuilder));

‎src/test/java/org/nlpcn/es4sql/AggregationTest.java

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@
66
import com.google.common.collect.Range;
77
import org.elasticsearch.action.search.SearchRequestBuilder;
88
import org.elasticsearch.action.search.SearchResponse;
9+
import org.elasticsearch.search.aggregations.Aggregation;
910
import org.elasticsearch.search.aggregations.Aggregations;
11+
import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter;
1012
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGrid;
1113
import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGrid;
1214
import org.elasticsearch.search.aggregations.bucket.nested.InternalNested;
@@ -411,7 +413,7 @@ public void geoHashGrid() throws SQLFeatureNotSupportedException, SqlParseExcept
411413
@Test
412414
public void groupByOnNestedFieldTest() throws Exception {
413415
Aggregations result = query(String.format("SELECT COUNT(*) FROM %s/nestedType GROUP BY nested(message.info)", TEST_INDEX));
414-
InternalNested nested = result.get("message.infoNested");
416+
InternalNested nested = result.get("message.info@NESTED");
415417
Terms infos = nested.getAggregations().get("message.info");
416418
Assert.assertEquals(3,infos.getBuckets().size());
417419
for(Terms.Bucket bucket : infos.getBuckets()) {
@@ -432,5 +434,45 @@ else if(key.equalsIgnoreCase("b")) {
432434
}
433435
}
434436

437+
@Test
438+
public void groupByTestWithFilter() throws Exception {
439+
Aggregations result = query(String.format("SELECT COUNT(*) FROM %s/account GROUP BY filter(gender='m'),gender", TEST_INDEX));
440+
InternalFilter filter = result.get("filter(gender = 'm')@FILTER");
441+
Terms gender = filter.getAggregations().get("gender");
442+
443+
for(Terms.Bucket bucket : gender.getBuckets()) {
444+
String key = bucket.getKey();
445+
long count = ((ValueCount) bucket.getAggregations().get("COUNT(*)")).getValue();
446+
if(key.equalsIgnoreCase("m")) {
447+
Assert.assertEquals(507, count);
448+
}
449+
else {
450+
throw new Exception(String.format("Unexpected key. expected: only m. found: %s", key));
451+
}
452+
}
453+
}
454+
455+
456+
@Test
457+
public void groupByOnNestedFieldWithFilterTest() throws Exception {
458+
Aggregations result = query(String.format("SELECT COUNT(*) FROM %s/nestedType GROUP BY nested(message.info),filter('myFilter',message.info = 'a')", TEST_INDEX));
459+
InternalNested nested = result.get("message.info@NESTED");
460+
InternalFilter filter = nested.getAggregations().get("myFilter@FILTER");
461+
Terms infos = filter.getAggregations().get("message.info");
462+
Assert.assertEquals(1,infos.getBuckets().size());
463+
for(Terms.Bucket bucket : infos.getBuckets()) {
464+
String key = bucket.getKey();
465+
long count = ((ValueCount) bucket.getAggregations().get("COUNT(*)")).getValue();
466+
if(key.equalsIgnoreCase("a")) {
467+
Assert.assertEquals(2, count);
468+
}
469+
470+
else {
471+
throw new Exception(String.format("Unexpected key. expected: only a . found: %s", key));
472+
}
473+
}
474+
}
475+
476+
435477

436478
}

0 commit comments

Comments
 (0)