From a6697d6397e96b03778f85f4e1bfba14688df31d Mon Sep 17 00:00:00 2001 From: clude_zhu Date: Tue, 27 Dec 2016 20:00:01 +0800 Subject: [PATCH 1/4] fix count distinct issue --- .../nlpcn/es4sql/query/maker/AggMaker.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/nlpcn/es4sql/query/maker/AggMaker.java b/src/main/java/org/nlpcn/es4sql/query/maker/AggMaker.java index f2c3ccda..a25ae345 100644 --- a/src/main/java/org/nlpcn/es4sql/query/maker/AggMaker.java +++ b/src/main/java/org/nlpcn/es4sql/query/maker/AggMaker.java @@ -20,6 +20,7 @@ import org.elasticsearch.search.aggregations.bucket.range.date.DateRangeAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.geobounds.GeoBoundsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.scripted.ScriptedMetricAggregationBuilder; @@ -570,14 +571,22 @@ private AbstractAggregationBuilder makeCountAgg(MethodField field) { // Cardinality is approximate DISTINCT. if ("DISTINCT".equals(field.getOption())) { - - if (field.getParams().size() == 1) { - return AggregationBuilders.cardinality(field.getAlias()).field(field.getParams().get(0).value.toString()); - } else { + CardinalityAggregationBuilder builder = AggregationBuilders.cardinality(field.getAlias()); + if(field.getParams().size() > 1){ Integer precision_threshold = (Integer) (field.getParams().get(1).value); - return AggregationBuilders.cardinality(field.getAlias()).precisionThreshold(precision_threshold).field(field.getParams().get(0).value.toString()); + builder = builder.precisionThreshold(precision_threshold); } + KVValue kvValue = field.getParams().get(0); + if (kvValue.key != null && kvValue.key.equals("script")) { + if (kvValue.value instanceof MethodField) { + return builder.script(new Script(((MethodField) kvValue.value).getParams().get(1).toString())); + } else { + return builder.script(new Script(kvValue.value.toString())); + } + }else{ + return builder.field(kvValue.value.toString()); + } } String fieldName = field.getParams().get(0).value.toString(); From 60f637da437e87ae540492d17727e5fe9babf927 Mon Sep 17 00:00:00 2001 From: clude_zhu Date: Tue, 27 Dec 2016 20:01:01 +0800 Subject: [PATCH 2/4] import jdbc feature: supports retrieve totalHits and datasets in one query --- ...ticSearchDruidPooledPreparedStatement.java | 7 +++--- .../druid/pool/ElasticSearchResultSet.java | 24 +++++++++++++++++++ .../ElasticSearchResultSetMetaDataBase.java | 14 +++++++++++ .../org/nlpcn/es4sql/jdbc/ObjectResult.java | 15 ++++++++++++ .../es4sql/jdbc/ObjectResultsExtractor.java | 5 ++-- 5 files changed, 59 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/alibaba/druid/pool/ElasticSearchDruidPooledPreparedStatement.java b/src/main/java/com/alibaba/druid/pool/ElasticSearchDruidPooledPreparedStatement.java index a7ec8fd6..d050e3ca 100644 --- a/src/main/java/com/alibaba/druid/pool/ElasticSearchDruidPooledPreparedStatement.java +++ b/src/main/java/com/alibaba/druid/pool/ElasticSearchDruidPooledPreparedStatement.java @@ -1,10 +1,8 @@ package com.alibaba.druid.pool; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; -import org.elasticsearch.client.transport.TransportClient; import org.elasticsearch.plugin.nlpcn.QueryActionElasticExecutor; -import org.elasticsearch.plugin.nlpcn.executors.CSVResult; -import org.elasticsearch.plugin.nlpcn.executors.CSVResultsExtractor; import org.elasticsearch.plugin.nlpcn.executors.CsvExtractorException; import org.nlpcn.es4sql.SearchDao; import org.nlpcn.es4sql.exception.SqlParseException; @@ -46,8 +44,9 @@ public ResultSet executeQuery() throws SQLException { ObjectResult extractor = getObjectResult(true, getSql(), false, false, true); List headers = extractor.getHeaders(); List> lines = extractor.getLines(); + long totalHits = extractor.getTotalHits(); - ResultSet rs = new ElasticSearchResultSet(this, headers, lines); + ResultSet rs = new ElasticSearchResultSet(this, headers, lines, totalHits); if (rs == null) { return null; diff --git a/src/main/java/com/alibaba/druid/pool/ElasticSearchResultSet.java b/src/main/java/com/alibaba/druid/pool/ElasticSearchResultSet.java index adfb2e3b..436b699e 100644 --- a/src/main/java/com/alibaba/druid/pool/ElasticSearchResultSet.java +++ b/src/main/java/com/alibaba/druid/pool/ElasticSearchResultSet.java @@ -20,6 +20,7 @@ public class ElasticSearchResultSet implements ResultSet { Iterator> iterator; List current = null; List headers = null; + long totalHits; private ResultSetMetaData metaData; @@ -31,6 +32,29 @@ public ElasticSearchResultSet(Statement statement, final List headers, f } + public ElasticSearchResultSet(Statement statement, final List headers, final List> lines, long totalHits) { + this.iterator = lines.iterator(); + this.headers = headers; + metaData = new ElasticSearchResultSetMetaDataBase(headers); + this.totalHits = totalHits; + + } + + // clude -- added - begin + public List getHeaders() { + return headers; + } + + public long getTotalHits() { + return totalHits; + } + + public void setTotalHits(long totalHits) { + this.totalHits = totalHits; + } + + // clude -- added - end + @Override public boolean next() throws SQLException { boolean hasNext = iterator.hasNext(); diff --git a/src/main/java/com/alibaba/druid/pool/ElasticSearchResultSetMetaDataBase.java b/src/main/java/com/alibaba/druid/pool/ElasticSearchResultSetMetaDataBase.java index 1b8f4b77..9588cb1d 100644 --- a/src/main/java/com/alibaba/druid/pool/ElasticSearchResultSetMetaDataBase.java +++ b/src/main/java/com/alibaba/druid/pool/ElasticSearchResultSetMetaDataBase.java @@ -3,6 +3,7 @@ import com.alibaba.druid.util.jdbc.ResultSetBase; import com.alibaba.druid.util.jdbc.ResultSetMetaDataBase; +import java.sql.SQLException; import java.util.ArrayList; import java.util.List; @@ -12,6 +13,19 @@ public class ElasticSearchResultSetMetaDataBase extends ResultSetMetaDataBase { private final List columns = new ArrayList(); + public List getColumns() { + return columns; + } + + public int getColumnCount() throws SQLException { + return this.columns.size(); + } + + public ColumnMetaData getColumn(int column) { + return this.columns.get(column - 1); + } + + public ElasticSearchResultSetMetaDataBase(List headers) { for(String column:headers){ ColumnMetaData columnMetaData = new ColumnMetaData(); diff --git a/src/main/java/org/nlpcn/es4sql/jdbc/ObjectResult.java b/src/main/java/org/nlpcn/es4sql/jdbc/ObjectResult.java index d6612629..7fe2dd42 100644 --- a/src/main/java/org/nlpcn/es4sql/jdbc/ObjectResult.java +++ b/src/main/java/org/nlpcn/es4sql/jdbc/ObjectResult.java @@ -8,12 +8,19 @@ public class ObjectResult { private final List headers; private final List> lines; + private long totalHits; public ObjectResult(List headers, List> lines) { this.headers = headers; this.lines = lines; } + public ObjectResult(List headers, List> lines, long totalHits) { + this.headers = headers; + this.lines = lines; + this.totalHits = totalHits; + } + public List getHeaders() { return headers; } @@ -21,4 +28,12 @@ public List getHeaders() { public List> getLines() { return lines; } + + public long getTotalHits() { + return totalHits; + } + + public void setTotalHits(long totalHits) { + this.totalHits = totalHits; + } } diff --git a/src/main/java/org/nlpcn/es4sql/jdbc/ObjectResultsExtractor.java b/src/main/java/org/nlpcn/es4sql/jdbc/ObjectResultsExtractor.java index ebd5bc8b..6b135bef 100644 --- a/src/main/java/org/nlpcn/es4sql/jdbc/ObjectResultsExtractor.java +++ b/src/main/java/org/nlpcn/es4sql/jdbc/ObjectResultsExtractor.java @@ -39,7 +39,8 @@ public ObjectResult extractResults(Object queryResult, boolean flat) throws Obje List> docsAsMap = new ArrayList<>(); List headers = createHeadersAndFillDocsMap(flat, hits, docsAsMap); List> lines = createLinesFromDocs(flat, docsAsMap, headers); - return new ObjectResult(headers, lines); + long totalHits = ((SearchHits) queryResult).getTotalHits(); + return new ObjectResult(headers, lines, totalHits); } if (queryResult instanceof Aggregations) { List headers = new ArrayList<>(); @@ -55,7 +56,7 @@ public ObjectResult extractResults(Object queryResult, boolean flat) throws Obje //Aggregations that inhrit from base //ScriptedMetric - return new ObjectResult(headers, lines); + return new ObjectResult(headers, lines, lines.size()); } return null; From 309f3026e54ffe0e9e7ab95fe6abf211262f35cc Mon Sep 17 00:00:00 2001 From: clude_zhu Date: Tue, 27 Dec 2016 20:01:15 +0800 Subject: [PATCH 3/4] add local deploy setting --- local-deploy.md | 6 ++++++ pom.xml | 57 ++++++++++++++++++++++++++++++------------------- 2 files changed, 41 insertions(+), 22 deletions(-) create mode 100644 local-deploy.md diff --git a/local-deploy.md b/local-deploy.md new file mode 100644 index 00000000..cbebeec5 --- /dev/null +++ b/local-deploy.md @@ -0,0 +1,6 @@ +* command 1 + mvn clean deploy -Dmaven.test.skip=true -Dgpg.skip=true + + +* command 2 + mvn deploy:deploy-file -Durl=http://10.57.115.230/artifactory/libs-release-local -DrepositoryId=releases -Dfile=/Users/clude/workspace/clude-git/elasticsearch-sql/target/elasticsearch-sql-5.1.1.0.jar -DgroupId=rg.nlpcn -DartifactId=lasticsearch-sql -Dversion=5.1.1.0 diff --git a/pom.xml b/pom.xml index 5a4a75c9..9dd87fa2 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 org.nlpcn elasticsearch-sql - 5.1.1.0 + 5.1.1.01 jar Query elasticsearch using SQL elasticsearch-sql @@ -292,35 +292,48 @@ - - org.apache.maven.plugins - maven-gpg-plugin - 1.4 - - - sign-artifacts - verify - - sign - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - sonatype-nexus-snapshots - Sonatype Nexus snapshot repository - https://oss.sonatype.org/content/repositories/snapshots + clude-snapshots + clude snapshots + http://10.57.115.230/artifactory/libs-snapshot-local - - sonatype-nexus-staging - Sonatype Nexus release repository - https://oss.sonatype.org/service/local/staging/deploy/maven2 + releases + clude artifactory release repository + http://10.57.115.230/artifactory/libs-release-local From 811d669faede4ee4795208e264cd71faa2f1a6cc Mon Sep 17 00:00:00 2001 From: clude_zhu Date: Wed, 28 Dec 2016 16:12:14 +0800 Subject: [PATCH 4/4] fix BigDecimal parsing issue there is an BigDecimal parsing issue when we set condition like "where amount > 10.02" --- src/main/java/org/nlpcn/es4sql/query/maker/Maker.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/nlpcn/es4sql/query/maker/Maker.java b/src/main/java/org/nlpcn/es4sql/query/maker/Maker.java index 7913ad9c..e81d970a 100644 --- a/src/main/java/org/nlpcn/es4sql/query/maker/Maker.java +++ b/src/main/java/org/nlpcn/es4sql/query/maker/Maker.java @@ -1,6 +1,7 @@ package org.nlpcn.es4sql.query.maker; import java.io.IOException; +import java.math.BigDecimal; import java.util.*; import com.alibaba.druid.sql.ast.SQLExpr; @@ -60,6 +61,8 @@ protected ToXContent make(Condition cond) throws SqlParseException { } else if (value instanceof SubQueryExpression){ x = make(cond,name,((SubQueryExpression)value).getValues()); + }else if (value instanceof BigDecimal){ // clude: fix a issue like where > 1.01 + x = make(cond,name,((BigDecimal)value).doubleValue()); } else { x = make(cond, name, value); }