Skip to content

Commit 2f6e154

Browse files
sparksburnittxinyunh
sparksburnitt
authored and
xinyunh
committed
fix typos, optimize imports
1 parent 1296b1a commit 2f6e154

22 files changed

+83
-129
lines changed

‎src/main/scala/org/apache/spark/sql/hbase/HBaseCatalog.scala

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,9 @@ import org.apache.hadoop.hbase.{Coprocessor, HColumnDescriptor, HTableDescriptor
2929
import org.apache.log4j.Logger
3030
import org.apache.spark.Logging
3131
import org.apache.spark.sql.SQLContext
32-
import org.apache.spark.sql.catalyst.{SimpleCatalystConf, CatalystConf}
3332
import org.apache.spark.sql.catalyst.analysis.{Catalog, OverrideCatalog}
3433
import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Subquery}
34+
import org.apache.spark.sql.catalyst.{CatalystConf, SimpleCatalystConf}
3535
import org.apache.spark.sql.hbase.HBaseCatalog._
3636
import org.apache.spark.sql.types._
3737

@@ -150,7 +150,7 @@ private[hbase] class HBaseCatalog(@transient hbaseContext: SQLContext,
150150

151151
deploySuccessfully_internal = Some(!results.isEmpty)
152152
if (results.isEmpty) {
153-
logger.warn( """Not deplyed successfully""")
153+
logger.warn( """CheckDirEndPoint coprocessor deployment failed.""")
154154
}
155155

156156
pwdIsAccessible = !results.containsValue(false)
@@ -189,7 +189,7 @@ private[hbase] class HBaseCatalog(@transient hbaseContext: SQLContext,
189189
families.foreach {
190190
case family =>
191191
if (!checkFamilyExists(hbaseTableName, family)) {
192-
throw new Exception(s"The HBase table doesn't contain the Column Family: $family")
192+
throw new Exception(s"HBase table does not contain column family: $family")
193193
}
194194
}
195195
}
@@ -349,7 +349,7 @@ private[hbase] class HBaseCatalog(@transient hbaseContext: SQLContext,
349349
def deleteTable(tableName: String): Unit = {
350350
val metadataTable = getMetadataTable
351351
if (!checkLogicalTableExist(tableName, metadataTable)) {
352-
throw new IllegalStateException(s"The logical table $tableName does not exist")
352+
throw new IllegalStateException(s"Logical table $tableName does not exist.")
353353
}
354354

355355
val delete = new Delete(Bytes.toBytes(tableName))

‎src/main/scala/org/apache/spark/sql/hbase/HBaseCriticalPoint.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@
1717
package org.apache.spark.sql.hbase
1818

1919
import org.apache.spark.sql.catalyst.expressions._
20-
import org.apache.spark.sql.types._
2120
import org.apache.spark.sql.hbase.catalyst.expressions.PartialPredicateOperations._
2221
import org.apache.spark.sql.hbase.types.Range
22+
import org.apache.spark.sql.types._
2323

2424
import scala.collection.mutable
2525
import scala.collection.mutable.ArrayBuffer

‎src/main/scala/org/apache/spark/sql/hbase/HBaseCustomFilter.scala

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,13 @@ import org.apache.spark.sql.hbase.util.{BinaryBytesUtils, DataTypeUtils, HBaseKV
3131
import org.apache.spark.sql.types.{AtomicType, DataType, StringType}
3232

3333
/**
34-
* The custom filter, it will skip the scan to the proper next position based on predicate
35-
* this filter will only deal with the predicate which has key columns inside
34+
* The custom filter. It will skip the scanner to the proper next position based on predicate.
35+
* This filter will only deal with the predicate containing key columns.
3636
*
37-
* The skip is multiple-dimensional on non-leading dimension keys in precense of the predicate's
38-
* range expressions; other types of expressions in the predicate will be eventually evaluated
37+
* The skip is multiple-dimensional on non-leading dimension keys in presence of the predicate's
38+
* range expressions; other types of expressions in the predicate will be eventually evaluated.
3939
*
40-
* The processing is stateful in that various info related to the previous processing is cahched,
40+
* The processing is stateful in that various info related to the previous processing is cached
4141
* and checked in the next invocations for maximum reuse.
4242
*/
4343
private[hbase] class HBaseCustomFilter extends FilterBase with Writable {
@@ -180,18 +180,18 @@ private[hbase] class HBaseCustomFilter extends FilterBase with Writable {
180180
* recursively reset the index of the current child and the value in the child's CPR
181181
* @param node the start level, it will also reset its children
182182
*/
183-
private def resetDecendents(node: Node): Unit = {
183+
private def resetDescendants(node: Node): Unit = {
184184
if (node.children != null) {
185185
node.currentChildIndex = 0
186186
for (child <- node.children) {
187187
resetNode(child)
188-
resetDecendents(child)
188+
resetDescendants(child)
189189
}
190190
}
191191
}
192192

193193
/**
194-
* A quick top-down check whether the new row is in the current CPRs
194+
* A quick top-down check whether the new row is in the current CPRs.
195195
* @param dimValues the current dimensional keys to check
196196
* @param dimLimit the lower bound of the dimensions to be checked with.
197197
* 0 for the most significant dimension
@@ -241,7 +241,7 @@ private[hbase] class HBaseCustomFilter extends FilterBase with Writable {
241241
remainingPredicate = null
242242
remainingPredicateBoundRef = null
243243
currentValues = inputValues
244-
resetDecendents(node)
244+
resetDescendants(node)
245245
val result = findNextHint(node)
246246
nextReturnCode = result._1
247247
if (nextReturnCode == ReturnCode.SEEK_NEXT_USING_HINT) {
@@ -384,7 +384,7 @@ private[hbase] class HBaseCustomFilter extends FilterBase with Writable {
384384
// cannot find a containing child but there is a larger child
385385
node.currentChildIndex = childIndex
386386
val child = node.children(childIndex)
387-
resetDecendents(child)
387+
resetDescendants(child)
388388
if (child.cpr != null) {
389389
child.currentValue = child.cpr.start.orNull
390390
if (child.currentValue != null && !child.cpr.startInclusive) {
@@ -407,7 +407,7 @@ private[hbase] class HBaseCustomFilter extends FilterBase with Writable {
407407
if (addOne(currentNode)) {
408408
val cmp = compareWithinRange(currentNode.cpr.dt, currentNode.currentValue, currentNode.cpr)
409409
if (cmp == 0) {
410-
resetDecendents(currentNode)
410+
resetDescendants(currentNode)
411411
return (ReturnCode.SEEK_NEXT_USING_HINT, buildRowKey())
412412
} else {
413413
require(cmp > 0, "Internal logical error: unexpected ordering of row key")
@@ -418,7 +418,7 @@ private[hbase] class HBaseCustomFilter extends FilterBase with Writable {
418418
// no look back: release the memory
419419
currentNode.children = null
420420
}
421-
resetDecendents(currentNode.parent)
421+
resetDescendants(currentNode.parent)
422422
currentNode.parent.currentChildIndex = childIndex
423423
return (ReturnCode.SEEK_NEXT_USING_HINT, buildRowKey())
424424
} else {
@@ -562,14 +562,14 @@ private[hbase] class HBaseCustomFilter extends FilterBase with Writable {
562562
node.children = Seq(Node(dt, dimIndex, node,
563563
cpr = new CriticalPointRange[t](None, false, None, false, dt, null)))
564564
}
565-
resetDecendents(node)
565+
resetDescendants(node)
566566
}
567567

568568
/**
569-
* do a full evaluation for the remaining predicate based on all the cell values
569+
* Do a full evaluation for the remaining predicate based on all the cell values.
570570
* @param kvs the list of cell
571571
*/
572-
private def fullEvalution(kvs: java.util.List[Cell]) = {
572+
private def fullEvaluation(kvs: java.util.List[Cell]) = {
573573
resetRow(workingRow)
574574
cellMap.clear()
575575
for (i <- 0 to kvs.size() - 1) {
@@ -617,7 +617,7 @@ private[hbase] class HBaseCustomFilter extends FilterBase with Writable {
617617
// If a later HBase release has this addressed, this check will be made unnecessary
618618
// to save some CPU cycles
619619
if (kvs.isEmpty) filterRowFlag = true
620-
else if (remainingPredicate != null) fullEvalution(kvs)
620+
else if (remainingPredicate != null) fullEvaluation(kvs)
621621
}
622622

623623
override def hasFilterRow: Boolean = {

‎src/main/scala/org/apache/spark/sql/hbase/HBasePartition.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,10 @@
1717
package org.apache.spark.sql.hbase
1818

1919
import org.apache.hadoop.hbase.regionserver.RegionScanner
20-
import org.apache.spark.{Logging, Partition}
2120
import org.apache.spark.sql.catalyst.expressions._
2221
import org.apache.spark.sql.hbase.catalyst.expressions.PartialPredicateOperations._
2322
import org.apache.spark.sql.hbase.types.{HBaseBytesType, Range}
23+
import org.apache.spark.{Logging, Partition}
2424

2525

2626
private[hbase] class HBasePartition(

‎src/main/scala/org/apache/spark/sql/hbase/HBasePartitioner.scala

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,9 @@
1717

1818
package org.apache.spark.sql.hbase
1919

20-
import java.io.{IOException, ObjectInputStream, ObjectOutputStream}
21-
2220
import org.apache.hadoop.hbase.util.Bytes
23-
import org.apache.spark.serializer.JavaSerializer
24-
import org.apache.spark.util.{CollectionsUtils, Utils}
25-
import org.apache.spark.{Partitioner, SparkEnv}
21+
import org.apache.spark.Partitioner
22+
import org.apache.spark.util.CollectionsUtils
2623

2724
object HBasePartitioner {
2825
implicit object HBaseRawOrdering extends Ordering[HBaseRawType] {

‎src/main/scala/org/apache/spark/sql/hbase/HBaseRelation.scala

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,17 @@
1717
package org.apache.spark.sql.hbase
1818

1919
import org.apache.hadoop.conf.Configuration
20-
import org.apache.hadoop.hbase.util.Bytes
21-
import org.apache.hadoop.hbase.{HBaseConfiguration, _}
2220
import org.apache.hadoop.hbase.client.{Get, HTable, Put, Result, Scan}
2321
import org.apache.hadoop.hbase.filter._
22+
import org.apache.hadoop.hbase.util.Bytes
23+
import org.apache.hadoop.hbase.{HBaseConfiguration, _}
2424
import org.apache.log4j.Logger
2525
import org.apache.spark.TaskContext
2626
import org.apache.spark.rdd.RDD
27-
import org.apache.spark.sql.DataFrame
28-
import org.apache.spark.sql.SQLContext
27+
import org.apache.spark.sql.{DataFrame, SQLContext}
2928
import org.apache.spark.sql.catalyst.expressions._
30-
import org.apache.spark.sql.hbase.catalyst.expressions.PartialPredicateOperations.partialPredicateReducer
3129
import org.apache.spark.sql.hbase.catalyst.NotPusher
30+
import org.apache.spark.sql.hbase.catalyst.expressions.PartialPredicateOperations.partialPredicateReducer
3231
import org.apache.spark.sql.hbase.types.Range
3332
import org.apache.spark.sql.hbase.util._
3433
import org.apache.spark.sql.sources.{BaseRelation, CatalystScan, InsertableRelation, LogicalRelation, RelationProvider}

‎src/main/scala/org/apache/spark/sql/hbase/HBaseSQLReaderRDD.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ object CoprocessorConstants {
4444
* @param useCustomFilter whether custom filter is in effect
4545
* @param output projection. For post coprocessor processing,
4646
* is the projection of the original scan
47-
* @param subplan coproecssor subplan to be sent to coprocessor
47+
* @param subplan coprocessor subplan to be sent to coprocessor
4848
* @param dummyRDD in-memory scan RDD, might be used to reconstruct the original subplan.
4949
* This is possible when decision to use coprocessor has to be made
5050
* by the slaves when its partition-specific predicate is

‎src/main/scala/org/apache/spark/sql/hbase/HBaseSerializer.scala

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -19,17 +19,6 @@ package org.apache.spark.sql.hbase
1919

2020
import java.io._
2121

22-
import org.apache.hadoop.hbase.{KeyValue, CellUtil, Cell}
23-
import org.apache.hadoop.hbase.exceptions.DeserializationException
24-
import org.apache.hadoop.hbase.filter.Filter.ReturnCode
25-
import org.apache.hadoop.hbase.filter.FilterBase
26-
import org.apache.hadoop.hbase.util.{Bytes, Writables}
27-
import org.apache.hadoop.io.Writable
28-
import org.apache.spark.sql.catalyst.expressions._
29-
import org.apache.spark.sql.hbase.util.{HBaseKVHelper, DataTypeUtils, BinaryBytesUtils}
30-
import org.apache.spark.sql.types.{DataType, AtomicType, StringType}
31-
import org.apache.spark.sql.hbase.catalyst.expressions.PartialPredicateOperations._
32-
3322
/**
3423
* the serializer to serialize / de-serialize the objects for HBase embedded execution,
3524
* may be made configurable and use the ones provided by Spark in the future.

‎src/main/scala/org/apache/spark/sql/hbase/ScanPredClassifier.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
package org.apache.spark.sql.hbase
1919

2020
import org.apache.spark.sql.catalyst.expressions._
21-
import org.apache.spark.sql.hbase.util.{BinaryBytesUtils, DataTypeUtils}
21+
import org.apache.spark.sql.hbase.util.DataTypeUtils
2222

2323
/**
2424
* Classifies a predicate into a pair of (pushdownable, non-pushdownable) predicates

‎src/main/scala/org/apache/spark/sql/hbase/catalyst/expressions/PartialPredicateOperations.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@ package org.apache.spark.sql.hbase.catalyst.expressions
1919

2020
import org.apache.spark.sql.catalyst.errors.TreeNodeException
2121
import org.apache.spark.sql.catalyst.expressions._
22-
import org.apache.spark.sql.types._
23-
import org.apache.spark.sql.hbase.types._
2422
import org.apache.spark.sql.hbase.types.RangeType._
23+
import org.apache.spark.sql.hbase.types._
24+
import org.apache.spark.sql.types._
2525

2626
object PartialPredicateOperations {
2727
// When the checkNullness argument of the partialReduce method is false, the partial

‎src/main/scala/org/apache/spark/sql/hbase/execution/AddCoprocessor.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@ package org.apache.spark.sql.hbase.execution
1919

2020
import org.apache.spark.sql.SQLContext
2121
import org.apache.spark.sql.catalyst.expressions._
22-
import org.apache.spark.sql.execution.expressions._
2322
import org.apache.spark.sql.catalyst.rules.Rule
2423
import org.apache.spark.sql.execution._
24+
import org.apache.spark.sql.execution.expressions._
2525
import org.apache.spark.sql.hbase._
2626

2727
private[hbase] case class AddCoprocessor(sqlContext: SQLContext) extends Rule[SparkPlan] {

‎src/main/scala/org/apache/spark/sql/hbase/execution/HBaseStrategies.scala

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -149,14 +149,14 @@ private[hbase] trait HBaseStrategies {
149149
//or it missed some mid dimensions in the rowkey,
150150
//that means we have to do it with the partial aggregation.
151151
//
152-
//If the groupingExpreesions are composed by all keys,
152+
//If the groupingExpressions are composed by all keys,
153153
//that means it need to be grouped by rowkey in all dimensions,
154154
//so we could do the aggregation for all directly.
155155
if (keysForGroup.size != groupingExpressions.size) aggrWithPartial
156156
else if (keysForGroup.size == hbaseRelation.keyColumns.size) aggrForAll
157157
else {
158158
val partitionsAfterFilter = scanNode.result.partitions
159-
val eachPartionApart = (0 to partitionsAfterFilter.length - 2).forall { case i =>
159+
val eachPartitionApart = (0 to partitionsAfterFilter.length - 2).forall { case i =>
160160
val headEnd = partitionsAfterFilter(i).asInstanceOf[HBasePartition]
161161
.end.get.asInstanceOf[HBaseRawType]
162162
val tailStart = partitionsAfterFilter(i + 1).asInstanceOf[HBasePartition]
@@ -165,7 +165,7 @@ private[hbase] trait HBaseStrategies {
165165
// for the given rowkey dimensions, we could not do the aggregation for all.
166166
distinguishedForGroupKeys(headEnd, tailStart, keysForGroup)
167167
}
168-
if (eachPartionApart) aggrForAll
168+
if (eachPartitionApart) aggrForAll
169169
else aggrWithPartial
170170
}
171171
}

‎src/main/scala/org/apache/spark/sql/hbase/execution/hbaseCommands.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ import org.apache.spark.sql.catalyst.plans.logical.Subquery
3535
import org.apache.spark.sql.execution.RunnableCommand
3636
import org.apache.spark.sql.hbase.HBasePartitioner.HBaseRawOrdering
3737
import org.apache.spark.sql.hbase._
38-
import org.apache.spark.sql.hbase.util.{Util, DataTypeUtils}
38+
import org.apache.spark.sql.hbase.util.{DataTypeUtils, Util}
3939
import org.apache.spark.sql.sources.LogicalRelation
4040
import org.apache.spark.sql.types._
4141
import org.apache.spark.{Logging, SerializableWritable, SparkEnv, TaskContext}

‎src/main/scala/org/apache/spark/sql/hbase/util/DataTypeUtils.scala

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,11 @@
1616
*/
1717
package org.apache.spark.sql.hbase.util
1818

19-
import com.google.protobuf.InvalidProtocolBufferException
20-
import org.apache.hadoop.hbase.exceptions.DeserializationException
21-
import org.apache.hadoop.hbase.filter.{ByteArrayComparable, BinaryComparator}
22-
import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos
23-
import org.apache.hadoop.hbase.util.{ByteStringer, Bytes}
19+
import org.apache.hadoop.hbase.filter.{BinaryComparator, ByteArrayComparable}
2420
import org.apache.spark.sql.catalyst.expressions.{Literal, MutableRow, Row}
2521
import org.apache.spark.sql.execution.SparkSqlSerializer
26-
import org.apache.spark.sql.types._
2722
import org.apache.spark.sql.hbase._
23+
import org.apache.spark.sql.types._
2824

2925
/**
3026
* Data Type conversion utilities

‎src/main/scala/org/apache/spark/sql/hbase/util/HBaseKVHelper.scala

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,6 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, Row}
2121
import org.apache.spark.sql.hbase._
2222
import org.apache.spark.sql.types._
2323

24-
import scala.collection.mutable.ArrayBuffer
25-
2624
object HBaseKVHelper {
2725
val delimiter: Byte = 0
2826

‎src/main/scala/org/apache/spark/sql/hbase/util/bytesUtils.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@
1717
package org.apache.spark.sql.hbase.util
1818

1919
import org.apache.hadoop.hbase.util.Bytes
20-
import org.apache.spark.sql.types._
2120
import org.apache.spark.sql.hbase._
21+
import org.apache.spark.sql.types._
2222

2323
trait BytesUtils {
2424
def create(dataType: DataType): ToBytesUtils

‎src/main/scala/org/apache/spark/sql/hbase/util/comparators.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import com.google.protobuf.InvalidProtocolBufferException
2121
import org.apache.hadoop.hbase.exceptions.DeserializationException
2222
import org.apache.hadoop.hbase.filter.ByteArrayComparable
2323
import org.apache.hadoop.hbase.protobuf.generated.ComparatorProtos
24-
import org.apache.hadoop.hbase.util.{Bytes, ByteStringer}
24+
import org.apache.hadoop.hbase.util.{ByteStringer, Bytes}
2525
import org.apache.spark.sql.hbase._
2626

2727
class CustomComparator(value: Array[Byte]) extends ByteArrayComparable(value) {

‎src/test/java/org/apache/spark/sql/hbase/api/java/JavaAPISuite.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,19 @@
1717

1818
package org.apache.spark.sql.hbase.api.java;
1919

20-
import java.io.Serializable;
21-
2220
import org.apache.hadoop.hbase.HBaseTestingUtility;
2321
import org.apache.hadoop.hbase.MiniHBaseCluster;
2422
import org.apache.hadoop.hbase.client.HBaseAdmin;
2523
import org.apache.spark.SparkConf;
24+
import org.apache.spark.api.java.JavaSparkContext;
25+
import org.apache.spark.sql.Row;
2626
import org.apache.spark.sql.SQLContext;
27-
import org.apache.spark.sql.hbase.*;
27+
import org.apache.spark.sql.hbase.HBaseSQLContext;
28+
import org.apache.spark.sql.hbase.TestBase;
2829
import org.junit.Before;
2930
import org.junit.Test;
3031

31-
import org.apache.spark.api.java.JavaSparkContext;
32-
import org.apache.spark.sql.Row;
32+
import java.io.Serializable;
3333

3434
public class JavaAPISuite extends TestBase implements Serializable {
3535
private transient JavaSparkContext sc;

‎src/test/scala/org/apache/spark/sql/hbase/BytesUtilsSuite.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,11 @@
1717

1818
package org.apache.spark.sql.hbase
1919

20-
import org.apache.spark.Logging
2120
import org.apache.hadoop.hbase.util.Bytes
22-
import org.apache.spark.sql.types._
21+
import org.apache.spark.Logging
2322
import org.apache.spark.sql.hbase.types.HBaseBytesType
2423
import org.apache.spark.sql.hbase.util.BinaryBytesUtils
24+
import org.apache.spark.sql.types._
2525
import org.scalatest.{BeforeAndAfterAll, FunSuite}
2626

2727
class BytesUtilsSuite extends FunSuite with BeforeAndAfterAll with Logging {

‎src/test/scala/org/apache/spark/sql/hbase/CriticalPointsTestSuite.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ package org.apache.spark.sql.hbase
1818

1919
import org.apache.spark._
2020
import org.apache.spark.sql.catalyst.expressions._
21-
import org.apache.spark.sql.types._
2221
import org.apache.spark.sql.hbase.util.{BinaryBytesUtils, HBaseKVHelper}
22+
import org.apache.spark.sql.types._
2323
import org.scalatest.{BeforeAndAfterAll, FunSuite}
2424

2525
import scala.collection.mutable.ArrayBuffer

0 commit comments

Comments
 (0)