Skip to content

Commit 91d4b34

Browse files
xinyunhxinyunh
committed
Fix the UTF8String type error
1 parent 3bc62e3 commit 91d4b34

File tree

2 files changed

+26
-24
lines changed

2 files changed

+26
-24
lines changed

‎src/test/scala/org/apache/spark/sql/hbase/HBasePartitionerSuite.scala

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import org.apache.spark.rdd.ShuffledRDD
2323
import org.apache.spark.sql.catalyst.expressions._
2424
import org.apache.spark.sql.hbase.util.{BinaryBytesUtils, HBaseKVHelper}
2525
import org.apache.spark.sql.types._
26+
import org.apache.spark.unsafe.types.UTF8String
2627

2728
import scala.collection.mutable.ArrayBuffer
2829

@@ -69,8 +70,8 @@ class HBasePartitionerSuite extends TestBase {
6970
KeyColumn("col3", StringType, 2), KeyColumn("col4", IntegerType, 3)))
7071

7172
assert(BinaryBytesUtils.toDouble(rowkey, keys.head._1) === 123.456)
72-
assert(BinaryBytesUtils.toUTF8String(rowkey, keys(1)._1, keys(1)._2) === "abcdef")
73-
assert(BinaryBytesUtils.toUTF8String(rowkey, keys(2)._1, keys(2)._2) === "")
73+
assert(BinaryBytesUtils.toUTF8String(rowkey, keys(1)._1, keys(1)._2) === UTF8String.fromString("abcdef"))
74+
assert(BinaryBytesUtils.toUTF8String(rowkey, keys(2)._1, keys(2)._2) === UTF8String.fromString(""))
7475
assert(BinaryBytesUtils.toInt(rowkey, keys(3)._1) === 1234)
7576
}
7677

@@ -88,7 +89,7 @@ class HBasePartitionerSuite extends TestBase {
8889
KeyColumn("col3", IntegerType, 2)))
8990

9091
assert(BinaryBytesUtils.toDouble(rowkey, keys.head._1) === 123.456)
91-
assert(BinaryBytesUtils.toUTF8String(rowkey, keys(1)._1, keys(1)._2) === "abcdef")
92+
assert(BinaryBytesUtils.toUTF8String(rowkey, keys(1)._1, keys(1)._2) === UTF8String.fromString("abcdef"))
9293
assert(BinaryBytesUtils.toInt(rowkey, keys(2)._1) === 1234)
9394
}
9495

‎src/test/scala/org/apache/spark/sql/hbase/TestBaseWithSplitData.scala

Lines changed: 22 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import org.apache.spark.sql.catalyst.InternalRow
2626
import org.apache.spark.sql.catalyst.expressions.{GenericInternalRow, GenericRow}
2727
import org.apache.spark.sql.hbase.util.{BinaryBytesUtils, DataTypeUtils, HBaseKVHelper}
2828
import org.apache.spark.sql.types._
29+
import org.apache.spark.unsafe.types.UTF8String
2930

3031
/**
3132
* HBaseMainTest
@@ -73,13 +74,13 @@ class TestBaseWithSplitData extends TestBase {
7374

7475
val splitKeys: Array[Array[Byte]] = if (useMultiplePartitions) {
7576
Array(
76-
new GenericInternalRow(Array(256, " p256 ", 128: Short)),
77-
new GenericInternalRow(Array(32, " p32 ", 256: Short)),
78-
new GenericInternalRow(Array(-32, " n32 ", 128: Short)),
79-
new GenericInternalRow(Array(-256, " n256 ", 256: Short)),
80-
new GenericInternalRow(Array(-128, " n128 ", 128: Short)),
81-
new GenericInternalRow(Array(0, " zero ", 256: Short)),
82-
new GenericInternalRow(Array(128, " p128 ", 512: Short))
77+
new GenericInternalRow(Array(256, UTF8String.fromString(" p256 "), 128: Short)),
78+
new GenericInternalRow(Array(32, UTF8String.fromString(" p32 "), 256: Short)),
79+
new GenericInternalRow(Array(-32, UTF8String.fromString(" n32 "), 128: Short)),
80+
new GenericInternalRow(Array(-256, UTF8String.fromString(" n256 "), 256: Short)),
81+
new GenericInternalRow(Array(-128, UTF8String.fromString(" n128 "), 128: Short)),
82+
new GenericInternalRow(Array(0, UTF8String.fromString(" zero "), 256: Short)),
83+
new GenericInternalRow(Array(128, UTF8String.fromString(" p128 "), 512: Short))
8384
).map(HBaseKVHelper.makeRowKey(_, Seq(IntegerType, StringType, ShortType)))
8485
} else {
8586
null
@@ -125,72 +126,72 @@ class TestBaseWithSplitData extends TestBase {
125126
htable.put(put)
126127
}
127128

128-
putNewTableIntoHBase(Seq(-257, " n257 ", 128: Short),
129+
putNewTableIntoHBase(Seq(-257, UTF8String.fromString(" n257 "), 128: Short),
129130
Seq(IntegerType, StringType, ShortType),
130131
Seq[Any](1.toByte, -2048, 12345678901234L, 1234.5678F),
131132
Seq(ByteType, IntegerType, LongType, FloatType))
132133

133-
putNewTableIntoHBase(Seq(-255, " n255 ", 128: Short),
134+
putNewTableIntoHBase(Seq(-255, UTF8String.fromString(" n255 "), 128: Short),
134135
Seq(IntegerType, StringType, ShortType),
135136
Seq[Any](2.toByte, -1024, 12345678901234L, 1234.5678F),
136137
Seq(ByteType, IntegerType, LongType, FloatType))
137138

138-
putNewTableIntoHBase(Seq(-129, " n129 ", 128: Short),
139+
putNewTableIntoHBase(Seq(-129, UTF8String.fromString(" n129 "), 128: Short),
139140
Seq(IntegerType, StringType, ShortType),
140141
Seq[Any](3.toByte, -512, 12345678901234L, 1234.5678F),
141142
Seq(ByteType, IntegerType, LongType, FloatType))
142143

143-
putNewTableIntoHBase(Seq(-127, " n127 ", 128: Short),
144+
putNewTableIntoHBase(Seq(-127, UTF8String.fromString(" n127 "), 128: Short),
144145
Seq(IntegerType, StringType, ShortType),
145146
Seq[Any](4.toByte, -256, 12345678901234L, 1234.5678F),
146147
Seq(ByteType, IntegerType, LongType, FloatType))
147148

148-
putNewTableIntoHBase(Seq(-33, " n33 ", 128: Short),
149+
putNewTableIntoHBase(Seq(-33, UTF8String.fromString(" n33 "), 128: Short),
149150
Seq(IntegerType, StringType, ShortType),
150151
Seq[Any](5.toByte, -128, 12345678901234L, 1234.5678F),
151152
Seq(ByteType, IntegerType, LongType, FloatType))
152153

153-
putNewTableIntoHBase(Seq(-31, " n31 ", 128: Short),
154+
putNewTableIntoHBase(Seq(-31, UTF8String.fromString(" n31 "), 128: Short),
154155
Seq(IntegerType, StringType, ShortType),
155156
Seq[Any](6.toByte, -64, 12345678901234L, 1234.5678F),
156157
Seq(ByteType, IntegerType, LongType, FloatType))
157158

158-
putNewTableIntoHBase(Seq(-1, " n1 ", 128: Short),
159+
putNewTableIntoHBase(Seq(-1, UTF8String.fromString(" n1 "), 128: Short),
159160
Seq(IntegerType, StringType, ShortType),
160161
Seq[Any](7.toByte, -1, 12345678901234L, 1234.5678F),
161162
Seq(ByteType, IntegerType, LongType, FloatType))
162163

163-
putNewTableIntoHBase(Seq(1, " p1 ", 128: Short),
164+
putNewTableIntoHBase(Seq(1, UTF8String.fromString(" p1 "), 128: Short),
164165
Seq(IntegerType, StringType, ShortType),
165166
Seq[Any](8.toByte, 1, 12345678901234L, 1234.5678F),
166167
Seq(ByteType, IntegerType, LongType, FloatType))
167168

168-
putNewTableIntoHBase(Seq(31, " p31 ", 128: Short),
169+
putNewTableIntoHBase(Seq(31, UTF8String.fromString(" p31 "), 128: Short),
169170
Seq(IntegerType, StringType, ShortType),
170171
Seq[Any](9.toByte, 4, 12345678901234L, 1234.5678F),
171172
Seq(ByteType, IntegerType, LongType, FloatType))
172173

173-
putNewTableIntoHBase(Seq(33, " p33 ", 128: Short),
174+
putNewTableIntoHBase(Seq(33, UTF8String.fromString(" p33 "), 128: Short),
174175
Seq(IntegerType, StringType, ShortType),
175176
Seq[Any](10.toByte, 64, 12345678901234L, 1234.5678F),
176177
Seq(ByteType, IntegerType, LongType, FloatType))
177178

178-
putNewTableIntoHBase(Seq(127, " p127 ", 128: Short),
179+
putNewTableIntoHBase(Seq(127, UTF8String.fromString(" p127 "), 128: Short),
179180
Seq(IntegerType, StringType, ShortType),
180181
Seq[Any](11.toByte, 128, 12345678901234L, 1234.5678F),
181182
Seq(ByteType, IntegerType, LongType, FloatType))
182183

183-
putNewTableIntoHBase(Seq(129, " p129 ", 128: Short),
184+
putNewTableIntoHBase(Seq(129, UTF8String.fromString(" p129 "), 128: Short),
184185
Seq(IntegerType, StringType, ShortType),
185186
Seq[Any](12.toByte, 256, 12345678901234L, 1234.5678F),
186187
Seq(ByteType, IntegerType, LongType, FloatType))
187188

188-
putNewTableIntoHBase(Seq(255, " p255 ", 128: Short),
189+
putNewTableIntoHBase(Seq(255, UTF8String.fromString(" p255 "), 128: Short),
189190
Seq(IntegerType, StringType, ShortType),
190191
Seq[Any](13.toByte, 512, 12345678901234L, 1234.5678F),
191192
Seq(ByteType, IntegerType, LongType, FloatType))
192193

193-
putNewTableIntoHBase(Seq(257, " p257 ", 128: Short),
194+
putNewTableIntoHBase(Seq(257, UTF8String.fromString(" p257 "), 128: Short),
194195
Seq(IntegerType, StringType, ShortType),
195196
Seq[Any](14.toByte, 1024, 12345678901234L, 1234.5678F),
196197
Seq(ByteType, IntegerType, LongType, FloatType))

0 commit comments

Comments
 (0)