[SPARK-12138][SQL] Escape \u in the generated comments of codegen

gatorsmile · yhuai · commit 93a0510a5d62 · 2015-12-06T11:17:10.000-08:00
When \u appears in a comment block (i.e. in /**/), code gen will break. So, in Expression and CodegenFallback, we escape \u to \\u. yhuai Please review it. I did reproduce it and it works after the fix. Thanks! Author: gatorsmile <gatorsmile@gmail.com> Closes apache#10155 from gatorsmile/escapeU. (cherry picked from commit 49efd03) Signed-off-by: Yin Huai <yhuai@databricks.com>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
@@ -180,7 +180,9 @@ abstract class Expression extends TreeNode[Expression] {
    * Returns the string representation of this expression that is safe to be put in
    * code comments of generated code.
    */
-  protected def toCommentSafeString: String = this.toString.replace("*/", "\\*\\/")
+  protected def toCommentSafeString: String = this.toString
+    .replace("*/", "\\*\\/")
+    .replace("\\u", "\\\\u")
 }
 
 
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CodeGenerationSuite.scala
@@ -143,4 +143,13 @@ class CodeGenerationSuite extends SparkFunSuite with ExpressionEvalHelper {
       true,
       InternalRow(UTF8String.fromString("*/")))
   }
+
+  test("\\u in the data") {
+    // When \ u appears in a comment block (i.e. in /**/), code gen will break.
+    // So, in Expression and CodegenFallback, we escape \ u to \\u.
+    checkEvaluation(
+      EqualTo(BoundReference(0, StringType, false), Literal.create("\\u", StringType)),
+      true,
+      InternalRow(UTF8String.fromString("\\u")))
+  }
 }

Original file line number	Diff line number	Diff line change
`@@ -180,7 +180,9 @@ abstract class Expression extends TreeNode[Expression] {`
`180`	`180`	`* Returns the string representation of this expression that is safe to be put in`
`181`	`181`	`* code comments of generated code.`
`182`	`182`	`*/`
`183`		`- protected def toCommentSafeString: String = this.toString.replace("/", "\\\\/")`
	`183`	`+ protected def toCommentSafeString: String = this.toString`
	`184`	`+ .replace("/", "\\\\/")`
	`185`	`+ .replace("\\u", "\\\\u")`
`184`	`186`	`}`
`185`	`187`
`186`	`188`
Original file line number	Diff line number	Diff line change
`@@ -143,4 +143,13 @@ class CodeGenerationSuite extends SparkFunSuite with ExpressionEvalHelper {`
`143`	`143`	`true,`
`144`	`144`	`InternalRow(UTF8String.fromString("*/")))`
`145`	`145`	`}`
	`146`	`+`
	`147`	`+ test("\\u in the data") {`
	`148`	`+ // When \ u appears in a comment block (i.e. in /**/), code gen will break.`
	`149`	`+ // So, in Expression and CodegenFallback, we escape \ u to \\u.`
	`150`	`+ checkEvaluation(`
	`151`	`+ EqualTo(BoundReference(0, StringType, false), Literal.create("\\u", StringType)),`
	`152`	`+ true,`
	`153`	`+ InternalRow(UTF8String.fromString("\\u")))`
	`154`	`+ }`
`146`	`155`	`}`