[SPARK-10593] [SQL] fix resolve output of Generate

Davies Liu · yhuai · commit c3112a92ff7d · 2015-09-22T11:07:54.000-07:00
The output of Generate should not be resolved as Reference. Author: Davies Liu <davies@databricks.com> Closes apache#8755 from davies/view. (cherry picked from commit 22d4015) Signed-off-by: Yin Huai <yhuai@databricks.com>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -378,6 +378,22 @@ class Analyzer(
         val newOrdering = resolveSortOrders(ordering, child, throws = false)
         Sort(newOrdering, global, child)
 
+      // A special case for Generate, because the output of Generate should not be resolved by
+      // ResolveReferences. Attributes in the output will be resolved by ResolveGenerate.
+      case g @ Generate(generator, join, outer, qualifier, output, child)
+        if child.resolved && !generator.resolved =>
+        val newG = generator transformUp {
+          case u @ UnresolvedAttribute(nameParts) =>
+            withPosition(u) { child.resolve(nameParts, resolver).getOrElse(u) }
+          case UnresolvedExtractValue(child, fieldExpr) =>
+            ExtractValue(child, fieldExpr, resolver)
+        }
+        if (newG.fastEquals(generator)) {
+          g
+        } else {
+          Generate(newG.asInstanceOf[Generator], join, outer, qualifier, output, child)
+        }
+
       case q: LogicalPlan =>
         logTrace(s"Attempting to resolve ${q.simpleString}")
         q transformExpressionsUp  {
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/QueryPlan.scala
@@ -18,7 +18,6 @@
 package org.apache.spark.sql.catalyst.plans
 
 import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeSet, Expression, VirtualColumn}
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.trees.TreeNode
 import org.apache.spark.sql.types.{DataType, StructType}
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicOperators.scala
@@ -68,7 +68,7 @@ case class Generate(
     generator.resolved &&
       childrenResolved &&
       generator.elementTypes.length == generatorOutput.length &&
-      !generatorOutput.exists(!_.resolved)
+      generatorOutput.forall(_.resolved)
   }
 
   // we don't want the gOutput to be taken as part of the expressions
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -1173,4 +1173,18 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils {
       checkAnswer(sqlContext.table("`db.t`"), df)
     }
   }
+
+  test("SPARK-10593 same column names in lateral view") {
+    val df = sqlContext.sql(
+    """
+      |select
+      |insideLayer2.json as a2
+      |from (select '{"layer1": {"layer2": "text inside layer 2"}}' json) test
+      |lateral view json_tuple(json, 'layer1') insideLayer1 as json
+      |lateral view json_tuple(insideLayer1.json, 'layer2') insideLayer2 as json
+    """.stripMargin
+    )
+
+    checkAnswer(df, Row("text inside layer 2") :: Nil)
+  }
 }

Original file line number	Diff line number	Diff line change
`@@ -68,7 +68,7 @@ case class Generate(`
`68`	`68`	`generator.resolved &&`
`69`	`69`	`childrenResolved &&`
`70`	`70`	`generator.elementTypes.length == generatorOutput.length &&`
`71`		`- !generatorOutput.exists(!_.resolved)`
	`71`	`+ generatorOutput.forall(_.resolved)`
`72`	`72`	`}`
`73`	`73`
`74`	`74`	`// we don't want the gOutput to be taken as part of the expressions`
Original file line number	Diff line number	Diff line change
`@@ -1173,4 +1173,18 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils {`
`1173`	`1173`	checkAnswer(sqlContext.table("`db.t`"), df)
`1174`	`1174`	`}`
`1175`	`1175`	`}`
	`1176`	`+`
	`1177`	`+ test("SPARK-10593 same column names in lateral view") {`
	`1178`	`+ val df = sqlContext.sql(`
	`1179`	`+ """`
	`1180`	`+ \|select`
	`1181`	`+ \|insideLayer2.json as a2`
	`1182`	`+ \|from (select '{"layer1": {"layer2": "text inside layer 2"}}' json) test`
	`1183`	`+ \|lateral view json_tuple(json, 'layer1') insideLayer1 as json`
	`1184`	`+ \|lateral view json_tuple(insideLayer1.json, 'layer2') insideLayer2 as json`
	`1185`	`+ """.stripMargin`
	`1186`	`+ )`
	`1187`	`+`
	`1188`	`+ checkAnswer(df, Row("text inside layer 2") :: Nil)`
	`1189`	`+ }`
`1176`	`1190`	`}`