[SPARK-12562][SQL] DataFrame.write.format(text) requires the column name to be called value

xguo27 · rxin · commit 84f8492c1555 · 2016-01-03T20:48:56.000-08:00
Author: Xiu Guo <xguo27@gmail.com> Closes apache#10515 from xguo27/SPARK-12562.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/text/DefaultSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/text/DefaultSource.scala
@@ -48,7 +48,7 @@ class DefaultSource extends HadoopFsRelationProvider with DataSourceRegister {
       partitionColumns: Option[StructType],
       parameters: Map[String, String]): HadoopFsRelation = {
     dataSchema.foreach(verifySchema)
-    new TextRelation(None, partitionColumns, paths)(sqlContext)
+    new TextRelation(None, dataSchema, partitionColumns, paths)(sqlContext)
   }
 
   override def shortName(): String = "text"
@@ -68,15 +68,16 @@ class DefaultSource extends HadoopFsRelationProvider with DataSourceRegister {
 
 private[sql] class TextRelation(
     val maybePartitionSpec: Option[PartitionSpec],
+    val textSchema: Option[StructType],
     override val userDefinedPartitionColumns: Option[StructType],
     override val paths: Array[String] = Array.empty[String],
     parameters: Map[String, String] = Map.empty[String, String])
     (@transient val sqlContext: SQLContext)
   extends HadoopFsRelation(maybePartitionSpec, parameters) {
 
-  /** Data schema is always a single column, named "value". */
-  override def dataSchema: StructType = new StructType().add("value", StringType)
-
+  /** Data schema is always a single column, named "value" if original Data source has no schema. */
+  override def dataSchema: StructType =
+    textSchema.getOrElse(new StructType().add("value", StringType))
   /** This is an internal data source that outputs internal row format. */
   override val needConversion: Boolean = false
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/text/TextSuite.scala
@@ -33,8 +33,8 @@ class TextSuite extends QueryTest with SharedSQLContext {
     verifyFrame(sqlContext.read.text(testFile))
   }
 
-  test("writing") {
-    val df = sqlContext.read.text(testFile)
+  test("SPARK-12562 verify write.text() can handle column name beyond `value`") {
+    val df = sqlContext.read.text(testFile).withColumnRenamed("value", "adwrasdf")
 
     val tempFile = Utils.createTempDir()
     tempFile.delete()

Original file line number	Diff line number	Diff line change
`@@ -33,8 +33,8 @@ class TextSuite extends QueryTest with SharedSQLContext {`
`33`	`33`	`verifyFrame(sqlContext.read.text(testFile))`
`34`	`34`	`}`
`35`	`35`
`36`		`- test("writing") {`
`37`		`- val df = sqlContext.read.text(testFile)`
	`36`	+ test("SPARK-12562 verify write.text() can handle column name beyond `value`") {
	`37`	`+ val df = sqlContext.read.text(testFile).withColumnRenamed("value", "adwrasdf")`
`38`	`38`
`39`	`39`	`val tempFile = Utils.createTempDir()`
`40`	`40`	`tempFile.delete()`