[SPARK-12242][SQL] Add DataFrame.transform method

rxin · rxin · commit f939c71b187c · 2015-12-10T22:23:26.000+08:00
Author: Reynold Xin <rxin@databricks.com> Closes apache#10226 from rxin/df-transform. (cherry picked from commit 76540b6) Signed-off-by: Reynold Xin <rxin@databricks.com>
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
@@ -84,7 +84,7 @@ class TypedColumn[-T, U](
  *   col("`a.column.with.dots`") // Escape `.` in column names.
  *   $"columnName"               // Scala short hand for a named column.
  *   expr("a + 1")               // A column that is constructed from a parsed SQL Expression.
- *   lit("1")                    // A column that produces a literal (constant) value.
+ *   lit("abc")                  // A column that produces a literal (constant) value.
  * }}}
  *
  * [[Column]] objects can be composed to form complex expressions:
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
@@ -1413,6 +1413,19 @@ class DataFrame private[sql](
    */
   def first(): Row = head()
 
+  /**
+   * Concise syntax for chaining custom transformations.
+   * {{{
+   *   def featurize(ds: DataFrame) = ...
+   *
+   *   df
+   *     .transform(featurize)
+   *     .transform(...)
+   * }}}
+   * @since 1.6.0
+   */
+  def transform[U](t: DataFrame => DataFrame): DataFrame = t(this)
+
   /**
    * Returns a new RDD by applying a function to all rows of this DataFrame.
    * @group rdd

Original file line number	Diff line number	Diff line change
`@@ -84,7 +84,7 @@ class TypedColumn[-T, U](`
`84`	`84`	* col("`a.column.with.dots`") // Escape `.` in column names.
`85`	`85`	`* $"columnName" // Scala short hand for a named column.`
`86`	`86`	`* expr("a + 1") // A column that is constructed from a parsed SQL Expression.`
`87`		`- * lit("1") // A column that produces a literal (constant) value.`
	`87`	`+ * lit("abc") // A column that produces a literal (constant) value.`
`88`	`88`	`* }}}`
`89`	`89`	`*`
`90`	`90`	`* [[Column]] objects can be composed to form complex expressions:`