Skip to content

Commit 9cf56c9

Browse files
yhuaiyhuai
committed
[SPARK-11469][SQL] Allow users to define nondeterministic udfs.
This is the first task (https://issues.apache.org/jira/browse/SPARK-11469) of https://issues.apache.org/jira/browse/SPARK-11438 Author: Yin Huai <yhuai@databricks.com> Closes apache#9393 from yhuai/udfNondeterministic.
1 parent efaa472 commit 9cf56c9

File tree

6 files changed

+262
-78
lines changed

6 files changed

+262
-78
lines changed

‎project/MimaExcludes.scala

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,53 @@ object MimaExcludes {
112112
"org.apache.spark.rdd.MapPartitionsWithPreparationRDD"),
113113
ProblemFilters.exclude[MissingClassProblem](
114114
"org.apache.spark.rdd.MapPartitionsWithPreparationRDD$")
115+
) ++ Seq(
116+
ProblemFilters.exclude[MissingMethodProblem](
117+
"org.apache.spark.sql.UDFRegistration.org$apache$spark$sql$UDFRegistration$$builder$2"),
118+
ProblemFilters.exclude[MissingMethodProblem](
119+
"org.apache.spark.sql.UDFRegistration.org$apache$spark$sql$UDFRegistration$$builder$3"),
120+
ProblemFilters.exclude[MissingMethodProblem](
121+
"org.apache.spark.sql.UDFRegistration.org$apache$spark$sql$UDFRegistration$$builder$4"),
122+
ProblemFilters.exclude[MissingMethodProblem](
123+
"org.apache.spark.sql.UDFRegistration.org$apache$spark$sql$UDFRegistration$$builder$5"),
124+
ProblemFilters.exclude[MissingMethodProblem](
125+
"org.apache.spark.sql.UDFRegistration.org$apache$spark$sql$UDFRegistration$$builder$6"),
126+
ProblemFilters.exclude[MissingMethodProblem](
127+
"org.apache.spark.sql.UDFRegistration.org$apache$spark$sql$UDFRegistration$$builder$7"),
128+
ProblemFilters.exclude[MissingMethodProblem](
129+
"org.apache.spark.sql.UDFRegistration.org$apache$spark$sql$UDFRegistration$$builder$8"),
130+
ProblemFilters.exclude[MissingMethodProblem](
131+
"org.apache.spark.sql.UDFRegistration.org$apache$spark$sql$UDFRegistration$$builder$9"),
132+
ProblemFilters.exclude[MissingMethodProblem](
133+
"org.apache.spark.sql.UDFRegistration.org$apache$spark$sql$UDFRegistration$$builder$10"),
134+
ProblemFilters.exclude[MissingMethodProblem](
135+
"org.apache.spark.sql.UDFRegistration.org$apache$spark$sql$UDFRegistration$$builder$11"),
136+
ProblemFilters.exclude[MissingMethodProblem](
137+
"org.apache.spark.sql.UDFRegistration.org$apache$spark$sql$UDFRegistration$$builder$12"),
138+
ProblemFilters.exclude[MissingMethodProblem](
139+
"org.apache.spark.sql.UDFRegistration.org$apache$spark$sql$UDFRegistration$$builder$13"),
140+
ProblemFilters.exclude[MissingMethodProblem](
141+
"org.apache.spark.sql.UDFRegistration.org$apache$spark$sql$UDFRegistration$$builder$14"),
142+
ProblemFilters.exclude[MissingMethodProblem](
143+
"org.apache.spark.sql.UDFRegistration.org$apache$spark$sql$UDFRegistration$$builder$15"),
144+
ProblemFilters.exclude[MissingMethodProblem](
145+
"org.apache.spark.sql.UDFRegistration.org$apache$spark$sql$UDFRegistration$$builder$16"),
146+
ProblemFilters.exclude[MissingMethodProblem](
147+
"org.apache.spark.sql.UDFRegistration.org$apache$spark$sql$UDFRegistration$$builder$17"),
148+
ProblemFilters.exclude[MissingMethodProblem](
149+
"org.apache.spark.sql.UDFRegistration.org$apache$spark$sql$UDFRegistration$$builder$18"),
150+
ProblemFilters.exclude[MissingMethodProblem](
151+
"org.apache.spark.sql.UDFRegistration.org$apache$spark$sql$UDFRegistration$$builder$19"),
152+
ProblemFilters.exclude[MissingMethodProblem](
153+
"org.apache.spark.sql.UDFRegistration.org$apache$spark$sql$UDFRegistration$$builder$20"),
154+
ProblemFilters.exclude[MissingMethodProblem](
155+
"org.apache.spark.sql.UDFRegistration.org$apache$spark$sql$UDFRegistration$$builder$21"),
156+
ProblemFilters.exclude[MissingMethodProblem](
157+
"org.apache.spark.sql.UDFRegistration.org$apache$spark$sql$UDFRegistration$$builder$22"),
158+
ProblemFilters.exclude[MissingMethodProblem](
159+
"org.apache.spark.sql.UDFRegistration.org$apache$spark$sql$UDFRegistration$$builder$23"),
160+
ProblemFilters.exclude[MissingMethodProblem](
161+
"org.apache.spark.sql.UDFRegistration.org$apache$spark$sql$UDFRegistration$$builder$24")
115162
)
116163
case v if v.startsWith("1.5") =>
117164
Seq(

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,18 @@ case class ScalaUDF(
3030
function: AnyRef,
3131
dataType: DataType,
3232
children: Seq[Expression],
33-
inputTypes: Seq[DataType] = Nil)
33+
inputTypes: Seq[DataType] = Nil,
34+
isDeterministic: Boolean = true)
3435
extends Expression with ImplicitCastInputTypes with CodegenFallback {
3536

3637
override def nullable: Boolean = true
3738

3839
override def toString: String = s"UDF(${children.mkString(",")})"
3940

41+
override def foldable: Boolean = deterministic && children.forall(_.foldable)
42+
43+
override def deterministic: Boolean = isDeterministic && children.forall(_.deterministic)
44+
4045
// scalastyle:off
4146

4247
/** This method has been generated by this script

0 commit comments

Comments
 (0)