spark-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Sumit Khanna <sumit.kha...@askme.in>
Subject Re: java.lang.UnsupportedOperationException: Cannot evaluate expression: fun_nm(input[0, string, true])
Date Tue, 16 Aug 2016 08:57:54 GMT
This is just the stacktrace,but where is it you ccalling the UDF?

Regards,
Sumit

On 16-Aug-2016 2:20 pm, "pseudo oduesp" <pseudo20140@gmail.com> wrote:

> hi,
> i cretae new columns with udf  after i try to filter this columns :
> i get this error why ?
>
> : java.lang.UnsupportedOperationException: Cannot evaluate expression:
> fun_nm(input[0, string, true])
>     at org.apache.spark.sql.catalyst.expressions.Unevaluable$class.
> eval(Expression.scala:221)
>     at org.apache.spark.sql.execution.python.PythonUDF.
> eval(PythonUDF.scala:27)
>     at org.apache.spark.sql.catalyst.expressions.BinaryExpression.
> eval(Expression.scala:408)
>     at org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin$.
> org$apache$spark$sql$catalyst$optimizer$EliminateOuterJoin$$
> canFilterOutNull(Optimizer.scala:1234)
>     at org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin$$
> anonfun$55.apply(Optimizer.scala:1248)
>     at org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin$$
> anonfun$55.apply(Optimizer.scala:1248)
>     at scala.collection.LinearSeqOptimized$class.
> exists(LinearSeqOptimized.scala:93)
>     at scala.collection.immutable.List.exists(List.scala:84)
>     at org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin$.
> org$apache$spark$sql$catalyst$optimizer$EliminateOuterJoin$$
> buildNewJoinType(Optimizer.scala:1248)
>     at org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin$$
> anonfun$apply$30.applyOrElse(Optimizer.scala:1264)
>     at org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin$$
> anonfun$apply$30.applyOrElse(Optimizer.scala:1262)
>     at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.
> apply(TreeNode.scala:279)
>     at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.
> apply(TreeNode.scala:279)
>     at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.
> withOrigin(TreeNode.scala:69)
>     at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(
> TreeNode.scala:278)
>     at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$
> transformDown$1.apply(TreeNode.scala:284)
>     at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$
> transformDown$1.apply(TreeNode.scala:284)
>     at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$5.
> apply(TreeNode.scala:321)
>     at org.apache.spark.sql.catalyst.trees.TreeNode.
> mapProductIterator(TreeNode.scala:179)
>     at org.apache.spark.sql.catalyst.trees.TreeNode.
> transformChildren(TreeNode.scala:319)
>     at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(
> TreeNode.scala:284)
>     at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$
> transformDown$1.apply(TreeNode.scala:284)
>     at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$
> transformDown$1.apply(TreeNode.scala:284)
>     at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$5.
> apply(TreeNode.scala:321)
>     at org.apache.spark.sql.catalyst.trees.TreeNode.
> mapProductIterator(TreeNode.scala:179)
>     at org.apache.spark.sql.catalyst.trees.TreeNode.
> transformChildren(TreeNode.scala:319)
>     at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(
> TreeNode.scala:284)
>     at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$
> transformDown$1.apply(TreeNode.scala:284)
>     at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$
> transformDown$1.apply(TreeNode.scala:284)
>     at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$5.
> apply(TreeNode.scala:321)
>     at org.apache.spark.sql.catalyst.trees.TreeNode.
> mapProductIterator(TreeNode.scala:179)
>     at org.apache.spark.sql.catalyst.trees.TreeNode.
> transformChildren(TreeNode.scala:319)
>     at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(
> TreeNode.scala:284)
>     at org.apache.spark.sql.catalyst.trees.TreeNode.transform(
> TreeNode.scala:268)
>     at org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin$.
> apply(Optimizer.scala:1262)
>     at org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin$.
> apply(Optimizer.scala:1225)
>     at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$
> execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:85)
>     at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$
> execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:82)
>     at scala.collection.IndexedSeqOptimized$class.
> foldl(IndexedSeqOptimized.scala:57)
>     at scala.collection.IndexedSeqOptimized$class.
> foldLeft(IndexedSeqOptimized.scala:66)
>     at scala.collection.mutable.WrappedArray.foldLeft(
> WrappedArray.scala:35)
>     at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$
> execute$1.apply(RuleExecutor.scala:82)
>     at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$
> execute$1.apply(RuleExecutor.scala:74)
>     at scala.collection.immutable.List.foreach(List.scala:381)
>     at org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(
> RuleExecutor.scala:74)
>     at org.apache.spark.sql.execution.QueryExecution.
> optimizedPlan$lzycompute(QueryExecution.scala:74)
>     at org.apache.spark.sql.execution.QueryExecution.
> optimizedPlan(QueryExecution.scala:74)
>     at org.apache.spark.sql.execution.QueryExecution.sparkPlan$lzycompute(
> QueryExecution.scala:78)
>     at org.apache.spark.sql.execution.QueryExecution.
> sparkPlan(QueryExecution.scala:76)
>     at org.apache.spark.sql.execution.QueryExecution.
> executedPlan$lzycompute(QueryExecution.scala:83)
>     at org.apache.spark.sql.execution.QueryExecution.
> executedPlan(QueryExecution.scala:83)
>     at org.apache.spark.sql.Dataset.withTypedCallback(Dataset.scala:2558)
>     at org.apache.spark.sql.Dataset.head(Dataset.scala:1924)
>     at org.apache.spark.sql.Dataset.take(Dataset.scala:2139)
>     at org.apache.spark.sql.Dataset.showString(Dataset.scala:239)
>     at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>     at sun.reflect.NativeMethodAccessorImpl.invoke(
> NativeMethodAccessorImpl.java:57)
>     at sun.reflect.DelegatingMethodAccessorImpl.invoke(
> DelegatingMethodAccessorImpl.java:43)
>     at java.lang.reflect.Method.invoke(Method.java:606)
>     at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:237)
>     at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
>     at py4j.Gateway.invoke(Gateway.java:280)
>     at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.
> java:128)
>     at py4j.commands.CallCommand.execute(CallCommand.java:79)
>     at py4j.GatewayConnection.run(GatewayConnection.java:211)
>     at java.lang.Thread.run(Thread.java:745)
>
>

Mime
View raw message