spark-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Yuta Morisawa <yu-moris...@kddi-research.jp>
Subject Anonymous functions cannot be found
Date Fri, 06 Sep 2019 07:06:29 GMT
Hi

I'm trying to use sparkContext.addJar method for adding new jar files
like TomCat. But in some cases, it does not work well.

The error message says an Executor can not load an anonymous function.
Why anonymous functions cannot be loaded in spite of adding a jar to all 
Executors?


This is my code.
---
package packageA
object AddJar{

  def main() ={
     val spark = SparkSession
       .builder
       .appName("AddJar")
       .getOrCreate()

     spark.sparkContext.addJar(jarPath)

     val urls:Array[URL] = Array(new URL(jarPath))
     val cl = new URLClassLoader(urls,parentClassloader)
     val loaded:Class[_] = cl.loadClass(className)
     val instance:Any = loaded.newInstance()
     var method: Method = loaded.getDeclaredMethod(methodName,classOf[T])
     method.invoke(instance, params)
   }
}

package packageA
class AnoFun {
   def main() = {
     val spark = SparkSession
       .builder
       .getOrCreate()

     val ds = Seq(Seq(1,2,3,4,5), Seq(2,3,4,5,6)).toDS()
      ds.foreach{ x =>
        x.fold(0)(_ + _)
      }
   }
}
---

and error messages

------
Exception in thread "Thread-30" java.lang.reflect.InvocationTargetException
         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
         at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
         at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
         at java.lang.reflect.Method.invoke(Method.java:498)
         at streamupdate.ExecClass.run(Streaming.scala:142)
         at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.ClassNotFoundException:
packageA.AnoFun$$anonfun$main$1$$anonfun$apply$1
         at java.net.URLClassLoader.findClass(URLClassLoader.java:382)
         at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
         at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
         at java.lang.Class.forName0(Native Method)
         at java.lang.Class.forName(Class.java:348)
         at
org.apache.spark.util.InnerClosureFinder$$anon$4.visitMethodInsn(ClosureCleaner.scala:552)
         at 
org.apache.xbean.asm6.ClassReader.readCode(ClassReader.java:2175)
         at
org.apache.xbean.asm6.ClassReader.readMethod(ClassReader.java:1238)
         at org.apache.xbean.asm6.ClassReader.accept(ClassReader.java:631)
         at org.apache.xbean.asm6.ClassReader.accept(ClassReader.java:355)
         at
org.apache.spark.util.ClosureCleaner$.getInnerClosureClasses(ClosureCleaner.scala:90)
         at
org.apache.spark.util.ClosureCleaner$.org$apache$spark$util$ClosureCleaner$$clean(ClosureCleaner.scala:269)
         at
org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:162)
         at org.apache.spark.SparkContext.clean(SparkContext.scala:2326)
         at org.apache.spark.rdd.RDD$$anonfun$foreach$1.apply(RDD.scala:926)
         at org.apache.spark.rdd.RDD$$anonfun$foreach$1.apply(RDD.scala:925)
         at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
         at
org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
         at org.apache.spark.rdd.RDD.withScope(RDD.scala:363)
         at org.apache.spark.rdd.RDD.foreach(RDD.scala:925)
         at
org.apache.spark.sql.Dataset$$anonfun$foreach$1.apply$mcV$sp(Dataset.scala:2716)
         at
org.apache.spark.sql.Dataset$$anonfun$foreach$1.apply(Dataset.scala:2716)
         at
org.apache.spark.sql.Dataset$$anonfun$foreach$1.apply(Dataset.scala:2716)
         at
org.apache.spark.sql.Dataset$$anonfun$withNewRDDExecutionId$1.apply(Dataset.scala:3349)
         at
org.apache.spark.sql.execution.SQLExecution$$anonfun$withNewExecutionId$1.apply(SQLExecution.scala:78)
         at
org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:125)
         at
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:73)
         at
org.apache.spark.sql.Dataset.withNewRDDExecutionId(Dataset.scala:3345)
         at org.apache.spark.sql.Dataset.foreach(Dataset.scala:2715)
         at packageA.AnoFun.main(AnoFun.scala:66)
         ... 6 more
------

--
Regards,



---------------------------------------------------------------------
To unsubscribe e-mail: user-unsubscribe@spark.apache.org


Mime
View raw message