spark-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Milad khajavi <khaj...@gmail.com>
Subject java.lang.ClassCastException: cannot assign instance of java.lang.invoke.SerializedLambda
Date Tue, 27 Jan 2015 18:49:27 GMT
Hi all,

I can run spark job pragmatically in j2SE with following code without any error:

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;

public class TestSpark {
    public static void main() {
        String sourcePath = "hdfs://spark:54310/input/*";

        SparkConf conf = new SparkConf().setAppName("TestLineCount");
        conf.setJars(new String[] { TestSpark.class.getProtectionDomain()
                .getCodeSource().getLocation().getPath() });
        conf.setMaster("spark://tootak:7077");
        conf.set("spark.driver.allowMultipleContexts", "true");

        @SuppressWarnings("resource")
        JavaSparkContext sc = new JavaSparkContext(conf);
        JavaRDD<String> log = sc.textFile(sourcePath);

        JavaRDD<String> lines = log.filter(x -> {
                return true;
        });

        System.out.println(lines.count());
    }
}


But when I run this snippet through the sparkjava.com framework as a
web-service, the  "java.lang.ClassCastException: cannot assign
instance of java.lang.invoke.SerializedLambda" occurred!

package blah;
import static spark.Spark.get;
import javax.ws.rs.core.Response;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import spark.servlet.SparkApplication;

public class App implements SparkApplication {

   @Override
   public void init() {
      get("/hello", (req, res) -> {
         String sourcePath = "hdfs://spark:54310/input/*";

         SparkConf conf = new SparkConf().setAppName("TestLineCount");
         conf.setJars(new String[] { App.class.getProtectionDomain()
               .getCodeSource().getLocation().getPath() });
         conf.setMaster("spark://tootak:7077");
         conf.set("spark.driver.allowMultipleContexts", "true");

         @SuppressWarnings("resource")
         JavaSparkContext sc = new JavaSparkContext(conf);
         JavaRDD<String> log = sc.textFile(sourcePath);

         JavaRDD<String> lines = log.filter(x -> {
            return true;
         });

         return Response.ok(lines.count()).build();
      });
   }
}


but when I run this web-service api with tomcat, following error occurs:

[task-result-getter-0] WARN org.apache.spark.scheduler.TaskSetManager
- Lost task 0.0 in stage 0.0 (TID 0, tootak):
java.lang.ClassCastException: cannot assign instance of
java.lang.invoke.SerializedLambda to field
org.apache.spark.api.java.JavaRDD$$anonfun$filter$1.f$1 of type
org.apache.spark.api.java.function.Function in instance of
org.apache.spark.api.java.JavaRDD$$anonfun$filter$1
at java.io.ObjectStreamClass$FieldReflector.setObjFieldValues(ObjectStreamClass.java:2089)
at java.io.ObjectStreamClass.setObjFieldValues(ObjectStreamClass.java:1261)
at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1999)
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1918)
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351)
at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993)
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1918)
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351)
at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993)
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1918)
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351)
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:371)
at org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:62)
at org.apache.spark.serializer.JavaSerializerInstance.deserialize(JavaSerializer.scala:87)
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:57)
at org.apache.spark.scheduler.Task.run(Task.scala:56)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:196)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)


Here is all error log:
https://gist.github.com/khajavi/89f802398d6f8c40e23e#file-error-log

Any suggestion how to resolve this problem?

-- 
Milād Khājavi
http://blog.khajavi.ir
Having the source means you can do it yourself.
I tried to change the world, but I couldn’t find the source code.

---------------------------------------------------------------------
To unsubscribe, e-mail: user-unsubscribe@spark.apache.org
For additional commands, e-mail: user-help@spark.apache.org


Mime
View raw message