carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [carbondata] 16/41: [CARBONDATA-3300] Fixed ClassNotFoundException when using UDF in spark-shell
Date Tue, 02 Apr 2019 02:41:36 GMT
This is an automated email from the ASF dual-hosted git repository.

ravipesala pushed a commit to branch branch-1.5
in repository https://gitbox.apache.org/repos/asf/carbondata.git

commit bfdff7ff65a831c5aa5baa5ed9c7ead990bafa40
Author: kunal642 <kunalkapoor642@gmail.com>
AuthorDate: Fri Feb 22 12:03:42 2019 +0530

    [CARBONDATA-3300] Fixed ClassNotFoundException when using UDF in spark-shell
    
    Analysis:
    When a spark-shell is run a scala interpreter session is started which is the main thread
for that shell. This session uses TranslatingClassLoader, therefore the UDF(  in the stacktrace)
that is defined would be loaded into TranslatingClassLoader.
    
    When deserialization happens an ObjectInputStream is create and the application tries
to read the object, the ObjectInputStream uses a native method(sun.misc.VM.latestUserDefinedLoader()
) call to determine the ClassLoader that will be used to load the class. This native method
returns URLClassLoader which is the parent of TranslatingClassLoader where the class was loaded.
    Because of this ClassNotFoundException is thrown.
    
    Class Loader Hierarchy
    
    ExtClassLoader(head) -> AppClassLoader -> URLClassLoader -> TranslatingClassLoader
    
    This looks like a bug in the java ObjectInputStream implementation as suggested by the
following post
    https://stackoverflow.com/questions/1771679/difference-between-threads-context-class-loader-and-normal-classloader
    
    Operation	Thread	Thread ClassLoader	ClassLoader
    Register	Main	Translating	Translating
    Serialize	Main	Translating	Translating
    Deserialize	Thread-1	Translating	URLClassLoader
    Solution:
    Use ClassLoaderObjectInputStream to specify the class loader that should be used to load
the class.
    
    This closes #3132
---
 .../org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java    | 4 +++-
 core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java    | 4 +++-
 .../java/org/apache/carbondata/core/util/ObjectSerializationUtil.java | 3 ++-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
index 88706b1..104ef1a 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/blocklet/BlockletInfo.java
@@ -31,6 +31,7 @@ import java.util.List;
 import org.apache.carbondata.core.metadata.blocklet.datachunk.DataChunk;
 import org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex;
 
+import org.apache.commons.io.input.ClassLoaderObjectInputStream;
 import org.apache.hadoop.io.Writable;
 
 /**
@@ -261,7 +262,8 @@ public class BlockletInfo implements Serializable, Writable {
 
   private DataChunk deserializeDataChunk(byte[] bytes) throws IOException {
     ByteArrayInputStream stream = new ByteArrayInputStream(bytes);
-    ObjectInputStream inputStream = new ObjectInputStream(stream);
+    ObjectInputStream inputStream =
+        new ClassLoaderObjectInputStream(Thread.currentThread().getContextClassLoader(),
stream);
     DataChunk dataChunk = null;
     try {
       dataChunk = (DataChunk) inputStream.readObject();
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index ffab9c8..d9f69e3 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -100,6 +100,7 @@ import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.input.ClassLoaderObjectInputStream;
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang3.StringEscapeUtils;
@@ -1536,7 +1537,8 @@ public final class CarbonUtil {
     ValueEncoderMeta meta = null;
     try {
       aos = new ByteArrayInputStream(encoderMeta);
-      objStream = new ObjectInputStream(aos);
+      objStream =
+          new ClassLoaderObjectInputStream(Thread.currentThread().getContextClassLoader(),
aos);
       meta = (ValueEncoderMeta) objStream.readObject();
     } catch (ClassNotFoundException e) {
       LOGGER.error(e.getMessage(), e);
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ObjectSerializationUtil.java
b/core/src/main/java/org/apache/carbondata/core/util/ObjectSerializationUtil.java
index 48c6e65..169a3da 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ObjectSerializationUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ObjectSerializationUtil.java
@@ -26,6 +26,7 @@ import java.util.zip.GZIPOutputStream;
 
 import org.apache.carbondata.common.logging.LogServiceFactory;
 
+import org.apache.commons.io.input.ClassLoaderObjectInputStream;
 import org.apache.log4j.Logger;
 
 /**
@@ -94,7 +95,7 @@ public class ObjectSerializationUtil {
     try {
       bais = new ByteArrayInputStream(bytes);
       gis = new GZIPInputStream(bais);
-      ois = new ObjectInputStream(gis);
+      ois = new ClassLoaderObjectInputStream(Thread.currentThread().getContextClassLoader(),
gis);
       return ois.readObject();
     } catch (ClassNotFoundException e) {
       throw new IOException("Could not read object", e);


Mime
View raw message