spark-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Taegeon Um <taegeo...@gmail.com>
Subject Spark 3.0 ArrayIndexOutOfBoundsException at RDDOperationScope.toJson
Date Tue, 30 Jun 2020 01:34:08 GMT
Hi, 

I’ve got the following exception when running a connected component example in Spark 3.0.0.
This code runs without exception in Spark 2.4. 

It throws the exception when calling RDDOperationScope.toJson method. 
I’m not sure why it throws the execution in Spark 3.0. There is no exception in Spark 2.4.


Do you have any idea? 
I’ve tested ConnectedComponentsExample.   

20/06/30 01:17:22 INFO rdd.RDDOperationScope: !!!!!!!!!!!1 with scope scope key: spark.rdd.scope,
name: textFile, oldScope: None, toJson                                                   
                                                                                         
    
Exception in thread "main" java.lang.ArrayIndexOutOfBoundsException: 28499               
                                                                                         
                                                                                         
            
        at com.thoughtworks.paranamer.BytecodeReadingParanamer$ClassReader.accept(BytecodeReadingParanamer.java:532)
                                                                                         
                                                                          
        at com.thoughtworks.paranamer.BytecodeReadingParanamer$ClassReader.access$200(BytecodeReadingParanamer.java:315)
                                                                                         
                                                                      
        at com.thoughtworks.paranamer.BytecodeReadingParanamer.lookupParameterNames(BytecodeReadingParanamer.java:102)
                                                                                         
                                                                        
        at com.thoughtworks.paranamer.CachingParanamer.lookupParameterNames(CachingParanamer.java:76)
                                                                                         
                                                                                         

        at com.fasterxml.jackson.module.scala.introspect.BeanIntrospector$.getCtorParams(BeanIntrospector.scala:45)
                                                                                         
                                                                           
        at com.fasterxml.jackson.module.scala.introspect.BeanIntrospector$.$anonfun$apply$1(BeanIntrospector.scala:59)
                                                                                         
                                                                        
        at com.fasterxml.jackson.module.scala.introspect.BeanIntrospector$.$anonfun$apply$1$adapted(BeanIntrospector.scala:59)
                                                                                         
                                                                
        at scala.collection.TraversableLike.$anonfun$flatMap$1(TraversableLike.scala:245)
                                                                                         
                                                                                         
            
        at scala.collection.Iterator.foreach(Iterator.scala:941)                         
                                                                                         
                                                                                         
            
        at scala.collection.Iterator.foreach$(Iterator.scala:941)                        
                                                                                         
                                                                                         
            
        at scala.collection.AbstractIterator.foreach(Iterator.scala:1429)                
                                                                                         
                                                                                         
            
        at scala.collection.IterableLike.foreach(IterableLike.scala:74)                  
                                                                                         
                                                                                         
            
        at scala.collection.IterableLike.foreach$(IterableLike.scala:73)                 
                                                                                         
                                                                                         
            
        at scala.collection.AbstractIterable.foreach(Iterable.scala:56)                  
                                                                                         
                                                                                         
            
        at scala.collection.TraversableLike.flatMap(TraversableLike.scala:245)           
                                                                                         
                                                                                         
            
        at scala.collection.TraversableLike.flatMap$(TraversableLike.scala:242)          
                                                                                         
                                                                                         
            
        at scala.collection.AbstractTraversable.flatMap(Traversable.scala:108)           
                                                                                         
                                                                                         
            
        at com.fasterxml.jackson.module.scala.introspect.BeanIntrospector$.findConstructorParam$1(BeanIntrospector.scala:59)
                                                                                         
                                                                  
        at com.fasterxml.jackson.module.scala.introspect.BeanIntrospector$.$anonfun$apply$19(BeanIntrospector.scala:181)
                                                                                         
                                                                      
        at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)    
                                                                                         
                                                                                         
            
        at scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)    
                                                                                         
                                                                                         
            
        at scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)   
                                                                                         
                                                                                         
            
        at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)           
                                                                                         
                                                                                         
            
        at scala.collection.TraversableLike.map(TraversableLike.scala:238)               
                                                                                         
                                                                                         
            
        at scala.collection.TraversableLike.map$(TraversableLike.scala:231)              
                                                                                         
                                                                                         
            
        at scala.collection.mutable.ArrayOps$ofRef.map(ArrayOps.scala:198)               
                                                                                         
                                                                                         
            
        at com.fasterxml.jackson.module.scala.introspect.BeanIntrospector$.$anonfun$apply$14(BeanIntrospector.scala:175)
                                                                                         
                                                                      
        at com.fasterxml.jackson.module.scala.introspect.BeanIntrospector$.$anonfun$apply$14$adapted(BeanIntrospector.scala:174)
                                                                                         
                                                              
        at scala.collection.TraversableLike.$anonfun$flatMap$1(TraversableLike.scala:245)
                                                                                         
                                                                                         
            
        at scala.collection.immutable.List.foreach(List.scala:392)                       
                                                                                         
                                                                                         
            
        at scala.collection.TraversableLike.flatMap(TraversableLike.scala:245)           
                                                                                         
                                                                                         
            
        at scala.collection.TraversableLike.flatMap$(TraversableLike.scala:242)          
                                                                                         
                                                                                         
            
        at scala.collection.immutable.List.flatMap(List.scala:355)                       
                                                                                         
                                                                                         
            
        at com.fasterxml.jackson.module.scala.introspect.BeanIntrospector$.apply(BeanIntrospector.scala:174)
                                                                                         
                                                                                   
        at com.fasterxml.jackson.module.scala.introspect.ScalaAnnotationIntrospector$._descriptorFor(ScalaAnnotationIntrospectorModule.scala:21)
                                                                                         
                                              
        at com.fasterxml.jackson.module.scala.introspect.ScalaAnnotationIntrospector$.fieldName(ScalaAnnotationIntrospectorModule.scala:29)
                                                                                         
                                                   
        at com.fasterxml.jackson.module.scala.introspect.ScalaAnnotationIntrospector$.findImplicitPropertyName(ScalaAnnotationIntrospectorModule.scala:77)
                                                                                         
                                    
        at com.fasterxml.jackson.databind.introspect.AnnotationIntrospectorPair.findImplicitPropertyName(AnnotationIntrospectorPair.java:490)
                                                                                         
                                                 
        at com.fasterxml.jackson.databind.introspect.POJOPropertiesCollector._addFields(POJOPropertiesCollector.java:380)
                                                                                         
                                                                     
        at com.fasterxml.jackson.databind.introspect.POJOPropertiesCollector.collectAll(POJOPropertiesCollector.java:308)
                                                                                         
                                                                     
        at com.fasterxml.jackson.databind.introspect.POJOPropertiesCollector.getJsonValueAccessor(POJOPropertiesCollector.java:196)
                                                                                         
                                                           
        at com.fasterxml.jackson.databind.introspect.BasicBeanDescription.findJsonValueAccessor(BasicBeanDescription.java:252)
                                                                                         
                                                                
        at com.fasterxml.jackson.databind.ser.BasicSerializerFactory.findSerializerByAnnotations(BasicSerializerFactory.java:346)
                                                                                         
                                                             
        at com.fasterxml.jackson.databind.ser.BeanSerializerFactory._createSerializer2(BeanSerializerFactory.java:216)
                                                                                         
                                                                        
        at com.fasterxml.jackson.databind.ser.BeanSerializerFactory.createSerializer(BeanSerializerFactory.java:165)
                                                                                         
                                                                          
        at com.fasterxml.jackson.databind.SerializerProvider._createUntypedSerializer(SerializerProvider.java:1388)
                                                                                         
                                                                           
        at com.fasterxml.jackson.databind.SerializerProvider._createAndCacheUntypedSerializer(SerializerProvider.java:1336)
        at com.fasterxml.jackson.databind.SerializerProvider.findValueSerializer(SerializerProvider.java:510)
                                                                                         
                                                                                  
        at com.fasterxml.jackson.databind.SerializerProvider.findTypedValueSerializer(SerializerProvider.java:713)
                                                                                         
                                                                            
        at com.fasterxml.jackson.databind.ser.DefaultSerializerProvider.serializeValue(DefaultSerializerProvider.java:308)
        at com.fasterxml.jackson.databind.ObjectMapper._configAndWriteValue(ObjectMapper.java:4094)
        at com.fasterxml.jackson.databind.ObjectMapper.writeValueAsString(ObjectMapper.java:3404)
        at org.apache.spark.rdd.RDDOperationScope.toJson(RDDOperationScope.scala:52)
        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:148)
        at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
        at org.apache.spark.SparkContext.withScope(SparkContext.scala:751)
        at org.apache.spark.SparkContext.textFile(SparkContext.scala:882)
        at org.apache.spark.graphx.GraphLoader$.edgeListFile(GraphLoader.scala:75)
        at org.apache.spark.examples.graphx.ConnectedComponentsExample$.main(ConnectedComponentsExample.scala:51)
        at org.apache.spark.examples.graphx.ConnectedComponentsExample.main(ConnectedComponentsExample.scala)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
        at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:928)
        at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:180)
        at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:203)
        at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:90)


Mime
View raw message