spark-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From matthes <mdiekst...@sensenetworks.com>
Subject Re: Setup an huge Unserializable Object in a mapper
Date Tue, 23 Sep 2014 18:50:39 GMT
I solved it :) I moved the lookupObject into the function where I create the
broadcast and now all works very well!

object lookupObject 
{ 
private var treeFile : org.apache.spark.broadcast.Broadcast[String] = _ 

def main(args: Array[String]): Unit = { 
… 
val treeFile = sc.broadcast(args(0)) 

object treeContainer 
{ 
          val tree : S2Lookup = loadTree 
          
          def dolookup(id : Long) : Boolean = 
          { 
            return tree.lookupSimple(new S2CellId(id)) 
          } 
          def loadTree() : S2Lookup = 
          { 
            val path = new Path(treeFile.value); // treeFile is everytime
null 
            val fileSystem = FileSystem.get(new Configuration()) 
            new S2Lookup(ConversionUtils.deserializeCovering(new
InputStreamReader(fileSystem.open(path)))) 
          } 
}

… 
} 

} 



--
View this message in context: http://apache-spark-user-list.1001560.n3.nabble.com/Setup-an-huge-Unserializable-Object-in-a-mapper-tp14817p14916.html
Sent from the Apache Spark User List mailing list archive at Nabble.com.

---------------------------------------------------------------------
To unsubscribe, e-mail: user-unsubscribe@spark.apache.org
For additional commands, e-mail: user-help@spark.apache.org


Mime
View raw message