more concretely, I was trying to make the query process a bit more fluent -some pseudocode but with correct types
val table:SparkTable[POJO] = new SparkTable[POJO](sqlContext,extractor:String=>POJO)
class SparkTable[T <: Product : ClassTag :TypeTag](val sqlContext:SQLContext, val extractor: (String) => (T) ) {
private[this] var location:Option[String] =None
private[this] var name:Option[String]=None
private[this] val sc = sqlContext.sparkContext
def withName(name:String):SparkTable[T]={..}
def atLocation(path:String):SparkTable[T]={.. }
def makeRDD(sqlQuery:String):SchemaRDD={
...
import sqlContext._
val rdd:RDD[String] = sc.textFile(this.location.get)
val rddT:RDD[T] = rdd.map(extractor)
val schemaRDD= createSchemaRDD(rddT)
schemaRDD.registerAsTable(name.get)
val all = sqlContext.sql(sqlQuery)
all
}
}