In the following example, after I used "typed.avg" to generate a TypedColumn, and I want to apply round on top of it? But why Spark complains about it? Because it doesn't know that it is a TypedColumn<Token, Double>?


Thanks


Yong



scala> spark.version

res20: String = 2.1.0

scala> case class Token(name: String, productId: Int, score: Double)

defined class Token

scala> val data = Token("aaa", 100, 0.12) :: Token("aaa", 200, 0.29) :: Token("bbb", 200, 0.53) :: Token("bbb", 300, 0.42) :: Nil

data: List[Token] = List(Token(aaa,100,0.12), Token(aaa,200,0.29), Token(bbb,200,0.53), Token(bbb,300,0.42))

scala> val dataset = data.toDS

dataset: org.apache.spark.sql.Dataset[Token] = [name: string, productId: int ... 1 more field]

scala> import org.apache.spark.sql.expressions.scalalang._

import org.apache.spark.sql.expressions.scalalang._

scala> dataset.groupByKey(_.productId).agg(typed.avg[Token](_.score)).show

+-----+-----------------------------------------+

|value|TypedAverage($line22.$readiwiw$Token)|

+-----+-----------------------------------------+

| 300| 0.42|

| 100| 0.12|

| 200| 0.41000000000000003|

+-----+-----------------------------------------+

scala> dataset.groupByKey(_.productId).agg(round(typed.avg[Token](_.score)))

<console>:36: error: type mismatch;

found : org.apache.spark.sql.Column

required: org.apache.spark.sql.TypedColumn[Token,?]

dataset.groupByKey(_.productId).agg(round(typed.avg[Token](_.score)))