spark-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Nads <nadersale...@gmail.com>
Subject Re: Typed datataset from Avro generated classes?
Date Sat, 05 Dec 2020 01:58:22 GMT
Same problem here.  A google search shows a few related jira tickets in
"Resolved" state but I am getting the same error in Spark 3.0.1.  I'm
pasting my `spark-shell` output below:

scala> import org.apache.spark.sql.Encoders
import org.apache.spark.sql.Encoders

scala> val linkageBean = Encoders.bean(classOf[MyAvroGeneratedClass])
java.lang.UnsupportedOperationException: Cannot have circular references in
bean class, but got the circular reference of class class
org.apache.avro.Schema
  at
org.apache.spark.sql.catalyst.JavaTypeInference$.inferDataType(JavaTypeInference.scala:142)
  at
org.apache.spark.sql.catalyst.JavaTypeInference$.$anonfun$inferDataType$1(JavaTypeInference.scala:150)
  at
scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
  at
scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
  at
scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
  at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)
  at scala.collection.TraversableLike.map(TraversableLike.scala:238)
  at scala.collection.TraversableLike.map$(TraversableLike.scala:231)
  at scala.collection.mutable.ArrayOps$ofRef.map(ArrayOps.scala:198)
  at
org.apache.spark.sql.catalyst.JavaTypeInference$.inferDataType(JavaTypeInference.scala:148)
  at
org.apache.spark.sql.catalyst.JavaTypeInference$.$anonfun$inferDataType$1(JavaTypeInference.scala:150)
  at
scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
  at
scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
  at
scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
  at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)
  at scala.collection.TraversableLike.map(TraversableLike.scala:238)
  at scala.collection.TraversableLike.map$(TraversableLike.scala:231)
  at scala.collection.mutable.ArrayOps$ofRef.map(ArrayOps.scala:198)
  at
org.apache.spark.sql.catalyst.JavaTypeInference$.inferDataType(JavaTypeInference.scala:148)
  at
org.apache.spark.sql.catalyst.JavaTypeInference$.inferDataType(JavaTypeInference.scala:126)
  at
org.apache.spark.sql.catalyst.JavaTypeInference$.$anonfun$inferDataType$1(JavaTypeInference.scala:150)
  at
scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:238)
  at
scala.collection.IndexedSeqOptimized.foreach(IndexedSeqOptimized.scala:36)
  at
scala.collection.IndexedSeqOptimized.foreach$(IndexedSeqOptimized.scala:33)
  at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:198)
  at scala.collection.TraversableLike.map(TraversableLike.scala:238)
  at scala.collection.TraversableLike.map$(TraversableLike.scala:231)
  at scala.collection.mutable.ArrayOps$ofRef.map(ArrayOps.scala:198)
  at
org.apache.spark.sql.catalyst.JavaTypeInference$.inferDataType(JavaTypeInference.scala:148)
  at
org.apache.spark.sql.catalyst.JavaTypeInference$.inferDataType(JavaTypeInference.scala:67)
  at
org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$.javaBean(ExpressionEncoder.scala:68)
  at org.apache.spark.sql.Encoders$.bean(Encoders.scala:154)
  ... 49 elided




--
Sent from: http://apache-spark-user-list.1001560.n3.nabble.com/

---------------------------------------------------------------------
To unsubscribe e-mail: user-unsubscribe@spark.apache.org


Mime
View raw message