spark-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Nimmi Cv <nimm...@gmail.com>
Subject Spark Yarn Java Out Of Memory on Complex Query Execution Plan
Date Tue, 12 Sep 2017 06:43:50 GMT
Exception in thread "main" java.lang.OutOfMemoryError at
java.lang.AbstractStringBuilder.hugeCapacity(AbstractStringBuilder.java:161)
at
java.lang.AbstractStringBuilder.newCapacity(AbstractStringBuilder.java:155)
at
java.lang.AbstractStringBuilder.ensureCapacityInternal(AbstractStringBuilder.java:125)
at java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:448)
at
java.lang.StringBuilder.append(StringBuilder.java:136) at
java.lang.StringBuilder.append(StringBuilder.java:131) at
scala.StringContext.standardInterpolator(StringContext.scala:125) at
scala.StringContext.s(StringContext.scala:95) at
org.apache.spark.sql.execution.QueryExecution.toString(QueryExecution.scala:230)
at
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:54)
at org.apache.spark.sql.Dataset.withNewExecutionId(Dataset.scala:2788) at
org.apache.spark.sql.Dataset.org
$apache$spark$sql$Dataset$$execute$1(Dataset.scala:2385)
at
org.apache.spark.sql.Dataset.org
$apache$spark$sql$Dataset$$collect(Dataset.scala:2392)
at org.apache.spark.sql.Dataset$$anonfun$count$1.apply(Dataset.scala:2420)
at org.apache.spark.sql.Dataset$$anonfun$count$1.apply(Dataset.scala:2419)
at org.apache.spark.sql.Dataset.withCallback(Dataset.scala:2801) at
org.apache.spark.sql.Dataset.count(Dataset.scala:2419) at
com.samsung.cloud.mopay.linking.controller.PostNotificLinkController.linkPostNotific(PostNotificLinkController.java:51)
at
com.samsung.cloud.mopay.linking.txn.TxnLinking.performTxnLinking(TxnLinking.java:26)
at com.samsung.cloud.mopay.linking.Linking.processData(Linking.java:199) at
com.samsung.cloud.mopay.linking.Linking.main(Linking.java:72) at
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498) at
org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:743)
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:187)
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:212) at
org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:126) at
org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)


I tried increasing
spark.driver.memory = 50g
spark.yarn.driver.memoryOverhead = 80g

Any help appreciated. I am stuck here for a while.

Thanks,
Nimmi

Mime
View raw message