spark-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Vikas Garg <sperry...@gmail.com>
Subject Issue with executer
Date Wed, 20 Jan 2021 12:04:11 GMT
Hi,

I am facing issue with spark executor. I am struggling with this issue
since last many days and unable to resolve the issue.

Below is the configuration I have given.

  val spark = SparkSession.builder()
    .appName("Spark Job")
    .master("local[*]")
    .config("spark.dynamicAllocation.enabled", true)
    .config("spark.shuffle.service.enabled", true)
    .config("spark.driver.maxResultSize", "8g")
    .config("spark.driver.memory", "8g")
    .config("spark.executor.memory", "8g")
    .config("spark.network.timeout", "3600s")
    .getOrCreate()

1/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning
fetch of 1 outstanding blocks

*java.io.IOException*: Failed to connect to
del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:253*)

            at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:195*)

            at
org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
*NettyBlockTransferService.scala:122*)

            at
org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
*RetryingBlockFetcher.java:141*)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(
*RetryingBlockFetcher.java:121*)

            at
org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(
*NettyBlockTransferService.scala:143*)

            at org.apache.spark.network.BlockTransferService.fetchBlockSync(
*BlockTransferService.scala:103*)

            at
org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(
*BlockManager.scala:1010*)

            at
org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(
*BlockManager.scala:954*)

            at scala.Option.orElse(*Option.scala:289*)

            at org.apache.spark.storage.BlockManager.getRemoteBlock(
*BlockManager.scala:954*)

            at org.apache.spark.storage.BlockManager.getRemoteBytes(
*BlockManager.scala:1092*)

            at
org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(
*TaskResultGetter.scala:88*)

            at
scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)

            at org.apache.spark.util.Utils$.logUncaughtExceptions(
*Utils.scala:1932*)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(
*TaskResultGetter.scala:63*)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(
*ThreadPoolExecutor.java:1149*)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(
*ThreadPoolExecutor.java:624*)

            at java.lang.Thread.run(*Thread.java:748*)

Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
Permission denied: no further information:
del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: *java.net.SocketException*: Permission denied: no further
information

            at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)

            at sun.nio.ch.SocketChannelImpl.finishConnect(
*SocketChannelImpl.java:715*)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
*NioSocketChannel.java:330*)

            at
io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
*AbstractNioChannel.java:334*)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(
*NioEventLoop.java:702*)

            at
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
*NioEventLoop.java:650*)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
*NioEventLoop.java:576*)

            at io.netty.channel.nio.NioEventLoop.run(*NioEventLoop.java:493*
)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
*SingleThreadEventExecutor.java:989*)

            at io.netty.util.internal.ThreadExecutorMap$2.run(
*ThreadExecutorMap.java:74*)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)

            at java.lang.Thread.run(*Thread.java:748*)

21/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning
fetch of 1 outstanding blocks

*java.io.IOException*: Failed to connect to
del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:253*)

            at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:195*)

            at
org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
*NettyBlockTransferService.scala:122*)

            at
org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
*RetryingBlockFetcher.java:141*)

            at org.apache.spark.network.shuffle.RetryingBlockFetcher.start(
*RetryingBlockFetcher.java:121*)

            at
org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(
*NettyBlockTransferService.scala:143*)

            at org.apache.spark.network.BlockTransferService.fetchBlockSync(
*BlockTransferService.scala:103*)

            at
org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(
*BlockManager.scala:1010*)

            at
org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(
*BlockManager.scala:954*)

            at scala.Option.orElse(*Option.scala:289*)

            at org.apache.spark.storage.BlockManager.getRemoteBlock(
*BlockManager.scala:954*)

            at org.apache.spark.storage.BlockManager.getRemoteBytes(
*BlockManager.scala:1092*)

            at
org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(
*TaskResultGetter.scala:88*)

            at
scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)

            at org.apache.spark.util.Utils$.logUncaughtExceptions(
*Utils.scala:1932*)

            at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(
*TaskResultGetter.scala:63*)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(
*ThreadPoolExecutor.java:1149*)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(
*ThreadPoolExecutor.java:624*)

            at java.lang.Thread.run(*Thread.java:748*)

Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
Permission denied: no further information:
del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: *java.net.SocketException*: Permission denied: no further
information

            at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)

            at sun.nio.ch.SocketChannelImpl.finishConnect(
*SocketChannelImpl.java:715*)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
*NioSocketChannel.java:330*)

            at
io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
*AbstractNioChannel.java:334*)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(
*NioEventLoop.java:702*)

            at
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
*NioEventLoop.java:650*)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
*NioEventLoop.java:576*)

            at io.netty.channel.nio.NioEventLoop.run(*NioEventLoop.java:493*
)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
*SingleThreadEventExecutor.java:989*)

            at io.netty.util.internal.ThreadExecutorMap$2.run(
*ThreadExecutorMap.java:74*)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)

            at java.lang.Thread.run(*Thread.java:748*)

21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning
fetch of 1 outstanding blocks (after 1 retries)

*java.io.IOException*: Failed to connect to
del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:253*)

            at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:195*)

            at
org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
*NettyBlockTransferService.scala:122*)

            at
org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
*RetryingBlockFetcher.java:141*)

            at
org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
*RetryingBlockFetcher.java:169*)

            at java.util.concurrent.Executors$RunnableAdapter.call(
*Executors.java:511*)

            at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(
*ThreadPoolExecutor.java:1149*)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(
*ThreadPoolExecutor.java:624*)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)

            at java.lang.Thread.run(*Thread.java:748*)

Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
Permission denied: no further information:
del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: *java.net.SocketException*: Permission denied: no further
information

            at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)

            at sun.nio.ch.SocketChannelImpl.finishConnect(
*SocketChannelImpl.java:715*)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
*NioSocketChannel.java:330*)

            at
io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
*AbstractNioChannel.java:334*)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(
*NioEventLoop.java:702*)

            at
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
*NioEventLoop.java:650*)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
*NioEventLoop.java:576*)

            at io.netty.channel.nio.NioEventLoop.run(*NioEventLoop.java:493*
)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
*SingleThreadEventExecutor.java:989*)

            at io.netty.util.internal.ThreadExecutorMap$2.run(
*ThreadExecutorMap.java:74*)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)

            at java.lang.Thread.run(*Thread.java:748*)

21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning
fetch of 1 outstanding blocks (after 1 retries)

*java.io.IOException*: Failed to connect to
del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:253*)

            at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:195*)

            at
org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
*NettyBlockTransferService.scala:122*)

            at
org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
*RetryingBlockFetcher.java:141*)

            at
org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
*RetryingBlockFetcher.java:169*)

            at java.util.concurrent.Executors$RunnableAdapter.call(
*Executors.java:511*)

            at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(
*ThreadPoolExecutor.java:1149*)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(
*ThreadPoolExecutor.java:624*)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)

            at java.lang.Thread.run(*Thread.java:748*)

Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
Permission denied: no further information:
del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: *java.net.SocketException*: Permission denied: no further
information

            at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)

            at sun.nio.ch.SocketChannelImpl.finishConnect(
*SocketChannelImpl.java:715*)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
*NioSocketChannel.java:330*)

            at
io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
*AbstractNioChannel.java:334*)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(
*NioEventLoop.java:702*)

            at
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
*NioEventLoop.java:650*)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
*NioEventLoop.java:576*)

            at io.netty.channel.nio.NioEventLoop.run(*NioEventLoop.java:493*
)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
*SingleThreadEventExecutor.java:989*)

            at io.netty.util.internal.ThreadExecutorMap$2.run(
*ThreadExecutorMap.java:74*)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)

            at java.lang.Thread.run(*Thread.java:748*)

21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning
fetch of 1 outstanding blocks (after 2 retries)

*java.io.IOException*: Failed to connect to
del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:253*)

            at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:195*)

            at
org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
*NettyBlockTransferService.scala:122*)

            at
org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
*RetryingBlockFetcher.java:141*)

            at
org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
*RetryingBlockFetcher.java:169*)

            at java.util.concurrent.Executors$RunnableAdapter.call(
*Executors.java:511*)

            at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(
*ThreadPoolExecutor.java:1149*)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(
*ThreadPoolExecutor.java:624*)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)

            at java.lang.Thread.run(*Thread.java:748*)

Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
Permission denied: no further information:
del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: *java.net.SocketException*: Permission denied: no further
information

            at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)

            at sun.nio.ch.SocketChannelImpl.finishConnect(
*SocketChannelImpl.java:715*)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
*NioSocketChannel.java:330*)

            at
io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
*AbstractNioChannel.java:334*)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(
*NioEventLoop.java:702*)

            at
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
*NioEventLoop.java:650*)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
*NioEventLoop.java:576*)

            at io.netty.channel.nio.NioEventLoop.run(*NioEventLoop.java:493*
)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
*SingleThreadEventExecutor.java:989*)

            at io.netty.util.internal.ThreadExecutorMap$2.run(
*ThreadExecutorMap.java:74*)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)

            at java.lang.Thread.run(*Thread.java:748*)

21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning
fetch of 1 outstanding blocks (after 2 retries)

*java.io.IOException*: Failed to connect to
del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:253*)

            at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:195*)

            at
org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
*NettyBlockTransferService.scala:122*)

            at
org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
*RetryingBlockFetcher.java:141*)

            at
org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
*RetryingBlockFetcher.java:169*)

            at java.util.concurrent.Executors$RunnableAdapter.call(
*Executors.java:511*)

            at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(
*ThreadPoolExecutor.java:1149*)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(
*ThreadPoolExecutor.java:624*)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)

            at java.lang.Thread.run(*Thread.java:748*)

Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
Permission denied: no further information:
del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: *java.net.SocketException*: Permission denied: no further
information

            at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)

            at sun.nio.ch.SocketChannelImpl.finishConnect(
*SocketChannelImpl.java:715*)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
*NioSocketChannel.java:330*)

            at
io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
*AbstractNioChannel.java:334*)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(
*NioEventLoop.java:702*)

            at
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
*NioEventLoop.java:650*)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
*NioEventLoop.java:576*)

            at io.netty.channel.nio.NioEventLoop.run(*NioEventLoop.java:493*
)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
*SingleThreadEventExecutor.java:989*)

            at io.netty.util.internal.ThreadExecutorMap$2.run(
*ThreadExecutorMap.java:74*)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)

            at java.lang.Thread.run(*Thread.java:748*)

21/01/20 17:07:12 ERROR RetryingBlockFetcher: Exception while beginning
fetch of 1 outstanding blocks (after 3 retries)

*java.io.IOException*: Failed to connect to
del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:253*)

            at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:195*)

            at
org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
*NettyBlockTransferService.scala:122*)

            at
org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
*RetryingBlockFetcher.java:141*)

            at
org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
*RetryingBlockFetcher.java:169*)

            at java.util.concurrent.Executors$RunnableAdapter.call(
*Executors.java:511*)

            at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(
*ThreadPoolExecutor.java:1149*)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(
*ThreadPoolExecutor.java:624*)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)

            at java.lang.Thread.run(*Thread.java:748*)

Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
Permission denied: no further information:
del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: *java.net.SocketException*: Permission denied: no further
information

            at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)

            at sun.nio.ch.SocketChannelImpl.finishConnect(
*SocketChannelImpl.java:715*)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
*NioSocketChannel.java:330*)

            at
io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
*AbstractNioChannel.java:334*)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(
*NioEventLoop.java:702*)

            at
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
*NioEventLoop.java:650*)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
*NioEventLoop.java:576*)

            at io.netty.channel.nio.NioEventLoop.run(*NioEventLoop.java:493*
)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
*SingleThreadEventExecutor.java:989*)

            at io.netty.util.internal.ThreadExecutorMap$2.run(
*ThreadExecutorMap.java:74*)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)

            at java.lang.Thread.run(*Thread.java:748*)

21/01/20 17:07:12 ERROR RetryingBlockFetcher: Exception while beginning
fetch of 1 outstanding blocks (after 3 retries)

*java.io.IOException*: Failed to connect to
del1-lhp-n99999.synapse.com/192.168.166.213:51348

            at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:253*)

            at
org.apache.spark.network.client.TransportClientFactory.createClient(
*TransportClientFactory.java:195*)

            at
org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
*NettyBlockTransferService.scala:122*)

            at
org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
*RetryingBlockFetcher.java:141*)

            at
org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
*RetryingBlockFetcher.java:169*)

            at java.util.concurrent.Executors$RunnableAdapter.call(
*Executors.java:511*)

            at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(
*ThreadPoolExecutor.java:1149*)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(
*ThreadPoolExecutor.java:624*)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)

            at java.lang.Thread.run(*Thread.java:748*)

Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
Permission denied: no further information:
del1-lhp-n99999.synapse.com/192.168.166.213:51348

Caused by: *java.net.SocketException*: Permission denied: no further
information

            at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)

            at sun.nio.ch.SocketChannelImpl.finishConnect(
*SocketChannelImpl.java:715*)

            at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
*NioSocketChannel.java:330*)

            at
io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
*AbstractNioChannel.java:334*)

            at io.netty.channel.nio.NioEventLoop.processSelectedKey(
*NioEventLoop.java:702*)

            at
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
*NioEventLoop.java:650*)

            at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
*NioEventLoop.java:576*)

            at io.netty.channel.nio.NioEventLoop.run(*NioEventLoop.java:493*
)

            at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
*SingleThreadEventExecutor.java:989*)

            at io.netty.util.internal.ThreadExecutorMap$2.run(
*ThreadExecutorMap.java:74*)

            at io.netty.util.concurrent.FastThreadLocalRunnable.run(
*FastThreadLocalRunnable.java:30*)

            at java.lang.Thread.run(*Thread.java:748*)

21/01/20 17:07:12 ERROR TaskSetManager: Task 0 in stage 23.0 failed 1
times; aborting job

21/01/20 17:07:12 ERROR TaskSetManager: Task 2 in stage 17.0 failed 1
times; aborting job

Exception in thread "main"
*org.apache.spark.sql.catalyst.errors.package$TreeNodeException*: execute,
tree:

Exchange hashpartitioning(claimid#1012, 200), true, [id=#742]





            at org.apache.spark.sql.catalyst.errors.package$.attachTree(
*package.scala:56*)

            at
org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.doExecute(
*ShuffleExchangeExec.scala:151*)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(
*SparkPlan.scala:175*)

            at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)

            at org.apache.spark.sql.execution.SparkPlan.execute(
*SparkPlan.scala:171*)

            at org.apache.spark.sql.execution.InputAdapter.inputRDD(
*WholeStageCodegenExec.scala:525*)

            at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs(
*WholeStageCodegenExec.scala:453*)

            at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs$(
*WholeStageCodegenExec.scala:452*)

            at org.apache.spark.sql.execution.InputAdapter.inputRDDs(
*WholeStageCodegenExec.scala:496*)

            at org.apache.spark.sql.execution.SortExec.inputRDDs(
*SortExec.scala:132*)

            at
org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(
*WholeStageCodegenExec.scala:720*)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(
*SparkPlan.scala:175*)

            at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)

            at org.apache.spark.sql.execution.SparkPlan.execute(
*SparkPlan.scala:171*)

            at
org.apache.spark.sql.execution.joins.SortMergeJoinExec.doExecute(
*SortMergeJoinExec.scala:182*)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(
*SparkPlan.scala:175*)

            at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)

            at org.apache.spark.sql.execution.SparkPlan.execute(
*SparkPlan.scala:171*)

            at org.apache.spark.sql.execution.ProjectExec.doExecute(
*basicPhysicalOperators.scala:76*)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(
*SparkPlan.scala:175*)

            at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)

            at org.apache.spark.sql.execution.SparkPlan.execute(
*SparkPlan.scala:171*)

            at org.apache.spark.sql.execution.SparkPlan.getByteArrayRdd(
*SparkPlan.scala:316*)

            at org.apache.spark.sql.execution.SparkPlan.executeTake(
*SparkPlan.scala:434*)

            at org.apache.spark.sql.execution.SparkPlan.executeTake(
*SparkPlan.scala:420*)

            at
org.apache.spark.sql.execution.CollectLimitExec.executeCollect(
*limit.scala:47*)

            at org.apache.spark.sql.Dataset.collectFromPlan(
*Dataset.scala:3627*)

            at org.apache.spark.sql.Dataset.$anonfun$head$1(
*Dataset.scala:2697*)

            at org.apache.spark.sql.Dataset.$anonfun$withAction$1(
*Dataset.scala:3618*)

            at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(
*SQLExecution.scala:100*)

            at
org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(
*SQLExecution.scala:160*)

            at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(
*SQLExecution.scala:87*)

            at org.apache.spark.sql.SparkSession.withActive(
*SparkSession.scala:764*)

            at
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(
*SQLExecution.scala:64*)

            at org.apache.spark.sql.Dataset.withAction(*Dataset.scala:3616*)

            at org.apache.spark.sql.Dataset.head(*Dataset.scala:2697*)

            at org.apache.spark.sql.Dataset.take(*Dataset.scala:2904*)

            at org.apache.spark.sql.Dataset.getRows(*Dataset.scala:300*)

            at org.apache.spark.sql.Dataset.showString(*Dataset.scala:337*)

            at org.apache.spark.sql.Dataset.show(*Dataset.scala:824*)

            at org.apache.spark.sql.Dataset.show(*Dataset.scala:783*)

            at org.apache.spark.sql.Dataset.show(*Dataset.scala:792*)

            at com.brighthealthplan.cognizant.output.MedicalClaim$.getDS(
*MedicalClaim.scala:257*)

            at
com.brighthealthplan.cognizant.sparkjobs.DataManager.writeLeft(
*DataManager.scala:192*)

            at com.brighthealthplan.cognizant.sparkjobs.Main$.main(
*Main.scala:107*)

            at
com.brighthealthplan.cognizant.sparkjobs.Main.main(Main.scala)

Caused by: *java.util.concurrent.ExecutionException*:
*org.apache.spark.SparkException*: Job aborted due to stage failure: Task 0
in stage 23.0 failed 1 times, most recent failure: Lost task 0.0 in stage
23.0 (TID 29, del1-lhp-n99999.synapse.com, executor driver): TaskResultLost
(result lost from block manager)

Driver stacktrace:

            at java.util.concurrent.FutureTask.report(*FutureTask.java:122*)

            at java.util.concurrent.FutureTask.get(*FutureTask.java:206*)

            at
org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.doExecuteBroadcast(
*BroadcastExchangeExec.scala:195*)

            at
org.apache.spark.sql.execution.InputAdapter.doExecuteBroadcast(
*WholeStageCodegenExec.scala:515*)

            at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeBroadcast$1(
*SparkPlan.scala:188*)

            at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)

            at org.apache.spark.sql.execution.SparkPlan.executeBroadcast(
*SparkPlan.scala:184*)

            at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.prepareBroadcast(
*BroadcastHashJoinExec.scala:116*)

            at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenOuter(
*BroadcastHashJoinExec.scala:258*)

            at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(
*BroadcastHashJoinExec.scala:101*)

            at org.apache.spark.sql.execution.CodegenSupport.consume(
*WholeStageCodegenExec.scala:194*)

            at org.apache.spark.sql.execution.CodegenSupport.consume$(
*WholeStageCodegenExec.scala:149*)

            at org.apache.spark.sql.execution.ProjectExec.consume(
*basicPhysicalOperators.scala:41*)

            at org.apache.spark.sql.execution.ProjectExec.doConsume(
*basicPhysicalOperators.scala:71*)

            at org.apache.spark.sql.execution.CodegenSupport.consume(
*WholeStageCodegenExec.scala:194*)

            at org.apache.spark.sql.execution.CodegenSupport.consume$(
*WholeStageCodegenExec.scala:149*)

            at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.consume(
*BroadcastHashJoinExec.scala:39*)

            at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenOuter(
*BroadcastHashJoinExec.scala:302*)

            at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(
*BroadcastHashJoinExec.scala:101*)

            at org.apache.spark.sql.execution.CodegenSupport.consume(
*WholeStageCodegenExec.scala:194*)

            at org.apache.spark.sql.execution.CodegenSupport.consume$(
*WholeStageCodegenExec.scala:149*)

            at org.apache.spark.sql.execution.ProjectExec.consume(
*basicPhysicalOperators.scala:41*)

            at org.apache.spark.sql.execution.ProjectExec.doConsume(
*basicPhysicalOperators.scala:71*)

            at org.apache.spark.sql.execution.CodegenSupport.consume(
*WholeStageCodegenExec.scala:194*)

            at org.apache.spark.sql.execution.CodegenSupport.consume$(
*WholeStageCodegenExec.scala:149*)

            at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.consume(
*BroadcastHashJoinExec.scala:39*)

            at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenOuter(
*BroadcastHashJoinExec.scala:302*)

            at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(
*BroadcastHashJoinExec.scala:101*)

            at org.apache.spark.sql.execution.CodegenSupport.consume(
*WholeStageCodegenExec.scala:194*)

            at org.apache.spark.sql.execution.CodegenSupport.consume$(
*WholeStageCodegenExec.scala:149*)

            at org.apache.spark.sql.execution.ProjectExec.consume(
*basicPhysicalOperators.scala:41*)

            at org.apache.spark.sql.execution.ProjectExec.doConsume(
*basicPhysicalOperators.scala:71*)

            at org.apache.spark.sql.execution.CodegenSupport.consume(
*WholeStageCodegenExec.scala:194*)

            at org.apache.spark.sql.execution.CodegenSupport.consume$(
*WholeStageCodegenExec.scala:149*)

            at org.apache.spark.sql.execution.InputAdapter.consume(
*WholeStageCodegenExec.scala:496*)

            at org.apache.spark.sql.execution.InputRDDCodegen.doProduce(
*WholeStageCodegenExec.scala:483*)

            at org.apache.spark.sql.execution.InputRDDCodegen.doProduce$(
*WholeStageCodegenExec.scala:456*)

            at org.apache.spark.sql.execution.InputAdapter.doProduce(
*WholeStageCodegenExec.scala:496*)

            at
org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
*WholeStageCodegenExec.scala:95*)

            at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)

            at org.apache.spark.sql.execution.CodegenSupport.produce(
*WholeStageCodegenExec.scala:90*)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(
*WholeStageCodegenExec.scala:90*)

            at org.apache.spark.sql.execution.InputAdapter.produce(
*WholeStageCodegenExec.scala:496*)

            at org.apache.spark.sql.execution.ProjectExec.doProduce(
*basicPhysicalOperators.scala:51*)

            at
org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
*WholeStageCodegenExec.scala:95*)

            at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)

            at org.apache.spark.sql.execution.CodegenSupport.produce(
*WholeStageCodegenExec.scala:90*)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(
*WholeStageCodegenExec.scala:90*)

            at org.apache.spark.sql.execution.ProjectExec.produce(
*basicPhysicalOperators.scala:41*)

            at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(
*BroadcastHashJoinExec.scala:95*)

            at
org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
*WholeStageCodegenExec.scala:95*)

            at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)

            at org.apache.spark.sql.execution.CodegenSupport.produce(
*WholeStageCodegenExec.scala:90*)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(
*WholeStageCodegenExec.scala:90*)

            at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(
*BroadcastHashJoinExec.scala:39*)

            at org.apache.spark.sql.execution.ProjectExec.doProduce(
*basicPhysicalOperators.scala:51*)

            at
org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
*WholeStageCodegenExec.scala:95*)

            at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)

            at org.apache.spark.sql.execution.CodegenSupport.produce(
*WholeStageCodegenExec.scala:90*)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(
*WholeStageCodegenExec.scala:90*)

            at org.apache.spark.sql.execution.ProjectExec.produce(
*basicPhysicalOperators.scala:41*)

            at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(
*BroadcastHashJoinExec.scala:95*)

            at
org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
*WholeStageCodegenExec.scala:95*)

            at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)

            at org.apache.spark.sql.execution.CodegenSupport.produce(
*WholeStageCodegenExec.scala:90*)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(
*WholeStageCodegenExec.scala:90*)

            at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(
*BroadcastHashJoinExec.scala:39*)

            at org.apache.spark.sql.execution.ProjectExec.doProduce(
*basicPhysicalOperators.scala:51*)

            at
org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
*WholeStageCodegenExec.scala:95*)

            at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)

            at org.apache.spark.sql.execution.CodegenSupport.produce(
*WholeStageCodegenExec.scala:90*)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(
*WholeStageCodegenExec.scala:90*)

            at org.apache.spark.sql.execution.ProjectExec.produce(
*basicPhysicalOperators.scala:41*)

            at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(
*BroadcastHashJoinExec.scala:95*)

            at
org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
*WholeStageCodegenExec.scala:95*)

            at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)

            at org.apache.spark.sql.execution.CodegenSupport.produce(
*WholeStageCodegenExec.scala:90*)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(
*WholeStageCodegenExec.scala:90*)

            at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(
*BroadcastHashJoinExec.scala:39*)

            at org.apache.spark.sql.execution.ProjectExec.doProduce(
*basicPhysicalOperators.scala:51*)

            at
org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
*WholeStageCodegenExec.scala:95*)

            at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)

            at org.apache.spark.sql.execution.CodegenSupport.produce(
*WholeStageCodegenExec.scala:90*)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(
*WholeStageCodegenExec.scala:90*)

            at org.apache.spark.sql.execution.ProjectExec.produce(
*basicPhysicalOperators.scala:41*)

            at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(
*BroadcastHashJoinExec.scala:95*)

            at
org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
*WholeStageCodegenExec.scala:95*)

            at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)

            at org.apache.spark.sql.execution.CodegenSupport.produce(
*WholeStageCodegenExec.scala:90*)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(
*WholeStageCodegenExec.scala:90*)

            at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(
*BroadcastHashJoinExec.scala:39*)

            at org.apache.spark.sql.execution.ProjectExec.doProduce(
*basicPhysicalOperators.scala:51*)

            at
org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
*WholeStageCodegenExec.scala:95*)

            at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)

            at org.apache.spark.sql.execution.CodegenSupport.produce(
*WholeStageCodegenExec.scala:90*)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(
*WholeStageCodegenExec.scala:90*)

            at org.apache.spark.sql.execution.ProjectExec.produce(
*basicPhysicalOperators.scala:41*)

            at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(
*BroadcastHashJoinExec.scala:95*)

            at
org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
*WholeStageCodegenExec.scala:95*)

            at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)

            at org.apache.spark.sql.execution.CodegenSupport.produce(
*WholeStageCodegenExec.scala:90*)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(
*WholeStageCodegenExec.scala:90*)

            at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(
*BroadcastHashJoinExec.scala:39*)

            at org.apache.spark.sql.execution.ProjectExec.doProduce(
*basicPhysicalOperators.scala:51*)

            at
org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
*WholeStageCodegenExec.scala:95*)

            at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)

            at org.apache.spark.sql.execution.CodegenSupport.produce(
*WholeStageCodegenExec.scala:90*)

            at org.apache.spark.sql.execution.CodegenSupport.produce$(
*WholeStageCodegenExec.scala:90*)

            at org.apache.spark.sql.execution.ProjectExec.produce(
*basicPhysicalOperators.scala:41*)

            at
org.apache.spark.sql.execution.WholeStageCodegenExec.doCodeGen(
*WholeStageCodegenExec.scala:632*)

            at
org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(
*WholeStageCodegenExec.scala:692*)

            at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(
*SparkPlan.scala:175*)

            at
org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
*SparkPlan.scala:213*)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)

            at org.apache.spark.sql.execution.SparkPlan.executeQuery(
*SparkPlan.scala:210*)

            at org.apache.spark.sql.execution.SparkPlan.execute(
*SparkPlan.scala:171*)

            at
org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.inputRDD$lzycompute(
*ShuffleExchangeExec.scala:106*)

            at
org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.inputRDD(
*ShuffleExchangeExec.scala:106*)

            at
org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.shuffleDependency$lzycompute(
*ShuffleExchangeExec.scala:139*)

            at
org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.shuffleDependency(
*ShuffleExchangeExec.scala:137*)

            at
org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.$anonfun$doExecute$1(
*ShuffleExchangeExec.scala:154*)

            at org.apache.spark.sql.catalyst.errors.package$.attachTree(
*package.scala:52*)

            ... 53 more

Caused by: *org.apache.spark.SparkException*: Job aborted due to stage
failure: Task 0 in stage 23.0 failed 1 times, most recent failure: Lost
task 0.0 in stage 23.0 (TID 29, del1-lhp-n99999.synapse.com, executor
driver): TaskResultLost (result lost from block manager)

Driver stacktrace:

            at
org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(
*DAGScheduler.scala:2059*)

            at
org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(
*DAGScheduler.scala:2008*)

            at
org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(
*DAGScheduler.scala:2007*)

            at scala.collection.mutable.ResizableArray.foreach(
*ResizableArray.scala:59*)

            at scala.collection.mutable.ResizableArray.foreach$(
*ResizableArray.scala:52*)

            at scala.collection.mutable.ArrayBuffer.foreach(
*ArrayBuffer.scala:48*)

            at org.apache.spark.scheduler.DAGScheduler.abortStage(
*DAGScheduler.scala:2007*)

            at
org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(
*DAGScheduler.scala:973*)

            at
org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(
*DAGScheduler.scala:973*)

            at scala.Option.foreach(*Option.scala:257*)

            at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(
*DAGScheduler.scala:973*)

            at
org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(
*DAGScheduler.scala:2239*)

            at
org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(
*DAGScheduler.scala:2188*)

            at
org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(
*DAGScheduler.scala:2177*)

            at org.apache.spark.util.EventLoop$$anon$1.run(
*EventLoop.scala:49*)

            at org.apache.spark.scheduler.DAGScheduler.runJob(
*DAGScheduler.scala:775*)

            at org.apache.spark.SparkContext.runJob(
*SparkContext.scala:2099*)

            at org.apache.spark.SparkContext.runJob(
*SparkContext.scala:2120*)

            at org.apache.spark.SparkContext.runJob(
*SparkContext.scala:2139*)

            at org.apache.spark.SparkContext.runJob(
*SparkContext.scala:2164*)

            at org.apache.spark.rdd.RDD.$anonfun$collect$1(*RDD.scala:1004*)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:151*)

            at org.apache.spark.rdd.RDDOperationScope$.withScope(
*RDDOperationScope.scala:112*)

            at org.apache.spark.rdd.RDD.withScope(*RDD.scala:388*)

            at org.apache.spark.rdd.RDD.collect(*RDD.scala:1003*)

            at
org.apache.spark.sql.execution.SparkPlan.executeCollectIterator(
*SparkPlan.scala:392*)

            at
org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.$anonfun$relationFuture$1(
*BroadcastExchangeExec.scala:120*)

            at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(
*SQLExecution.scala:182*)

            at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)

            at java.util.concurrent.ThreadPoolExecutor.runWorker(
*ThreadPoolExecutor.java:1149*)

            at java.util.concurrent.ThreadPoolExecutor$Worker.run(
*ThreadPoolExecutor.java:624*)

            at java.lang.Thread.run(*Thread.java:748*)

Mime
View raw message