spark-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Vikas Garg <sperry...@gmail.com>
Subject Re: Issue with executer
Date Wed, 20 Jan 2021 12:30:12 GMT
Hi Sean,

This works fine when I run with very less number of files. Looks like the
issue is with memory or may be something else. I am unable to figure it out.

On Wed, 20 Jan 2021 at 17:58, Sean Owen <srowen@gmail.com> wrote:

> Does not look related to Spark - you have network connectivity problems.
>
> On Wed, Jan 20, 2021 at 6:05 AM Vikas Garg <sperry.it@gmail.com> wrote:
>
>> Hi,
>>
>> I am facing issue with spark executor. I am struggling with this issue
>> since last many days and unable to resolve the issue.
>>
>> Below is the configuration I have given.
>>
>>   val spark = SparkSession.builder()
>>     .appName("Spark Job")
>>     .master("local[*]")
>>     .config("spark.dynamicAllocation.enabled", true)
>>     .config("spark.shuffle.service.enabled", true)
>>     .config("spark.driver.maxResultSize", "8g")
>>     .config("spark.driver.memory", "8g")
>>     .config("spark.executor.memory", "8g")
>>     .config("spark.network.timeout", "3600s")
>>     .getOrCreate()
>>
>> 1/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning
>> fetch of 1 outstanding blocks
>>
>> *java.io.IOException*: Failed to connect to
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>>             at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:253*)
>>
>>             at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:195*)
>>
>>             at
>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>> *NettyBlockTransferService.scala:122*)
>>
>>             at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>> *RetryingBlockFetcher.java:141*)
>>
>>             at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.start(
>> *RetryingBlockFetcher.java:121*)
>>
>>             at
>> org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(
>> *NettyBlockTransferService.scala:143*)
>>
>>             at
>> org.apache.spark.network.BlockTransferService.fetchBlockSync(
>> *BlockTransferService.scala:103*)
>>
>>             at
>> org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(
>> *BlockManager.scala:1010*)
>>
>>             at
>> org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(
>> *BlockManager.scala:954*)
>>
>>             at scala.Option.orElse(*Option.scala:289*)
>>
>>             at org.apache.spark.storage.BlockManager.getRemoteBlock(
>> *BlockManager.scala:954*)
>>
>>             at org.apache.spark.storage.BlockManager.getRemoteBytes(
>> *BlockManager.scala:1092*)
>>
>>             at
>> org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(
>> *TaskResultGetter.scala:88*)
>>
>>             at
>> scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)
>>
>>             at org.apache.spark.util.Utils$.logUncaughtExceptions(
>> *Utils.scala:1932*)
>>
>>             at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(
>> *TaskResultGetter.scala:63*)
>>
>>             at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>>             at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>>             at java.lang.Thread.run(*Thread.java:748*)
>>
>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>> Permission denied: no further information:
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> Caused by: *java.net.SocketException*: Permission denied: no further
>> information
>>
>>             at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>>
>>             at sun.nio.ch.SocketChannelImpl.finishConnect(
>> *SocketChannelImpl.java:715*)
>>
>>             at
>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>> *NioSocketChannel.java:330*)
>>
>>             at
>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>> *AbstractNioChannel.java:334*)
>>
>>             at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>> *NioEventLoop.java:702*)
>>
>>             at
>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>> *NioEventLoop.java:650*)
>>
>>             at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>> *NioEventLoop.java:576*)
>>
>>             at io.netty.channel.nio.NioEventLoop.run(
>> *NioEventLoop.java:493*)
>>
>>             at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>> *SingleThreadEventExecutor.java:989*)
>>
>>             at io.netty.util.internal.ThreadExecutorMap$2.run(
>> *ThreadExecutorMap.java:74*)
>>
>>             at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>>             at java.lang.Thread.run(*Thread.java:748*)
>>
>> 21/01/20 17:06:57 ERROR RetryingBlockFetcher: Exception while beginning
>> fetch of 1 outstanding blocks
>>
>> *java.io.IOException*: Failed to connect to
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>>             at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:253*)
>>
>>             at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:195*)
>>
>>             at
>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>> *NettyBlockTransferService.scala:122*)
>>
>>             at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>> *RetryingBlockFetcher.java:141*)
>>
>>             at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.start(
>> *RetryingBlockFetcher.java:121*)
>>
>>             at
>> org.apache.spark.network.netty.NettyBlockTransferService.fetchBlocks(
>> *NettyBlockTransferService.scala:143*)
>>
>>             at
>> org.apache.spark.network.BlockTransferService.fetchBlockSync(
>> *BlockTransferService.scala:103*)
>>
>>             at
>> org.apache.spark.storage.BlockManager.fetchRemoteManagedBuffer(
>> *BlockManager.scala:1010*)
>>
>>             at
>> org.apache.spark.storage.BlockManager.$anonfun$getRemoteBlock$8(
>> *BlockManager.scala:954*)
>>
>>             at scala.Option.orElse(*Option.scala:289*)
>>
>>             at org.apache.spark.storage.BlockManager.getRemoteBlock(
>> *BlockManager.scala:954*)
>>
>>             at org.apache.spark.storage.BlockManager.getRemoteBytes(
>> *BlockManager.scala:1092*)
>>
>>             at
>> org.apache.spark.scheduler.TaskResultGetter$$anon$3.$anonfun$run$1(
>> *TaskResultGetter.scala:88*)
>>
>>             at
>> scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:12)
>>
>>             at org.apache.spark.util.Utils$.logUncaughtExceptions(
>> *Utils.scala:1932*)
>>
>>             at org.apache.spark.scheduler.TaskResultGetter$$anon$3.run(
>> *TaskResultGetter.scala:63*)
>>
>>             at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>>             at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>>             at java.lang.Thread.run(*Thread.java:748*)
>>
>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>> Permission denied: no further information:
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> Caused by: *java.net.SocketException*: Permission denied: no further
>> information
>>
>>             at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>>
>>             at sun.nio.ch.SocketChannelImpl.finishConnect(
>> *SocketChannelImpl.java:715*)
>>
>>             at
>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>> *NioSocketChannel.java:330*)
>>
>>             at
>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>> *AbstractNioChannel.java:334*)
>>
>>             at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>> *NioEventLoop.java:702*)
>>
>>             at
>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>> *NioEventLoop.java:650*)
>>
>>             at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>> *NioEventLoop.java:576*)
>>
>>             at io.netty.channel.nio.NioEventLoop.run(
>> *NioEventLoop.java:493*)
>>
>>             at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>> *SingleThreadEventExecutor.java:989*)
>>
>>             at io.netty.util.internal.ThreadExecutorMap$2.run(
>> *ThreadExecutorMap.java:74*)
>>
>>             at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>>             at java.lang.Thread.run(*Thread.java:748*)
>>
>> 21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning
>> fetch of 1 outstanding blocks (after 1 retries)
>>
>> *java.io.IOException*: Failed to connect to
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>>             at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:253*)
>>
>>             at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:195*)
>>
>>             at
>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>> *NettyBlockTransferService.scala:122*)
>>
>>             at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>> *RetryingBlockFetcher.java:141*)
>>
>>             at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>> *RetryingBlockFetcher.java:169*)
>>
>>             at java.util.concurrent.Executors$RunnableAdapter.call(
>> *Executors.java:511*)
>>
>>             at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
>>
>>             at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>>             at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>>             at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>>             at java.lang.Thread.run(*Thread.java:748*)
>>
>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>> Permission denied: no further information:
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> Caused by: *java.net.SocketException*: Permission denied: no further
>> information
>>
>>             at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>>
>>             at sun.nio.ch.SocketChannelImpl.finishConnect(
>> *SocketChannelImpl.java:715*)
>>
>>             at
>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>> *NioSocketChannel.java:330*)
>>
>>             at
>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>> *AbstractNioChannel.java:334*)
>>
>>             at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>> *NioEventLoop.java:702*)
>>
>>             at
>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>> *NioEventLoop.java:650*)
>>
>>             at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>> *NioEventLoop.java:576*)
>>
>>             at io.netty.channel.nio.NioEventLoop.run(
>> *NioEventLoop.java:493*)
>>
>>             at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>> *SingleThreadEventExecutor.java:989*)
>>
>>             at io.netty.util.internal.ThreadExecutorMap$2.run(
>> *ThreadExecutorMap.java:74*)
>>
>>             at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>>             at java.lang.Thread.run(*Thread.java:748*)
>>
>> 21/01/20 17:07:02 ERROR RetryingBlockFetcher: Exception while beginning
>> fetch of 1 outstanding blocks (after 1 retries)
>>
>> *java.io.IOException*: Failed to connect to
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>>             at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:253*)
>>
>>             at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:195*)
>>
>>             at
>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>> *NettyBlockTransferService.scala:122*)
>>
>>             at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>> *RetryingBlockFetcher.java:141*)
>>
>>             at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>> *RetryingBlockFetcher.java:169*)
>>
>>             at java.util.concurrent.Executors$RunnableAdapter.call(
>> *Executors.java:511*)
>>
>>             at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
>>
>>             at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>>             at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>>             at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>>             at java.lang.Thread.run(*Thread.java:748*)
>>
>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>> Permission denied: no further information:
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> Caused by: *java.net.SocketException*: Permission denied: no further
>> information
>>
>>             at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>>
>>             at sun.nio.ch.SocketChannelImpl.finishConnect(
>> *SocketChannelImpl.java:715*)
>>
>>             at
>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>> *NioSocketChannel.java:330*)
>>
>>             at
>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>> *AbstractNioChannel.java:334*)
>>
>>             at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>> *NioEventLoop.java:702*)
>>
>>             at
>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>> *NioEventLoop.java:650*)
>>
>>             at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>> *NioEventLoop.java:576*)
>>
>>             at io.netty.channel.nio.NioEventLoop.run(
>> *NioEventLoop.java:493*)
>>
>>             at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>> *SingleThreadEventExecutor.java:989*)
>>
>>             at io.netty.util.internal.ThreadExecutorMap$2.run(
>> *ThreadExecutorMap.java:74*)
>>
>>             at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>>             at java.lang.Thread.run(*Thread.java:748*)
>>
>> 21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning
>> fetch of 1 outstanding blocks (after 2 retries)
>>
>> *java.io.IOException*: Failed to connect to
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>>             at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:253*)
>>
>>             at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:195*)
>>
>>             at
>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>> *NettyBlockTransferService.scala:122*)
>>
>>             at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>> *RetryingBlockFetcher.java:141*)
>>
>>             at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>> *RetryingBlockFetcher.java:169*)
>>
>>             at java.util.concurrent.Executors$RunnableAdapter.call(
>> *Executors.java:511*)
>>
>>             at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
>>
>>             at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>>             at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>>             at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>>             at java.lang.Thread.run(*Thread.java:748*)
>>
>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>> Permission denied: no further information:
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> Caused by: *java.net.SocketException*: Permission denied: no further
>> information
>>
>>             at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>>
>>             at sun.nio.ch.SocketChannelImpl.finishConnect(
>> *SocketChannelImpl.java:715*)
>>
>>             at
>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>> *NioSocketChannel.java:330*)
>>
>>             at
>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>> *AbstractNioChannel.java:334*)
>>
>>             at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>> *NioEventLoop.java:702*)
>>
>>             at
>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>> *NioEventLoop.java:650*)
>>
>>             at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>> *NioEventLoop.java:576*)
>>
>>             at io.netty.channel.nio.NioEventLoop.run(
>> *NioEventLoop.java:493*)
>>
>>             at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>> *SingleThreadEventExecutor.java:989*)
>>
>>             at io.netty.util.internal.ThreadExecutorMap$2.run(
>> *ThreadExecutorMap.java:74*)
>>
>>             at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>>             at java.lang.Thread.run(*Thread.java:748*)
>>
>> 21/01/20 17:07:07 ERROR RetryingBlockFetcher: Exception while beginning
>> fetch of 1 outstanding blocks (after 2 retries)
>>
>> *java.io.IOException*: Failed to connect to
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>>             at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:253*)
>>
>>             at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:195*)
>>
>>             at
>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>> *NettyBlockTransferService.scala:122*)
>>
>>             at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>> *RetryingBlockFetcher.java:141*)
>>
>>             at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>> *RetryingBlockFetcher.java:169*)
>>
>>             at java.util.concurrent.Executors$RunnableAdapter.call(
>> *Executors.java:511*)
>>
>>             at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
>>
>>             at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>>             at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>>             at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>>             at java.lang.Thread.run(*Thread.java:748*)
>>
>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>> Permission denied: no further information:
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> Caused by: *java.net.SocketException*: Permission denied: no further
>> information
>>
>>             at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>>
>>             at sun.nio.ch.SocketChannelImpl.finishConnect(
>> *SocketChannelImpl.java:715*)
>>
>>             at
>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>> *NioSocketChannel.java:330*)
>>
>>             at
>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>> *AbstractNioChannel.java:334*)
>>
>>             at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>> *NioEventLoop.java:702*)
>>
>>             at
>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>> *NioEventLoop.java:650*)
>>
>>             at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>> *NioEventLoop.java:576*)
>>
>>             at io.netty.channel.nio.NioEventLoop.run(
>> *NioEventLoop.java:493*)
>>
>>             at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>> *SingleThreadEventExecutor.java:989*)
>>
>>             at io.netty.util.internal.ThreadExecutorMap$2.run(
>> *ThreadExecutorMap.java:74*)
>>
>>             at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>>             at java.lang.Thread.run(*Thread.java:748*)
>>
>> 21/01/20 17:07:12 ERROR RetryingBlockFetcher: Exception while beginning
>> fetch of 1 outstanding blocks (after 3 retries)
>>
>> *java.io.IOException*: Failed to connect to
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>>             at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:253*)
>>
>>             at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:195*)
>>
>>             at
>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>> *NettyBlockTransferService.scala:122*)
>>
>>             at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>> *RetryingBlockFetcher.java:141*)
>>
>>             at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>> *RetryingBlockFetcher.java:169*)
>>
>>             at java.util.concurrent.Executors$RunnableAdapter.call(
>> *Executors.java:511*)
>>
>>             at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
>>
>>             at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>>             at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>>             at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>>             at java.lang.Thread.run(*Thread.java:748*)
>>
>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>> Permission denied: no further information:
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> Caused by: *java.net.SocketException*: Permission denied: no further
>> information
>>
>>             at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>>
>>             at sun.nio.ch.SocketChannelImpl.finishConnect(
>> *SocketChannelImpl.java:715*)
>>
>>             at
>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>> *NioSocketChannel.java:330*)
>>
>>             at
>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>> *AbstractNioChannel.java:334*)
>>
>>             at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>> *NioEventLoop.java:702*)
>>
>>             at
>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>> *NioEventLoop.java:650*)
>>
>>             at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>> *NioEventLoop.java:576*)
>>
>>             at io.netty.channel.nio.NioEventLoop.run(
>> *NioEventLoop.java:493*)
>>
>>             at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>> *SingleThreadEventExecutor.java:989*)
>>
>>             at io.netty.util.internal.ThreadExecutorMap$2.run(
>> *ThreadExecutorMap.java:74*)
>>
>>             at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>>             at java.lang.Thread.run(*Thread.java:748*)
>>
>> 21/01/20 17:07:12 ERROR RetryingBlockFetcher: Exception while beginning
>> fetch of 1 outstanding blocks (after 3 retries)
>>
>> *java.io.IOException*: Failed to connect to
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>>             at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:253*)
>>
>>             at
>> org.apache.spark.network.client.TransportClientFactory.createClient(
>> *TransportClientFactory.java:195*)
>>
>>             at
>> org.apache.spark.network.netty.NettyBlockTransferService$$anon$2.createAndStart(
>> *NettyBlockTransferService.scala:122*)
>>
>>             at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.fetchAllOutstanding(
>> *RetryingBlockFetcher.java:141*)
>>
>>             at
>> org.apache.spark.network.shuffle.RetryingBlockFetcher.lambda$initiateRetry$0(
>> *RetryingBlockFetcher.java:169*)
>>
>>             at java.util.concurrent.Executors$RunnableAdapter.call(
>> *Executors.java:511*)
>>
>>             at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
>>
>>             at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>>             at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>>             at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>>             at java.lang.Thread.run(*Thread.java:748*)
>>
>> Caused by: *io.netty.channel.AbstractChannel$AnnotatedSocketException*:
>> Permission denied: no further information:
>> del1-lhp-n99999.synapse.com/192.168.166.213:51348
>>
>> Caused by: *java.net.SocketException*: Permission denied: no further
>> information
>>
>>             at sun.nio.ch.SocketChannelImpl.checkConnect(*Native Method*)
>>
>>             at sun.nio.ch.SocketChannelImpl.finishConnect(
>> *SocketChannelImpl.java:715*)
>>
>>             at
>> io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(
>> *NioSocketChannel.java:330*)
>>
>>             at
>> io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(
>> *AbstractNioChannel.java:334*)
>>
>>             at io.netty.channel.nio.NioEventLoop.processSelectedKey(
>> *NioEventLoop.java:702*)
>>
>>             at
>> io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(
>> *NioEventLoop.java:650*)
>>
>>             at io.netty.channel.nio.NioEventLoop.processSelectedKeys(
>> *NioEventLoop.java:576*)
>>
>>             at io.netty.channel.nio.NioEventLoop.run(
>> *NioEventLoop.java:493*)
>>
>>             at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(
>> *SingleThreadEventExecutor.java:989*)
>>
>>             at io.netty.util.internal.ThreadExecutorMap$2.run(
>> *ThreadExecutorMap.java:74*)
>>
>>             at io.netty.util.concurrent.FastThreadLocalRunnable.run(
>> *FastThreadLocalRunnable.java:30*)
>>
>>             at java.lang.Thread.run(*Thread.java:748*)
>>
>> 21/01/20 17:07:12 ERROR TaskSetManager: Task 0 in stage 23.0 failed 1
>> times; aborting job
>>
>> 21/01/20 17:07:12 ERROR TaskSetManager: Task 2 in stage 17.0 failed 1
>> times; aborting job
>>
>> Exception in thread "main"
>> *org.apache.spark.sql.catalyst.errors.package$TreeNodeException*:
>> execute, tree:
>>
>> Exchange hashpartitioning(claimid#1012, 200), true, [id=#742]
>>
>>
>>
>>
>>
>>             at org.apache.spark.sql.catalyst.errors.package$.attachTree(
>> *package.scala:56*)
>>
>>             at
>> org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.doExecute(
>> *ShuffleExchangeExec.scala:151*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(
>> *SparkPlan.scala:175*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>>             at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.execute(
>> *SparkPlan.scala:171*)
>>
>>             at org.apache.spark.sql.execution.InputAdapter.inputRDD(
>> *WholeStageCodegenExec.scala:525*)
>>
>>             at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs(
>> *WholeStageCodegenExec.scala:453*)
>>
>>             at org.apache.spark.sql.execution.InputRDDCodegen.inputRDDs$(
>> *WholeStageCodegenExec.scala:452*)
>>
>>             at org.apache.spark.sql.execution.InputAdapter.inputRDDs(
>> *WholeStageCodegenExec.scala:496*)
>>
>>             at org.apache.spark.sql.execution.SortExec.inputRDDs(
>> *SortExec.scala:132*)
>>
>>             at
>> org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(
>> *WholeStageCodegenExec.scala:720*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(
>> *SparkPlan.scala:175*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>>             at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.execute(
>> *SparkPlan.scala:171*)
>>
>>             at
>> org.apache.spark.sql.execution.joins.SortMergeJoinExec.doExecute(
>> *SortMergeJoinExec.scala:182*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(
>> *SparkPlan.scala:175*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>>             at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.execute(
>> *SparkPlan.scala:171*)
>>
>>             at org.apache.spark.sql.execution.ProjectExec.doExecute(
>> *basicPhysicalOperators.scala:76*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(
>> *SparkPlan.scala:175*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>>             at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.execute(
>> *SparkPlan.scala:171*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.getByteArrayRdd(
>> *SparkPlan.scala:316*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.executeTake(
>> *SparkPlan.scala:434*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.executeTake(
>> *SparkPlan.scala:420*)
>>
>>             at
>> org.apache.spark.sql.execution.CollectLimitExec.executeCollect(
>> *limit.scala:47*)
>>
>>             at org.apache.spark.sql.Dataset.collectFromPlan(
>> *Dataset.scala:3627*)
>>
>>             at org.apache.spark.sql.Dataset.$anonfun$head$1(
>> *Dataset.scala:2697*)
>>
>>             at org.apache.spark.sql.Dataset.$anonfun$withAction$1(
>> *Dataset.scala:3618*)
>>
>>             at
>> org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$5(
>> *SQLExecution.scala:100*)
>>
>>             at
>> org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(
>> *SQLExecution.scala:160*)
>>
>>             at
>> org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(
>> *SQLExecution.scala:87*)
>>
>>             at org.apache.spark.sql.SparkSession.withActive(
>> *SparkSession.scala:764*)
>>
>>             at
>> org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(
>> *SQLExecution.scala:64*)
>>
>>             at org.apache.spark.sql.Dataset.withAction(
>> *Dataset.scala:3616*)
>>
>>             at org.apache.spark.sql.Dataset.head(*Dataset.scala:2697*)
>>
>>             at org.apache.spark.sql.Dataset.take(*Dataset.scala:2904*)
>>
>>             at org.apache.spark.sql.Dataset.getRows(*Dataset.scala:300*)
>>
>>             at org.apache.spark.sql.Dataset.showString(
>> *Dataset.scala:337*)
>>
>>             at org.apache.spark.sql.Dataset.show(*Dataset.scala:824*)
>>
>>             at org.apache.spark.sql.Dataset.show(*Dataset.scala:783*)
>>
>>             at org.apache.spark.sql.Dataset.show(*Dataset.scala:792*)
>>
>>             at com.brighthealthplan.cognizant.output.MedicalClaim$.getDS(
>> *MedicalClaim.scala:257*)
>>
>>             at
>> com.brighthealthplan.cognizant.sparkjobs.DataManager.writeLeft(
>> *DataManager.scala:192*)
>>
>>             at com.brighthealthplan.cognizant.sparkjobs.Main$.main(
>> *Main.scala:107*)
>>
>>             at
>> com.brighthealthplan.cognizant.sparkjobs.Main.main(Main.scala)
>>
>> Caused by: *java.util.concurrent.ExecutionException*:
>> *org.apache.spark.SparkException*: Job aborted due to stage failure:
>> Task 0 in stage 23.0 failed 1 times, most recent failure: Lost task 0.0 in
>> stage 23.0 (TID 29, del1-lhp-n99999.synapse.com, executor driver):
>> TaskResultLost (result lost from block manager)
>>
>> Driver stacktrace:
>>
>>             at java.util.concurrent.FutureTask.report(
>> *FutureTask.java:122*)
>>
>>             at java.util.concurrent.FutureTask.get(*FutureTask.java:206*)
>>
>>             at
>> org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.doExecuteBroadcast(
>> *BroadcastExchangeExec.scala:195*)
>>
>>             at
>> org.apache.spark.sql.execution.InputAdapter.doExecuteBroadcast(
>> *WholeStageCodegenExec.scala:515*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeBroadcast$1(
>> *SparkPlan.scala:188*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>>             at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.executeBroadcast(
>> *SparkPlan.scala:184*)
>>
>>             at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.prepareBroadcast(
>> *BroadcastHashJoinExec.scala:116*)
>>
>>             at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenOuter(
>> *BroadcastHashJoinExec.scala:258*)
>>
>>             at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(
>> *BroadcastHashJoinExec.scala:101*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.consume(
>> *WholeStageCodegenExec.scala:194*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.consume$(
>> *WholeStageCodegenExec.scala:149*)
>>
>>             at org.apache.spark.sql.execution.ProjectExec.consume(
>> *basicPhysicalOperators.scala:41*)
>>
>>             at org.apache.spark.sql.execution.ProjectExec.doConsume(
>> *basicPhysicalOperators.scala:71*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.consume(
>> *WholeStageCodegenExec.scala:194*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.consume$(
>> *WholeStageCodegenExec.scala:149*)
>>
>>             at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.consume(
>> *BroadcastHashJoinExec.scala:39*)
>>
>>             at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenOuter(
>> *BroadcastHashJoinExec.scala:302*)
>>
>>             at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(
>> *BroadcastHashJoinExec.scala:101*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.consume(
>> *WholeStageCodegenExec.scala:194*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.consume$(
>> *WholeStageCodegenExec.scala:149*)
>>
>>             at org.apache.spark.sql.execution.ProjectExec.consume(
>> *basicPhysicalOperators.scala:41*)
>>
>>             at org.apache.spark.sql.execution.ProjectExec.doConsume(
>> *basicPhysicalOperators.scala:71*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.consume(
>> *WholeStageCodegenExec.scala:194*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.consume$(
>> *WholeStageCodegenExec.scala:149*)
>>
>>             at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.consume(
>> *BroadcastHashJoinExec.scala:39*)
>>
>>             at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.codegenOuter(
>> *BroadcastHashJoinExec.scala:302*)
>>
>>             at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doConsume(
>> *BroadcastHashJoinExec.scala:101*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.consume(
>> *WholeStageCodegenExec.scala:194*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.consume$(
>> *WholeStageCodegenExec.scala:149*)
>>
>>             at org.apache.spark.sql.execution.ProjectExec.consume(
>> *basicPhysicalOperators.scala:41*)
>>
>>             at org.apache.spark.sql.execution.ProjectExec.doConsume(
>> *basicPhysicalOperators.scala:71*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.consume(
>> *WholeStageCodegenExec.scala:194*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.consume$(
>> *WholeStageCodegenExec.scala:149*)
>>
>>             at org.apache.spark.sql.execution.InputAdapter.consume(
>> *WholeStageCodegenExec.scala:496*)
>>
>>             at org.apache.spark.sql.execution.InputRDDCodegen.doProduce(
>> *WholeStageCodegenExec.scala:483*)
>>
>>             at org.apache.spark.sql.execution.InputRDDCodegen.doProduce$(
>> *WholeStageCodegenExec.scala:456*)
>>
>>             at org.apache.spark.sql.execution.InputAdapter.doProduce(
>> *WholeStageCodegenExec.scala:496*)
>>
>>             at
>> org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
>> *WholeStageCodegenExec.scala:95*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>>             at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.produce(
>> *WholeStageCodegenExec.scala:90*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.produce$(
>> *WholeStageCodegenExec.scala:90*)
>>
>>             at org.apache.spark.sql.execution.InputAdapter.produce(
>> *WholeStageCodegenExec.scala:496*)
>>
>>             at org.apache.spark.sql.execution.ProjectExec.doProduce(
>> *basicPhysicalOperators.scala:51*)
>>
>>             at
>> org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
>> *WholeStageCodegenExec.scala:95*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>>             at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.produce(
>> *WholeStageCodegenExec.scala:90*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.produce$(
>> *WholeStageCodegenExec.scala:90*)
>>
>>             at org.apache.spark.sql.execution.ProjectExec.produce(
>> *basicPhysicalOperators.scala:41*)
>>
>>             at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(
>> *BroadcastHashJoinExec.scala:95*)
>>
>>             at
>> org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
>> *WholeStageCodegenExec.scala:95*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>>             at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.produce(
>> *WholeStageCodegenExec.scala:90*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.produce$(
>> *WholeStageCodegenExec.scala:90*)
>>
>>             at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(
>> *BroadcastHashJoinExec.scala:39*)
>>
>>             at org.apache.spark.sql.execution.ProjectExec.doProduce(
>> *basicPhysicalOperators.scala:51*)
>>
>>             at
>> org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
>> *WholeStageCodegenExec.scala:95*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>>             at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.produce(
>> *WholeStageCodegenExec.scala:90*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.produce$(
>> *WholeStageCodegenExec.scala:90*)
>>
>>             at org.apache.spark.sql.execution.ProjectExec.produce(
>> *basicPhysicalOperators.scala:41*)
>>
>>             at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(
>> *BroadcastHashJoinExec.scala:95*)
>>
>>             at
>> org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
>> *WholeStageCodegenExec.scala:95*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>>             at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.produce(
>> *WholeStageCodegenExec.scala:90*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.produce$(
>> *WholeStageCodegenExec.scala:90*)
>>
>>             at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(
>> *BroadcastHashJoinExec.scala:39*)
>>
>>             at org.apache.spark.sql.execution.ProjectExec.doProduce(
>> *basicPhysicalOperators.scala:51*)
>>
>>             at
>> org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
>> *WholeStageCodegenExec.scala:95*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>>             at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.produce(
>> *WholeStageCodegenExec.scala:90*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.produce$(
>> *WholeStageCodegenExec.scala:90*)
>>
>>             at org.apache.spark.sql.execution.ProjectExec.produce(
>> *basicPhysicalOperators.scala:41*)
>>
>>             at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(
>> *BroadcastHashJoinExec.scala:95*)
>>
>>             at
>> org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
>> *WholeStageCodegenExec.scala:95*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>>             at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.produce(
>> *WholeStageCodegenExec.scala:90*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.produce$(
>> *WholeStageCodegenExec.scala:90*)
>>
>>             at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(
>> *BroadcastHashJoinExec.scala:39*)
>>
>>             at org.apache.spark.sql.execution.ProjectExec.doProduce(
>> *basicPhysicalOperators.scala:51*)
>>
>>             at
>> org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
>> *WholeStageCodegenExec.scala:95*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>>             at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.produce(
>> *WholeStageCodegenExec.scala:90*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.produce$(
>> *WholeStageCodegenExec.scala:90*)
>>
>>             at org.apache.spark.sql.execution.ProjectExec.produce(
>> *basicPhysicalOperators.scala:41*)
>>
>>             at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(
>> *BroadcastHashJoinExec.scala:95*)
>>
>>             at
>> org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
>> *WholeStageCodegenExec.scala:95*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>>             at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.produce(
>> *WholeStageCodegenExec.scala:90*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.produce$(
>> *WholeStageCodegenExec.scala:90*)
>>
>>             at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(
>> *BroadcastHashJoinExec.scala:39*)
>>
>>             at org.apache.spark.sql.execution.ProjectExec.doProduce(
>> *basicPhysicalOperators.scala:51*)
>>
>>             at
>> org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
>> *WholeStageCodegenExec.scala:95*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>>             at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.produce(
>> *WholeStageCodegenExec.scala:90*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.produce$(
>> *WholeStageCodegenExec.scala:90*)
>>
>>             at org.apache.spark.sql.execution.ProjectExec.produce(
>> *basicPhysicalOperators.scala:41*)
>>
>>             at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.doProduce(
>> *BroadcastHashJoinExec.scala:95*)
>>
>>             at
>> org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
>> *WholeStageCodegenExec.scala:95*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>>             at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.produce(
>> *WholeStageCodegenExec.scala:90*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.produce$(
>> *WholeStageCodegenExec.scala:90*)
>>
>>             at
>> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.produce(
>> *BroadcastHashJoinExec.scala:39*)
>>
>>             at org.apache.spark.sql.execution.ProjectExec.doProduce(
>> *basicPhysicalOperators.scala:51*)
>>
>>             at
>> org.apache.spark.sql.execution.CodegenSupport.$anonfun$produce$1(
>> *WholeStageCodegenExec.scala:95*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>>             at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.produce(
>> *WholeStageCodegenExec.scala:90*)
>>
>>             at org.apache.spark.sql.execution.CodegenSupport.produce$(
>> *WholeStageCodegenExec.scala:90*)
>>
>>             at org.apache.spark.sql.execution.ProjectExec.produce(
>> *basicPhysicalOperators.scala:41*)
>>
>>             at
>> org.apache.spark.sql.execution.WholeStageCodegenExec.doCodeGen(
>> *WholeStageCodegenExec.scala:632*)
>>
>>             at
>> org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(
>> *WholeStageCodegenExec.scala:692*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(
>> *SparkPlan.scala:175*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(
>> *SparkPlan.scala:213*)
>>
>>             at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.executeQuery(
>> *SparkPlan.scala:210*)
>>
>>             at org.apache.spark.sql.execution.SparkPlan.execute(
>> *SparkPlan.scala:171*)
>>
>>             at
>> org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.inputRDD$lzycompute(
>> *ShuffleExchangeExec.scala:106*)
>>
>>             at
>> org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.inputRDD(
>> *ShuffleExchangeExec.scala:106*)
>>
>>             at
>> org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.shuffleDependency$lzycompute(
>> *ShuffleExchangeExec.scala:139*)
>>
>>             at
>> org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.shuffleDependency(
>> *ShuffleExchangeExec.scala:137*)
>>
>>             at
>> org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.$anonfun$doExecute$1(
>> *ShuffleExchangeExec.scala:154*)
>>
>>             at org.apache.spark.sql.catalyst.errors.package$.attachTree(
>> *package.scala:52*)
>>
>>             ... 53 more
>>
>> Caused by: *org.apache.spark.SparkException*: Job aborted due to stage
>> failure: Task 0 in stage 23.0 failed 1 times, most recent failure: Lost
>> task 0.0 in stage 23.0 (TID 29, del1-lhp-n99999.synapse.com, executor
>> driver): TaskResultLost (result lost from block manager)
>>
>> Driver stacktrace:
>>
>>             at
>> org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(
>> *DAGScheduler.scala:2059*)
>>
>>             at
>> org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2(
>> *DAGScheduler.scala:2008*)
>>
>>             at
>> org.apache.spark.scheduler.DAGScheduler.$anonfun$abortStage$2$adapted(
>> *DAGScheduler.scala:2007*)
>>
>>             at scala.collection.mutable.ResizableArray.foreach(
>> *ResizableArray.scala:59*)
>>
>>             at scala.collection.mutable.ResizableArray.foreach$(
>> *ResizableArray.scala:52*)
>>
>>             at scala.collection.mutable.ArrayBuffer.foreach(
>> *ArrayBuffer.scala:48*)
>>
>>             at org.apache.spark.scheduler.DAGScheduler.abortStage(
>> *DAGScheduler.scala:2007*)
>>
>>             at
>> org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1(
>> *DAGScheduler.scala:973*)
>>
>>             at
>> org.apache.spark.scheduler.DAGScheduler.$anonfun$handleTaskSetFailed$1$adapted(
>> *DAGScheduler.scala:973*)
>>
>>             at scala.Option.foreach(*Option.scala:257*)
>>
>>             at
>> org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(
>> *DAGScheduler.scala:973*)
>>
>>             at
>> org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(
>> *DAGScheduler.scala:2239*)
>>
>>             at
>> org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(
>> *DAGScheduler.scala:2188*)
>>
>>             at
>> org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(
>> *DAGScheduler.scala:2177*)
>>
>>             at org.apache.spark.util.EventLoop$$anon$1.run(
>> *EventLoop.scala:49*)
>>
>>             at org.apache.spark.scheduler.DAGScheduler.runJob(
>> *DAGScheduler.scala:775*)
>>
>>             at org.apache.spark.SparkContext.runJob(
>> *SparkContext.scala:2099*)
>>
>>             at org.apache.spark.SparkContext.runJob(
>> *SparkContext.scala:2120*)
>>
>>             at org.apache.spark.SparkContext.runJob(
>> *SparkContext.scala:2139*)
>>
>>             at org.apache.spark.SparkContext.runJob(
>> *SparkContext.scala:2164*)
>>
>>             at org.apache.spark.rdd.RDD.$anonfun$collect$1(
>> *RDD.scala:1004*)
>>
>>             at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:151*)
>>
>>             at org.apache.spark.rdd.RDDOperationScope$.withScope(
>> *RDDOperationScope.scala:112*)
>>
>>             at org.apache.spark.rdd.RDD.withScope(*RDD.scala:388*)
>>
>>             at org.apache.spark.rdd.RDD.collect(*RDD.scala:1003*)
>>
>>             at
>> org.apache.spark.sql.execution.SparkPlan.executeCollectIterator(
>> *SparkPlan.scala:392*)
>>
>>             at
>> org.apache.spark.sql.execution.exchange.BroadcastExchangeExec.$anonfun$relationFuture$1(
>> *BroadcastExchangeExec.scala:120*)
>>
>>             at
>> org.apache.spark.sql.execution.SQLExecution$.$anonfun$withThreadLocalCaptured$1(
>> *SQLExecution.scala:182*)
>>
>>             at java.util.concurrent.FutureTask.run(*FutureTask.java:266*)
>>
>>             at java.util.concurrent.ThreadPoolExecutor.runWorker(
>> *ThreadPoolExecutor.java:1149*)
>>
>>             at java.util.concurrent.ThreadPoolExecutor$Worker.run(
>> *ThreadPoolExecutor.java:624*)
>>
>>             at java.lang.Thread.run(*Thread.java:748*)
>>
>>

Mime
View raw message