lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Apache Jenkins Server <jenk...@builds.apache.org>
Subject [JENKINS] Lucene-Solr-NightlyTests-trunk - Build # 822 - Still Failing
Date Thu, 15 Oct 2015 11:44:08 GMT
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-trunk/822/

5 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIDistributedZkTest

Error Message:
298 threads leaked from SUITE scope at org.apache.solr.cloud.CollectionsAPIDistributedZkTest:     1) Thread[id=20903, name=searcherExecutor-5107-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=22223, name=coreZkRegister-4308-thread-30-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:shard2 c:awholynewstresscollection_collection5_0 r:core_node25, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1071)         at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1041)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:997)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=20423, name=searcherExecutor-4604-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=22261, name=coreZkRegister-4308-thread-44-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:shard2 c:awholynewstresscollection_collection5_0 r:core_node25, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1071)         at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1041)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:997)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=22325, name=coreZkRegister-4308-thread-54-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:shard2 c:awholynewstresscollection_collection5_0 r:core_node25, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=20722, name=searcherExecutor-4962-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=20418, name=searcherExecutor-4603-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=20009, name=searcherExecutor-4452-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=20721, name=searcherExecutor-4897-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=20600, name=searcherExecutor-4836-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   11) Thread[id=22236, name=coreZkRegister-4308-thread-36-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:shard2 c:awholynewstresscollection_collection5_0 r:core_node25, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1071)         at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1041)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:997)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   12) Thread[id=22199, name=coreZkRegister-4379-thread-66-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1071)         at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1041)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:997)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   13) Thread[id=21723, name=coreZkRegister-4308-thread-12-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:shard2 c:awholynewstresscollection_collection5_0 r:core_node25, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   14) Thread[id=19685, name=qtp863314704-19685-selector-ServerConnectorManager@6fb78d43/0, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)         at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)         at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)         at java.lang.Thread.run(Thread.java:745)   15) Thread[id=20419, name=searcherExecutor-4605-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   16) Thread[id=21774, name=coreZkRegister-4379-thread-44-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   17) Thread[id=20727, name=searcherExecutor-4950-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   18) Thread[id=22713, name=qtp1064931382-22713, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590)         at java.lang.Thread.run(Thread.java:745)   19) Thread[id=22185, name=coreZkRegister-4379-thread-63-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   20) Thread[id=22201, name=coreZkRegister-4379-thread-67-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   21) Thread[id=21767, name=coreZkRegister-4379-thread-41-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1071)         at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1041)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:997)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   22) Thread[id=19705, name=Thread-13726, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1155)   23) Thread[id=22437, name=RecoveryThread-awholynewstresscollection_collection0_0_shard1_replica1, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:519)         at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:225)   24) Thread[id=19908, name=Thread-13811, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1155)   25) Thread[id=21543, name=coreZkRegister-4379-thread-25-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   26) Thread[id=20683, name=searcherExecutor-4882-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   27) Thread[id=22209, name=coreZkRegister-4379-thread-69-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   28) Thread[id=21703, name=coreZkRegister-4308-thread-3-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:shard2 c:awholynewstresscollection_collection5_0 r:core_node25, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1071)         at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1041)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1024)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   29) Thread[id=22191, name=qtp863314704-22191, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590)         at java.lang.Thread.run(Thread.java:745)   30) Thread[id=19806, name=Scheduler-202435354, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   31) Thread[id=22320, name=coreZkRegister-4308-thread-50-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:shard2 c:awholynewstresscollection_collection5_0 r:core_node25, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   32) Thread[id=21798, name=coreZkRegister-4308-thread-25-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:shard2 c:awholynewstresscollection_collection5_0 r:core_node25, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   33) Thread[id=22699, name=zkCallback-1125-thread-8-processing-n:127.0.0.1:41155_-EventThread, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)   34) Thread[id=21763, name=coreZkRegister-4379-thread-38-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   35) Thread[id=19892, name=Scheduler-568486637, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   36) Thread[id=20507, name=searcherExecutor-4734-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   37) Thread[id=20004, name=searcherExecutor-4443-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   38) Thread[id=20967, name=searcherExecutor-5125-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   39) Thread[id=19780, name=Scheduler-955816862, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   40) Thread[id=22654, name=qtp1064931382-22654, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590)         at java.lang.Thread.run(Thread.java:745)   41) Thread[id=21515, name=coreZkRegister-4379-thread-10-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   42) Thread[id=20707, name=searcherExecutor-4900-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   43) Thread[id=20685, name=searcherExecutor-4876-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   44) Thread[id=21888, name=coreZkRegister-4297-thread-38-processing-n:127.0.0.1:47648_ x:awholynewstresscollection_collection4_0_shard1_replica3 s:shard1 c:awholynewstresscollection_collection4_0 r:core_node21, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1071)         at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1041)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:997)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   45) Thread[id=21508, name=coreZkRegister-4379-thread-7-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   46) Thread[id=19686, name=qtp863314704-19686-selector-ServerConnectorManager@6fb78d43/1, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)         at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)         at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)         at java.lang.Thread.run(Thread.java:745)   47) Thread[id=19808, name=Scheduler-1488862227, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   48) Thread[id=22213, name=coreZkRegister-4379-thread-71-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   49) Thread[id=20937, name=searcherExecutor-5129-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   50) Thread[id=22518, name=RecoveryThread-awholynewstresscollection_collection0_0_shard2_replica3, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:519)         at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:225)   51) Thread[id=22305, name=coreZkRegister-4308-thread-49-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:shard2 c:awholynewstresscollection_collection5_0 r:core_node25, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   52) Thread[id=21542, name=coreZkRegister-4379-thread-24-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   53) Thread[id=21562, name=coreZkRegister-4275-thread-14, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   54) Thread[id=19635, name=SessionTracker, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Object.wait(Native Method)         at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)   55) Thread[id=20481, name=searcherExecutor-4696-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   56) Thread[id=21815, name=coreZkRegister-4379-thread-48-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   57) Thread[id=21721, name=coreZkRegister-4308-thread-10-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:shard2 c:awholynewstresscollection_collection5_0 r:core_node25, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1071)         at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1041)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:997)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   58) Thread[id=20561, name=searcherExecutor-4832-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   59) Thread[id=20635, name=searcherExecutor-4842-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   60) Thread[id=20710, name=searcherExecutor-4934-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   61) Thread[id=19765, name=searcherExecutor-4303-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   62) Thread[id=21817, name=coreZkRegister-4379-thread-50-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   63) Thread[id=20942, name=searcherExecutor-5127-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   64) Thread[id=21389, name=zkCallback-1120-thread-8-processing-n:127.0.0.1:47648_-SendThread(127.0.0.1:36328), state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)   65) Thread[id=22174, name=coreZkRegister-4379-thread-53-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1071)         at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1041)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:997)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   66) Thread[id=21923, name=coreZkRegister-4297-thread-55-processing-n:127.0.0.1:47648_ x:awholynewstresscollection_collection4_0_shard1_replica3 s:shard1 c:awholynewstresscollection_collection4_0 r:core_node21, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1071)         at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1041)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:997)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   67) Thread[id=22457, name=RecoveryThread-awholynewstresscollection_collection4_0_shard3_replica2, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:519)         at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:225)   68) Thread[id=19694, name=org.eclipse.jetty.server.session.HashSessionManager@543280feTimer, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   69) Thread[id=20541, name=searcherExecutor-4779-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   70) Thread[id=20619, name=searcherExecutor-4848-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   71) Thread[id=20538, name=searcherExecutor-4774-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   72) Thread[id=21864, name=coreZkRegister-4297-thread-30-processing-n:127.0.0.1:47648_ x:awholynewstresscollection_collection4_0_shard1_replica3 s:shard1 c:awholynewstresscollection_collection4_0 r:core_node21, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1071)         at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1041)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:997)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   73) Thread[id=20425, name=searcherExecutor-4613-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   74) Thread[id=21692, name=coreZkRegister-4308-thread-2-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:shard2 c:awholynewstresscollection_collection5_0 r:core_node25, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1071)         at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1041)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1024)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   75) Thread[id=22973, name=zkCallback-1120-thread-66, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   76) Thread[id=19885, name=qtp1498626839-19885-selector-ServerConnectorManager@20bdaaac/1, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)         at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)         at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)         at java.lang.Thread.run(Thread.java:745)   77) Thread[id=20003, name=searcherExecutor-4442-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   78) Thread[id=20542, name=searcherExecutor-4785-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   79) Thread[id=19914, name=searcherExecutor-4385-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   80) Thread[id=20010, name=searcherExecutor-4453-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   81) Thread[id=20628, name=searcherExecutor-4860-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   82) Thread[id=22773, name=qtp863314704-22773, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590)         at java.lang.Thread.run(Thread.java:745)   83) Thread[id=22178, name=coreZkRegister-4379-thread-57-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   84) Thread[id=19634, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:178)         at java.lang.Thread.run(Thread.java:745)   85) Thread[id=19772, name=qtp2028463532-19772-selector-ServerConnectorManager@29d21925/0, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:79)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.SelectorManager$ManagedSelector.select(SelectorManager.java:600)         at org.eclipse.jetty.io.SelectorManager$ManagedSelector.run(SelectorManager.java:549)         at org.eclipse.jetty.util.thread.NonBlockingThread.run(NonBlockingThread.java:52)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)         at java.lang.Thread.run(Thread.java:745)   86) Thread[id=19774, name=qtp2028463532-19774-acceptor-0@376b39eb-ServerConnector@41d2a1fb{HTTP/1.1}{127.0.0.1:41155}, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:377)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:500)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:635)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:555)         at java.lang.Thread.run(Thread.java:745)   87) Thread[id=21768, name=coreZkRegister-4379-thread-42-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   88) Thread[id=22486, name=RecoveryThread-awholynewstresscollection_collection4_0_shard5_replica1, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:519)         at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:225)   89) Thread[id=23053, name=zkCallback-1112-thread-64, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   90) Thread[id=20350, name=qtp1498626839-20350, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590)         at java.lang.Thread.run(Thread.java:745)   91) Thread[id=20949, name=searcherExecutor-5128-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   92) Thread[id=19817, name=searcherExecutor-4323-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   93) Thread[id=20608, name=searcherExecutor-4854-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   94) Thread[id=20630, name=searcherExecutor-4857-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   95) Thread[id=20454, name=searcherExecutor-4641-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   96) Thread[id=22214, name=coreZkRegister-4379-thread-72-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   97) Thread[id=20990, name=qtp1064931382-20990, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:389)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:531)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$700(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:590)         at java.lang.Thread.run(Thread.java:745)   98) Thread[id=22179, name=coreZkRegister-4379-thread-58-processing-n:127.0.0.1:59292_ x:awholynewstresscollection_collection4_0_shard7_replica1 s:shard7 c:awholynewstresscollection_collection4_0 r:core_node20, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   99) Thread[id=20492, name=searcherExecutor-4714-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   100) Thread[id=21805, name=coreZkRegister-4308-thread-26-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:shard2 c:awholynewstresscollection_collection5_0 r:core_node25, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:635)         at org.apache.solr.common.cloud.ZkStateReader.getLeaderUrl(ZkStateReader.java:604)         at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1001)         at org.apache.solr.cloud.ZkController.register(ZkController.java:938)         at org.apache.solr.cloud.ZkController$RegisterCoreAsync.call(ZkController.java:233)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   101) Thread[id=22237, name=coreZkRegister-4308-thread-37-processing-n:127.0.0.1:41155_ x:awholynewstresscollection_collection5_0_shard2_replica3 s:sh

[...truncated too long message...]

utor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   243) Thread[id=20596, name=searcherExecutor-4838-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   244) Thread[id=21550, name=coreZkRegister-4275-thread-2, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   245) Thread[id=22238, name=coreZkRegister-4308-thread-38, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   246) Thread[id=20484, name=searcherExecutor-4686-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   247) Thread[id=20532, name=searcherExecutor-4765-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   248) Thread[id=22974, name=zkCallback-1120-thread-67, state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   249) Thread[id=20724, name=searcherExecutor-4899-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   250) Thread[id=20939, name=searcherExecutor-5120-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   251) Thread[id=22698, name=zkCallback-1125-thread-8-processing-n:127.0.0.1:41155_-SendThread(127.0.0.1:36328), state=TIMED_WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([358EC306AB0C651]:0)
   [junit4] Completed [420/552] on J0 in 1725.93s, 1 test, 3 errors <<< FAILURES!

[...truncated 412 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/build.xml:782: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/build.xml:719: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/build.xml:233: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/solr/common-build.xml:516: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/lucene/common-build.xml:1432: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-trunk/lucene/common-build.xml:987: There were test failures: 552 suites (8 ignored), 2170 tests, 2 suite-level errors, 2 errors, 1 failure, 408 ignored (17 assumptions) [seed: 358EC306AB0C651]

Total time: 190 minutes 42 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
No prior successful build to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



Mime
View raw message