lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Policeman Jenkins Server <jenk...@thetaphi.de>
Subject [JENKINS-EA] Lucene-Solr-7.x-Linux (64bit/jdk-10-ea+32) - Build # 1054 - Still Unstable!
Date Tue, 26 Dec 2017 18:42:47 GMT
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Linux/1054/
Java: 64bit/jdk-10-ea+32 -XX:+UseCompressedOops -XX:+UseSerialGC

5 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.BasicZkTest

Error Message:
SolrCore.getOpenCount()==2

Stack Trace:
java.lang.RuntimeException: SolrCore.getOpenCount()==2
	at __randomizedtesting.SeedInfo.seed([64BE0DB95B558480]:0)
	at org.apache.solr.util.TestHarness.close(TestHarness.java:379)
	at org.apache.solr.SolrTestCaseJ4.deleteCore(SolrTestCaseJ4.java:792)
	at org.apache.solr.cloud.AbstractZkTestCase.azt_afterClass(AbstractZkTestCase.java:147)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.base/java.lang.Thread.run(Thread.java:844)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.BasicZkTest

Error Message:
SolrCore.getOpenCount()==2

Stack Trace:
java.lang.RuntimeException: SolrCore.getOpenCount()==2
	at __randomizedtesting.SeedInfo.seed([64BE0DB95B558480]:0)
	at org.apache.solr.util.TestHarness.close(TestHarness.java:379)
	at org.apache.solr.SolrTestCaseJ4.deleteCore(SolrTestCaseJ4.java:792)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:288)
	at jdk.internal.reflect.GeneratedMethodAccessor39.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.base/java.lang.Thread.run(Thread.java:844)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.BasicZkTest

Error Message:
28 threads leaked from SUITE scope at org.apache.solr.cloud.BasicZkTest:     1) Thread[id=16160, name=SUITE-BasicZkTest-seed#[64BE0DB95B558480]-worker-SendThread(127.0.0.1:46239), state=RUNNABLE, group=TGRP-BasicZkTest]         at java.base@10-ea/sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at java.base@10-ea/sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:265)         at java.base@10-ea/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)         at java.base@10-ea/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:89)         at java.base@10-ea/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:100)         at app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)         at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)    2) Thread[id=16180, name=zkCallback-2390-thread-5, state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)         at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)    3) Thread[id=16184, name=SessionTracker, state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/java.lang.Object.wait(Native Method)         at app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)    4) Thread[id=16168, name=ScheduledTrigger-5284-thread-1, state=WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)         at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)         at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)    5) Thread[id=16177, name=coreZkRegister-5279-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)         at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)    6) Thread[id=16156, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)    7) Thread[id=16178, name=zkCallback-2390-thread-3, state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)         at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)    8) Thread[id=16164, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)    9) Thread[id=16179, name=zkCallback-2390-thread-4, state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)         at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)   10) Thread[id=16161, name=SUITE-BasicZkTest-seed#[64BE0DB95B558480]-worker-EventThread, state=WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)         at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)         at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)   11) Thread[id=16181, name=ScheduledTrigger-5284-thread-4, state=WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)         at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)         at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)   12) Thread[id=16162, name=zkConnectionManagerCallback-2391-thread-1, state=WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)         at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)   13) Thread[id=16183, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:46239, state=RUNNABLE, group=TGRP-BasicZkTest]         at java.base@10-ea/sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at java.base@10-ea/sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:265)         at java.base@10-ea/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)         at java.base@10-ea/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:89)         at java.base@10-ea/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:100)         at app//org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)   14) Thread[id=16174, name=ScheduledTrigger-5284-thread-3, state=WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)         at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)         at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)   15) Thread[id=16165, name=OverseerCollectionConfigSetProcessor-99241951144050691-88.99.242.108:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)         at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2116)         at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:463)         at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235)         at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:228)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)   16) Thread[id=16169, name=ScheduledTrigger-5284-thread-2, state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)         at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2116)         at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)         at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)   17) Thread[id=16186, name=ProcessThread(sid:0 cport:46239):, state=WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)         at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)         at app//org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)   18) Thread[id=16182, name=Thread-6525, state=WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/java.lang.Object.wait(Native Method)         at java.base@10-ea/java.lang.Thread.join(Thread.java:1353)         at java.base@10-ea/java.lang.Thread.join(Thread.java:1427)         at app//org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)         at app//org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:310)         at app//org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:491)   19) Thread[id=16166, name=OverseerAutoScalingTriggerThread-99241951144050691-88.99.242.108:8983_-n_0000000000, state=WAITING, group=Overseer autoscaling triggers]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)         at app//org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:166)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)   20) Thread[id=16187, name=zkCallback-2390-thread-6, state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)         at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)   21) Thread[id=16167, name=zkCallback-2390-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)         at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)   22) Thread[id=16176, name=searcherExecutor-5289-thread-1, state=WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)         at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)   23) Thread[id=16188, name=zkCallback-2390-thread-7, state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)         at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)   24) Thread[id=16163, name=OverseerStateUpdate-99241951144050691-88.99.242.108:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)         at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2116)         at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:463)         at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:203)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)   25) Thread[id=16171, name=Thread-6524, state=WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/java.lang.Object.wait(Native Method)         at java.base@10-ea/java.lang.Object.wait(Object.java:328)         at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:1719)   26) Thread[id=16173, name=zkCallback-2390-thread-2, state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)         at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)   27) Thread[id=16155, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/java.lang.Thread.sleep(Native Method)         at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)   28) Thread[id=16172, name=watches-2392-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)         at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 28 threads leaked from SUITE scope at org.apache.solr.cloud.BasicZkTest: 
   1) Thread[id=16160, name=SUITE-BasicZkTest-seed#[64BE0DB95B558480]-worker-SendThread(127.0.0.1:46239), state=RUNNABLE, group=TGRP-BasicZkTest]
        at java.base@10-ea/sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at java.base@10-ea/sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:265)
        at java.base@10-ea/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
        at java.base@10-ea/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:89)
        at java.base@10-ea/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:100)
        at app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
   2) Thread[id=16180, name=zkCallback-2390-thread-5, state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
   3) Thread[id=16184, name=SessionTracker, state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/java.lang.Object.wait(Native Method)
        at app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146)
   4) Thread[id=16168, name=ScheduledTrigger-5284-thread-1, state=WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)
        at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
        at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
   5) Thread[id=16177, name=coreZkRegister-5279-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
   6) Thread[id=16156, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
   7) Thread[id=16178, name=zkCallback-2390-thread-3, state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
   8) Thread[id=16164, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
   9) Thread[id=16179, name=zkCallback-2390-thread-4, state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
  10) Thread[id=16161, name=SUITE-BasicZkTest-seed#[64BE0DB95B558480]-worker-EventThread, state=WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)
        at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
  11) Thread[id=16181, name=ScheduledTrigger-5284-thread-4, state=WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)
        at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
        at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
  12) Thread[id=16162, name=zkConnectionManagerCallback-2391-thread-1, state=WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)
        at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
  13) Thread[id=16183, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:46239, state=RUNNABLE, group=TGRP-BasicZkTest]
        at java.base@10-ea/sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at java.base@10-ea/sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:265)
        at java.base@10-ea/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
        at java.base@10-ea/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:89)
        at java.base@10-ea/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:100)
        at app//org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
  14) Thread[id=16174, name=ScheduledTrigger-5284-thread-3, state=WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)
        at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
        at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
  15) Thread[id=16165, name=OverseerCollectionConfigSetProcessor-99241951144050691-88.99.242.108:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2116)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:463)
        at app//org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235)
        at app//org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:228)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
  16) Thread[id=16169, name=ScheduledTrigger-5284-thread-2, state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2116)
        at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
  17) Thread[id=16186, name=ProcessThread(sid:0 cport:46239):, state=WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)
        at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at app//org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)
  18) Thread[id=16182, name=Thread-6525, state=WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/java.lang.Object.wait(Native Method)
        at java.base@10-ea/java.lang.Thread.join(Thread.java:1353)
        at java.base@10-ea/java.lang.Thread.join(Thread.java:1427)
        at app//org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297)
        at app//org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:310)
        at app//org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:491)
  19) Thread[id=16166, name=OverseerAutoScalingTriggerThread-99241951144050691-88.99.242.108:8983_-n_0000000000, state=WAITING, group=Overseer autoscaling triggers]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)
        at app//org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:166)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
  20) Thread[id=16187, name=zkCallback-2390-thread-6, state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
  21) Thread[id=16167, name=zkCallback-2390-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
  22) Thread[id=16176, name=searcherExecutor-5289-thread-1, state=WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)
        at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
  23) Thread[id=16188, name=zkCallback-2390-thread-7, state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
  24) Thread[id=16163, name=OverseerStateUpdate-99241951144050691-88.99.242.108:8983_-n_0000000000, state=TIMED_WAITING, group=Overseer state updater.]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2116)
        at app//org.apache.solr.cloud.ZkDistributedQueue.peekElements(ZkDistributedQueue.java:463)
        at app//org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:203)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
  25) Thread[id=16171, name=Thread-6524, state=WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/java.lang.Object.wait(Native Method)
        at java.base@10-ea/java.lang.Object.wait(Object.java:328)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:1719)
  26) Thread[id=16173, name=zkCallback-2390-thread-2, state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
  27) Thread[id=16155, name=Connection evictor, state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/java.lang.Thread.sleep(Native Method)
        at app//org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
  28) Thread[id=16172, name=watches-2392-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
	at __randomizedtesting.SeedInfo.seed([64BE0DB95B558480]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.BasicZkTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=16160, name=SUITE-BasicZkTest-seed#[64BE0DB95B558480]-worker-SendThread(127.0.0.1:46239), state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/java.lang.Thread.sleep(Native Method)         at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)    2) Thread[id=16162, name=zkConnectionManagerCallback-2391-thread-1, state=WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)         at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)    3) Thread[id=16180, name=zkCallback-2390-thread-5, state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)         at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)    4) Thread[id=16174, name=ScheduledTrigger-5284-thread-3, state=WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)         at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)         at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)    5) Thread[id=16168, name=ScheduledTrigger-5284-thread-1, state=WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)         at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)         at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)    6) Thread[id=16177, name=coreZkRegister-5279-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)         at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)    7) Thread[id=16169, name=ScheduledTrigger-5284-thread-2, state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)         at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2116)         at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)         at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)    8) Thread[id=16178, name=zkCallback-2390-thread-3, state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)         at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)    9) Thread[id=16187, name=zkCallback-2390-thread-6, state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)         at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)   10) Thread[id=16167, name=zkCallback-2390-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)         at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)   11) Thread[id=16176, name=searcherExecutor-5289-thread-1, state=WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)         at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)   12) Thread[id=16188, name=zkCallback-2390-thread-7, state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)         at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)   13) Thread[id=16171, name=Thread-6524, state=WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/java.lang.Object.wait(Native Method)         at java.base@10-ea/java.lang.Object.wait(Object.java:328)         at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:1719)   14) Thread[id=16173, name=zkCallback-2390-thread-2, state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)         at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)   15) Thread[id=16179, name=zkCallback-2390-thread-4, state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)         at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)   16) Thread[id=16172, name=watches-2392-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)         at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)   17) Thread[id=16181, name=ScheduledTrigger-5284-thread-4, state=WAITING, group=TGRP-BasicZkTest]         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)         at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)         at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)         at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)         at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=16160, name=SUITE-BasicZkTest-seed#[64BE0DB95B558480]-worker-SendThread(127.0.0.1:46239), state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/java.lang.Thread.sleep(Native Method)
        at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
   2) Thread[id=16162, name=zkConnectionManagerCallback-2391-thread-1, state=WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)
        at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
   3) Thread[id=16180, name=zkCallback-2390-thread-5, state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
   4) Thread[id=16174, name=ScheduledTrigger-5284-thread-3, state=WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)
        at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
        at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
   5) Thread[id=16168, name=ScheduledTrigger-5284-thread-1, state=WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)
        at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
        at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
   6) Thread[id=16177, name=coreZkRegister-5279-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
   7) Thread[id=16169, name=ScheduledTrigger-5284-thread-2, state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2116)
        at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182)
        at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
   8) Thread[id=16178, name=zkCallback-2390-thread-3, state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
   9) Thread[id=16187, name=zkCallback-2390-thread-6, state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
  10) Thread[id=16167, name=zkCallback-2390-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
  11) Thread[id=16176, name=searcherExecutor-5289-thread-1, state=WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)
        at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
  12) Thread[id=16188, name=zkCallback-2390-thread-7, state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
  13) Thread[id=16171, name=Thread-6524, state=WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/java.lang.Object.wait(Native Method)
        at java.base@10-ea/java.lang.Object.wait(Object.java:328)
        at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:1719)
  14) Thread[id=16173, name=zkCallback-2390-thread-2, state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
  15) Thread[id=16179, name=zkCallback-2390-thread-4, state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
  16) Thread[id=16172, name=watches-2392-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
        at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
  17) Thread[id=16181, name=ScheduledTrigger-5284-thread-4, state=WAITING, group=TGRP-BasicZkTest]
        at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
        at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
        at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)
        at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
        at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
        at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
	at __randomizedtesting.SeedInfo.seed([64BE0DB95B558480]:0)


FAILED:  org.apache.solr.cloud.MoveReplicaHDFSTest.testNormalFailedMove

Error Message:
Error from server at http://127.0.0.1:37913/solr: create the collection time out:180s

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:37913/solr: create the collection time out:180s
	at __randomizedtesting.SeedInfo.seed([64BE0DB95B558480:43B512F177171A99]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.MoveReplicaTest.testFailedMove(MoveReplicaTest.java:261)
	at org.apache.solr.cloud.MoveReplicaHDFSTest.testNormalFailedMove(MoveReplicaHDFSTest.java:67)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.lang.reflect.Method.invoke(Method.java:564)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.base/java.lang.Thread.run(Thread.java:844)




Build Log:
[...truncated 12511 lines...]
   [junit4] Suite: org.apache.solr.cloud.MoveReplicaHDFSTest
   [junit4]   2> 735000 INFO  (SUITE-MoveReplicaHDFSTest-seed#[64BE0DB95B558480]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.MoveReplicaHDFSTest_64BE0DB95B558480-001/init-core-data-001
   [junit4]   2> 735000 WARN  (SUITE-MoveReplicaHDFSTest-seed#[64BE0DB95B558480]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=128 numCloses=128
   [junit4]   2> 735000 INFO  (SUITE-MoveReplicaHDFSTest-seed#[64BE0DB95B558480]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 735001 INFO  (SUITE-MoveReplicaHDFSTest-seed#[64BE0DB95B558480]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason="", ssl=0.0/0.0, value=0.0/0.0, clientAuth=0.0/0.0)
   [junit4]   2> 735001 INFO  (SUITE-MoveReplicaHDFSTest-seed#[64BE0DB95B558480]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.MoveReplicaHDFSTest_64BE0DB95B558480-001/tempDir-001
   [junit4]   2> 735001 INFO  (SUITE-MoveReplicaHDFSTest-seed#[64BE0DB95B558480]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 735005 INFO  (Thread-2061) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 735005 INFO  (Thread-2061) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 735008 ERROR (Thread-2061) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 735105 INFO  (SUITE-MoveReplicaHDFSTest-seed#[64BE0DB95B558480]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:43257
   [junit4]   2> 735106 INFO  (zkConnectionManagerCallback-7409-thread-1) [    ] o.a.s.c.c.ConnectionManager zkClient has connected
   [junit4]   2> 735108 INFO  (jetty-launcher-7406-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 735108 INFO  (jetty-launcher-7406-thread-3) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 735108 INFO  (jetty-launcher-7406-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 735108 INFO  (jetty-launcher-7406-thread-4) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 735109 INFO  (jetty-launcher-7406-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2c42692{/solr,null,AVAILABLE}
   [junit4]   2> 735110 INFO  (jetty-launcher-7406-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1107d261{/solr,null,AVAILABLE}
   [junit4]   2> 735110 INFO  (jetty-launcher-7406-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@390723a4{/solr,null,AVAILABLE}
   [junit4]   2> 735110 INFO  (jetty-launcher-7406-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4bc23d92{/solr,null,AVAILABLE}
   [junit4]   2> 735111 INFO  (jetty-launcher-7406-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@649ff077{HTTP/1.1,[http/1.1]}{127.0.0.1:41083}
   [junit4]   2> 735111 INFO  (jetty-launcher-7406-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@cc420f1{HTTP/1.1,[http/1.1]}{127.0.0.1:43661}
   [junit4]   2> 735111 INFO  (jetty-launcher-7406-thread-1) [    ] o.e.j.s.Server Started @738095ms
   [junit4]   2> 735111 INFO  (jetty-launcher-7406-thread-2) [    ] o.e.j.s.Server Started @738095ms
   [junit4]   2> 735111 INFO  (jetty-launcher-7406-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=41083}
   [junit4]   2> 7

[...truncated too long message...]

ecutor.runWorker(ThreadPoolExecutor.java:1121)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
   [junit4]    >         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
   [junit4]    >    9) Thread[id=16187, name=zkCallback-2390-thread-6, state=TIMED_WAITING, group=TGRP-BasicZkTest]
   [junit4]    >         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
   [junit4]    >         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
   [junit4]    >   10) Thread[id=16167, name=zkCallback-2390-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest]
   [junit4]    >         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
   [junit4]    >         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
   [junit4]    >   11) Thread[id=16176, name=searcherExecutor-5289-thread-1, state=WAITING, group=TGRP-BasicZkTest]
   [junit4]    >         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
   [junit4]    >         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
   [junit4]    >   12) Thread[id=16188, name=zkCallback-2390-thread-7, state=TIMED_WAITING, group=TGRP-BasicZkTest]
   [junit4]    >         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
   [junit4]    >         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
   [junit4]    >   13) Thread[id=16171, name=Thread-6524, state=WAITING, group=TGRP-BasicZkTest]
   [junit4]    >         at java.base@10-ea/java.lang.Object.wait(Native Method)
   [junit4]    >         at java.base@10-ea/java.lang.Object.wait(Object.java:328)
   [junit4]    >         at app//org.apache.solr.core.CloserThread.run(CoreContainer.java:1719)
   [junit4]    >   14) Thread[id=16173, name=zkCallback-2390-thread-2, state=TIMED_WAITING, group=TGRP-BasicZkTest]
   [junit4]    >         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
   [junit4]    >         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
   [junit4]    >   15) Thread[id=16179, name=zkCallback-2390-thread-4, state=TIMED_WAITING, group=TGRP-BasicZkTest]
   [junit4]    >         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
   [junit4]    >         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
   [junit4]    >   16) Thread[id=16172, name=watches-2392-thread-1, state=TIMED_WAITING, group=TGRP-BasicZkTest]
   [junit4]    >         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:234)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:462)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:937)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1060)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
   [junit4]    >         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
   [junit4]    >   17) Thread[id=16181, name=ScheduledTrigger-5284-thread-4, state=WAITING, group=TGRP-BasicZkTest]
   [junit4]    >         at java.base@10-ea/jdk.internal.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.locks.LockSupport.park(LockSupport.java:194)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2074)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1121)
   [junit4]    >         at java.base@10-ea/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
   [junit4]    >         at java.base@10-ea/java.lang.Thread.run(Thread.java:844)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([64BE0DB95B558480]:0)
   [junit4] Completed [381/764 (2!)] on J0 in 31.36s, 1 test, 4 errors <<< FAILURES!

[...truncated 42069 lines...]

Mime
View raw message