lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Policeman Jenkins Server <jenk...@thetaphi.de>
Subject [JENKINS] Lucene-Solr-7.x-MacOSX (64bit/jdk1.8.0) - Build # 117 - Still Unstable!
Date Thu, 17 Aug 2017 22:25:59 GMT
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-MacOSX/117/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC

5 tests failed.
FAILED:  org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForImplicitRouter

Error Message:
Error from server at http://127.0.0.1:51593/solr/withShardField: Async exception during distributed update: Error from server at http://127.0.0.1:51595/solr/withShardField_b_replica_n18: Server Error    request: http://127.0.0.1:51595/solr/withShardField_b_replica_n18/update?update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A51593%2Fsolr%2FwithShardField_a_replica_n6%2F&wt=javabin&version=2 Remote error message: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_EC96E5085E5D0EAC-001/tempDir-001/node2/withShardField_b_replica_n18/data/tlog/tlog.0000000000000000000 (Too many open files)

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:51593/solr/withShardField: Async exception during distributed update: Error from server at http://127.0.0.1:51595/solr/withShardField_b_replica_n18: Server Error



request: http://127.0.0.1:51595/solr/withShardField_b_replica_n18/update?update.distrib=TOLEADER&distrib.from=http%3A%2F%2F127.0.0.1%3A51593%2Fsolr%2FwithShardField_a_replica_n6%2F&wt=javabin&version=2
Remote error message: java.io.FileNotFoundException: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_EC96E5085E5D0EAC-001/tempDir-001/node2/withShardField_b_replica_n18/data/tlog/tlog.0000000000000000000 (Too many open files)
	at __randomizedtesting.SeedInfo.seed([EC96E5085E5D0EAC:B9C60D9AF2A4C15C]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1121)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
	at org.apache.solr.cloud.CustomCollectionTest.testRouteFieldForImplicitRouter(CustomCollectionTest.java:141)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.CustomCollectionTest.testCreateShardRepFactor

Error Message:
Error from server at http://127.0.0.1:51595/solr: Could not find collection : testCreateShardRepFactor

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:51595/solr: Could not find collection : testCreateShardRepFactor
	at __randomizedtesting.SeedInfo.seed([EC96E5085E5D0EAC:18786F8C46091B6C]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:627)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:253)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:242)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1121)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:862)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:793)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
	at org.apache.solr.cloud.CustomCollectionTest.testCreateShardRepFactor(CustomCollectionTest.java:190)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.DistribCursorPagingTest.test

Error Message:


Stack Trace:
java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([EC96E5085E5D0EAC:64C2DAD2F0A16354]:0)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.getNodeName(JettySolrRunner.java:345)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:416)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:334)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:991)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DistribCursorPagingTest

Error Message:
18 threads leaked from SUITE scope at org.apache.solr.cloud.DistribCursorPagingTest:     1) Thread[id=19484, name=zkCallback-2472-thread-4, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=19469, name=Connection evictor, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=19461, name=qtp904986691-19461, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=19462, name=qtp904986691-19462, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=19464, name=qtp904986691-19464, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=19476, name=zkCallback-2472-thread-1-processing-n:127.0.0.1:51824_ryb%2Fy, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:389)         at org.apache.solr.cloud.Overseer.createOverseerNode(Overseer.java:758)         at org.apache.solr.cloud.Overseer.getCollectionQueue(Overseer.java:707)         at org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.<init>(OverseerCollectionConfigSetProcessor.java:48)         at org.apache.solr.cloud.Overseer.start(Overseer.java:521)         at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:784)         at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)         at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)         at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)         at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)         at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269)         at org.apache.solr.common.cloud.SolrZkClient$3$$Lambda$250/979262338.run(Unknown Source)         at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)         at java.util.concurrent.FutureTask.run(FutureTask.java:266)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)         at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$117/1071763235.run(Unknown Source)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=19485, name=zkCallback-2472-thread-5, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=19459, name=qtp904986691-19459, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)    9) Thread[id=19475, name=TEST-DistribCursorPagingTest.test-seed#[EC96E5085E5D0EAC]-EventThread, state=WAITING, group=TGRP-DistribCursorPagingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)   10) Thread[id=19458, name=qtp904986691-19458-acceptor-0@41eb8b7f-ServerConnector@77a9c0d2{HTTP/1.1,[http/1.1]}{127.0.0.1:51824}, state=RUNNABLE, group=TGRP-DistribCursorPagingTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   11) Thread[id=19465, name=org.eclipse.jetty.server.session.HashSessionManager@2eb2f1fcTimer, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   12) Thread[id=19457, name=qtp904986691-19457, state=RUNNABLE, group=TGRP-DistribCursorPagingTest]         at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)         at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)   13) Thread[id=19474, name=TEST-DistribCursorPagingTest.test-seed#[EC96E5085E5D0EAC]-SendThread(127.0.0.1:51812), state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)   14) Thread[id=19480, name=zkCallback-2472-thread-2, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   15) Thread[id=19460, name=qtp904986691-19460, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   16) Thread[id=19483, name=zkCallback-2472-thread-3, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)   17) Thread[id=19463, name=qtp904986691-19463, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)         at java.lang.Thread.run(Thread.java:748)   18) Thread[id=19470, name=Connection evictor, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 18 threads leaked from SUITE scope at org.apache.solr.cloud.DistribCursorPagingTest: 
   1) Thread[id=19484, name=zkCallback-2472-thread-4, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=19469, name=Connection evictor, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=19461, name=qtp904986691-19461, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=19462, name=qtp904986691-19462, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=19464, name=qtp904986691-19464, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=19476, name=zkCallback-2472-thread-1-processing-n:127.0.0.1:51824_ryb%2Fy, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:389)
        at org.apache.solr.cloud.Overseer.createOverseerNode(Overseer.java:758)
        at org.apache.solr.cloud.Overseer.getCollectionQueue(Overseer.java:707)
        at org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.<init>(OverseerCollectionConfigSetProcessor.java:48)
        at org.apache.solr.cloud.Overseer.start(Overseer.java:521)
        at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:784)
        at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
        at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
        at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)
        at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)
        at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269)
        at org.apache.solr.common.cloud.SolrZkClient$3$$Lambda$250/979262338.run(Unknown Source)
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
        at java.util.concurrent.FutureTask.run(FutureTask.java:266)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
        at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$117/1071763235.run(Unknown Source)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=19485, name=zkCallback-2472-thread-5, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=19459, name=qtp904986691-19459, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
   9) Thread[id=19475, name=TEST-DistribCursorPagingTest.test-seed#[EC96E5085E5D0EAC]-EventThread, state=WAITING, group=TGRP-DistribCursorPagingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
  10) Thread[id=19458, name=qtp904986691-19458-acceptor-0@41eb8b7f-ServerConnector@77a9c0d2{HTTP/1.1,[http/1.1]}{127.0.0.1:51824}, state=RUNNABLE, group=TGRP-DistribCursorPagingTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  11) Thread[id=19465, name=org.eclipse.jetty.server.session.HashSessionManager@2eb2f1fcTimer, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  12) Thread[id=19457, name=qtp904986691-19457, state=RUNNABLE, group=TGRP-DistribCursorPagingTest]
        at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
        at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
  13) Thread[id=19474, name=TEST-DistribCursorPagingTest.test-seed#[EC96E5085E5D0EAC]-SendThread(127.0.0.1:51812), state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060)
  14) Thread[id=19480, name=zkCallback-2472-thread-2, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  15) Thread[id=19460, name=qtp904986691-19460, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  16) Thread[id=19483, name=zkCallback-2472-thread-3, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
  17) Thread[id=19463, name=qtp904986691-19463, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
        at java.lang.Thread.run(Thread.java:748)
  18) Thread[id=19470, name=Connection evictor, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([EC96E5085E5D0EAC]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DistribCursorPagingTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=19484, name=zkCallback-2472-thread-4, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=19457, name=qtp904986691-19457, state=RUNNABLE, group=TGRP-DistribCursorPagingTest]         at sun.nio.ch.KQueueArrayWrapper.interrupt(Native Method)         at sun.nio.ch.KQueueArrayWrapper.interrupt(KQueueArrayWrapper.java:203)         at sun.nio.ch.KQueueSelectorImpl.wakeup(KQueueSelectorImpl.java:257)         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:116)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=19474, name=TEST-DistribCursorPagingTest.test-seed#[EC96E5085E5D0EAC]-SendThread(127.0.0.1:51812), state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)    4) Thread[id=19480, name=zkCallback-2472-thread-2, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    5) Thread[id=19476, name=zkCallback-2472-thread-1, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    6) Thread[id=19485, name=zkCallback-2472-thread-5, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    7) Thread[id=19483, name=zkCallback-2472-thread-3, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    8) Thread[id=19465, name=org.eclipse.jetty.server.session.HashSessionManager@2eb2f1fcTimer, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=19484, name=zkCallback-2472-thread-4, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=19457, name=qtp904986691-19457, state=RUNNABLE, group=TGRP-DistribCursorPagingTest]
        at sun.nio.ch.KQueueArrayWrapper.interrupt(Native Method)
        at sun.nio.ch.KQueueArrayWrapper.interrupt(KQueueArrayWrapper.java:203)
        at sun.nio.ch.KQueueSelectorImpl.wakeup(KQueueSelectorImpl.java:257)
        at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
        at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:116)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=19474, name=TEST-DistribCursorPagingTest.test-seed#[EC96E5085E5D0EAC]-SendThread(127.0.0.1:51812), state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
   4) Thread[id=19480, name=zkCallback-2472-thread-2, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   5) Thread[id=19476, name=zkCallback-2472-thread-1, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   6) Thread[id=19485, name=zkCallback-2472-thread-5, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   7) Thread[id=19483, name=zkCallback-2472-thread-3, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   8) Thread[id=19465, name=org.eclipse.jetty.server.session.HashSessionManager@2eb2f1fcTimer, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([EC96E5085E5D0EAC]:0)




Build Log:
[...truncated 12492 lines...]
   [junit4] Suite: org.apache.solr.cloud.CustomCollectionTest
   [junit4]   2> 2461801 INFO  (SUITE-CustomCollectionTest-seed#[EC96E5085E5D0EAC]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_EC96E5085E5D0EAC-001/init-core-data-001
   [junit4]   2> 2461802 INFO  (SUITE-CustomCollectionTest-seed#[EC96E5085E5D0EAC]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 2461803 INFO  (SUITE-CustomCollectionTest-seed#[EC96E5085E5D0EAC]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
   [junit4]   2> 2461804 INFO  (SUITE-CustomCollectionTest-seed#[EC96E5085E5D0EAC]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_EC96E5085E5D0EAC-001/tempDir-001
   [junit4]   2> 2461804 INFO  (SUITE-CustomCollectionTest-seed#[EC96E5085E5D0EAC]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2461804 INFO  (Thread-4031) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2461804 INFO  (Thread-4031) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2461809 ERROR (Thread-4031) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2461906 INFO  (SUITE-CustomCollectionTest-seed#[EC96E5085E5D0EAC]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:51591
   [junit4]   2> 2461949 INFO  (jetty-launcher-2416-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2461949 INFO  (jetty-launcher-2416-thread-3) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2461949 INFO  (jetty-launcher-2416-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2461950 INFO  (jetty-launcher-2416-thread-4) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2461952 INFO  (jetty-launcher-2416-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@109d43ee{/solr,null,AVAILABLE}
   [junit4]   2> 2461953 INFO  (jetty-launcher-2416-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@795799d1{/solr,null,AVAILABLE}
   [junit4]   2> 2461953 INFO  (jetty-launcher-2416-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@20c14bc0{/solr,null,AVAILABLE}
   [junit4]   2> 2461953 INFO  (jetty-launcher-2416-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1c52e74f{/solr,null,AVAILABLE}
   [junit4]   2> 2461954 INFO  (jetty-launcher-2416-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7b112449{HTTP/1.1,[http/1.1]}{127.0.0.1:51594}
   [junit4]   2> 2461954 INFO  (jetty-launcher-2416-thread-4) [    ] o.e.j.s.AbstractConnector Started ServerConnector@5d5320c4{HTTP/1.1,[http/1.1]}{127.0.0.1:51596}
   [junit4]   2> 2461954 INFO  (jetty-launcher-2416-thread-3) [    ] o.e.j.s.Server Started @2465670ms
   [junit4]   2> 2461954 INFO  (jetty-launcher-2416-thread-4) [    ] o.e.j.s.Server Started @2465670ms
   [junit4]   2> 2461954 INFO  (jetty-launcher-2416-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=51594}
   [junit4]   2> 2461954 INFO  (jetty-launcher-2416-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=51596}
   [junit4]   2> 2461954 ERROR (jetty-launcher-2416-thread-3) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2461954 ERROR (jetty-launcher-2416-thread-4) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2461954 INFO  (jetty-launcher-2416-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@4e3c5ee8{HTTP/1.1,[http/1.1]}{127.0.0.1:51595}
   [junit4]   2> 2461954 INFO  (jetty-launcher-2416-thread-2) [    ] o.e.j.s.Server Started @2465670ms
   [junit4]   2> 2461954 INFO  (jetty-launcher-2416-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=51595}
   [junit4]   2> 2461954 INFO  (jetty-launcher-2416-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 2461954 ERROR (jetty-launcher-2416-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2461954 INFO  (jetty-launcher-2416-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2461954 INFO  (jetty-launcher-2416-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2461954 INFO  (jetty-launcher-2416-thread-4) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 2461954 INFO  (jetty-launcher-2416-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-17T22:05:56.669Z
   [junit4]   2> 2461954 INFO  (jetty-launcher-2416-thread-4) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2461954 INFO  (jetty-launcher-2416-thread-4) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2461955 INFO  (jetty-launcher-2416-thread-4) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-17T22:05:56.669Z
   [junit4]   2> 2461955 INFO  (jetty-launcher-2416-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 2461955 INFO  (jetty-launcher-2416-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2461955 INFO  (jetty-launcher-2416-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2461955 INFO  (jetty-launcher-2416-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-17T22:05:56.670Z
   [junit4]   2> 2461958 INFO  (jetty-launcher-2416-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@41404650{HTTP/1.1,[http/1.1]}{127.0.0.1:51593}
   [junit4]   2> 2461959 INFO  (jetty-launcher-2416-thread-1) [    ] o.e.j.s.Server Started @2465675ms
   [junit4]   2> 2461959 INFO  (jetty-launcher-2416-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=51593}
   [junit4]   2> 2461959 ERROR (jetty-launcher-2416-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2461959 INFO  (jetty-launcher-2416-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.1.0
   [junit4]   2> 2461959 INFO  (jetty-launcher-2416-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2461959 INFO  (jetty-launcher-2416-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 2461959 INFO  (jetty-launcher-2416-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-17T22:05:56.674Z
   [junit4]   2> 2461961 INFO  (jetty-launcher-2416-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2461962 INFO  (jetty-launcher-2416-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2461962 INFO  (jetty-launcher-2416-thread-4) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2461962 INFO  (jetty-launcher-2416-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2461968 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15df23b43aa0002, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2461970 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15df23b43aa0001, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2461971 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15df23b43aa0003, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2461972 INFO  (jetty-launcher-2416-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:51591/solr
   [junit4]   2> 2461972 INFO  (jetty-launcher-2416-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:51591/solr
   [junit4]   2> 2461973 INFO  (jetty-launcher-2416-thread-4) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:51591/solr
   [junit4]   2> 2461973 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15df23b43aa0004, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2461975 INFO  (jetty-launcher-2416-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:51591/solr
   [junit4]   2> 2461976 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15df23b43aa0005, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 2462149 INFO  (jetty-launcher-2416-thread-4) [n:127.0.0.1:51596_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2462150 INFO  (jetty-launcher-2416-thread-1) [n:127.0.0.1:51593_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2462150 INFO  (jetty-launcher-2416-thread-3) [n:127.0.0.1:51594_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2462152 INFO  (jetty-launcher-2416-thread-2) [n:127.0.0.1:51595_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2462153 INFO  (jetty-launcher-2416-thread-4) [n:127.0.0.1:51596_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:51596_solr
   [junit4]   2> 2462154 INFO  (jetty-launcher-2416-thread-1) [n:127.0.0.1:51593_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51593_solr
   [junit4]   2> 2462154 INFO  (jetty-launcher-2416-thread-3) [n:127.0.0.1:51594_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51594_solr
   [junit4]   2> 2462154 INFO  (jetty-launcher-2416-thread-4) [n:127.0.0.1:51596_solr    ] o.a.s.c.Overseer Overseer (id=98501103224225803-127.0.0.1:51596_solr-n_0000000000) starting
   [junit4]   2> 2462155 INFO  (jetty-launcher-2416-thread-2) [n:127.0.0.1:51595_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51595_solr
   [junit4]   2> 2462158 INFO  (zkCallback-2435-thread-1-processing-n:127.0.0.1:51594_solr) [n:127.0.0.1:51594_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2462159 INFO  (zkCallback-2438-thread-1-processing-n:127.0.0.1:51593_solr) [n:127.0.0.1:51593_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2462161 INFO  (zkCallback-2436-thread-1-processing-n:127.0.0.1:51595_solr) [n:127.0.0.1:51595_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2462163 INFO  (zkCallback-2437-thread-1-processing-n:127.0.0.1:51596_solr) [n:127.0.0.1:51596_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2462167 INFO  (zkCallback-2436-thread-1-processing-n:127.0.0.1:51595_solr) [n:127.0.0.1:51595_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2462168 INFO  (zkCallback-2435-thread-1-processing-n:127.0.0.1:51594_solr) [n:127.0.0.1:51594_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2462168 INFO  (zkCallback-2437-thread-1-processing-n:127.0.0.1:51596_solr) [n:127.0.0.1:51596_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2462168 INFO  (zkCallback-2438-thread-1-processing-n:127.0.0.1:51593_solr) [n:127.0.0.1:51593_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 2462218 INFO  (jetty-launcher-2416-thread-4) [n:127.0.0.1:51596_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51596_solr
   [junit4]   2> 2462221 INFO  (zkCallback-2438-thread-1-processing-n:127.0.0.1:51593_solr) [n:127.0.0.1:51593_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2462222 INFO  (zkCallback-2436-thread-1-processing-n:127.0.0.1:51595_solr) [n:127.0.0.1:51595_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2462222 INFO  (zkCallback-2435-thread-1-processing-n:127.0.0.1:51594_solr) [n:127.0.0.1:51594_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2462222 INFO  (zkCallback-2437-thread-1-processing-n:127.0.0.1:51596_solr) [n:127.0.0.1:51596_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 2462263 INFO  (jetty-launcher-2416-thread-1) [n:127.0.0.1:51593_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51593.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@723c747c
   [junit4]   2> 2462270 INFO  (jetty-launcher-2416-thread-1) [n:127.0.0.1:51593_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51593.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@723c747c
   [junit4]   2> 2462270 INFO  (jetty-launcher-2416-thread-1) [n:127.0.0.1:51593_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51593.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@723c747c
   [junit4]   2> 2462285 INFO  (jetty-launcher-2416-thread-1) [n:127.0.0.1:51593_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_EC96E5085E5D0EAC-001/tempDir-001/node1/.
   [junit4]   2> 2462299 INFO  (jetty-launcher-2416-thread-3) [n:127.0.0.1:51594_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51594.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@723c747c
   [junit4]   2> 2462305 INFO  (jetty-launcher-2416-thread-3) [n:127.0.0.1:51594_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51594.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@723c747c
   [junit4]   2> 2462305 INFO  (jetty-launcher-2416-thread-3) [n:127.0.0.1:51594_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51594.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@723c747c
   [junit4]   2> 2462306 INFO  (jetty-launcher-2416-thread-3) [n:127.0.0.1:51594_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_EC96E5085E5D0EAC-001/tempDir-001/node3/.
   [junit4]   2> 2462379 INFO  (jetty-launcher-2416-thread-4) [n:127.0.0.1:51596_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51596.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@723c747c
   [junit4]   2> 2462391 INFO  (jetty-launcher-2416-thread-2) [n:127.0.0.1:51595_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51595.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@723c747c
   [junit4]   2> 2462436 INFO  (jetty-launcher-2416-thread-4) [n:127.0.0.1:51596_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51596.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@723c747c
   [junit4]   2> 2462437 INFO  (jetty-launcher-2416-thread-4) [n:127.0.0.1:51596_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51596.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@723c747c
   [junit4]   2> 2462441 INFO  (jetty-launcher-2416-thread-4) [n:127.0.0.1:51596_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_EC96E5085E5D0EAC-001/tempDir-001/node4/.
   [junit4]   2> 2462443 INFO  (jetty-launcher-2416-thread-2) [n:127.0.0.1:51595_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51595.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@723c747c
   [junit4]   2> 2462443 INFO  (jetty-launcher-2416-thread-2) [n:127.0.0.1:51595_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51595.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@723c747c
   [junit4]   2> 2462444 INFO  (jetty-launcher-2416-thread-2) [n:127.0.0.1:51595_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.CustomCollectionTest_EC96E5085E5D0EAC-001/tempDir-001/node2/.
   [junit4]   2> 2462499 INFO  (SUITE-CustomCollectionTest-seed#[EC96E5085E5D0EAC]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 2462501 INFO  (SUITE-CustomCollectionTest-seed#[EC96E5085E5D0EAC]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:51591/solr ready
   [junit4]   2> 2462642 INFO  (TEST-CustomCollectionTest.testRouteFieldForHashRouter-seed#[EC96E5085E5D0EAC]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testRouteFieldForHashRouter
   [junit4]   2> 2462644 INFO  (TEST-CustomCollectionTest.testRouteFieldForHashRouter-seed#[EC96E5085E5D0EAC]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 2462653 INFO  (qtp1846104485-19007) [n:127.0.0.1:51595_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&maxShardsPerNode=3&router.field=shard_s&name=routeFieldColl&nrtReplicas=2&action=CREATE&numShards=4&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2462656 INFO  (OverseerThreadFactory-8112-thread-1-processing-n:127.0.0.1:51596_solr) [n:127.0.0.1:51596_solr    ] o.a.s.c.CreateCollectionCmd Create collection routeFieldColl
   [junit4]   2> 2462783 INFO  (OverseerStateUpdate-98501103224225803-127.0.0.1:51596_solr-n_0000000000) [n:127.0.0.1:51596_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"routeFieldColl",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"routeFieldColl_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:51593/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 2462789 INFO  (OverseerStateUpdate-98501103224225803-127.0.0.1:51596_solr-n_0000000000) [n:127.0.0.1:51596_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"routeFieldColl",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"routeFieldColl_shard1_replica_n2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:51595/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 2462795 INFO  (OverseerStateUpdate-98501103224225803-127.0.0.1:51596_solr-n_0000000000) [n:127.0.0.1:51596_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"routeFieldColl",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"routeFieldColl_shard2_replica_n4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:51596/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 2462801 INFO  (OverseerStateUpdate-98501103224225803-127.0.0.1:51596_solr-n_0000000000) [n:127.0.0.1:51596_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"routeFieldColl",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"routeFieldColl_shard2_replica_n6",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:51594/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 2462807 INFO  (OverseerStateUpdate-98501103224225803-127.0.0.1:51596_solr-n_0000000000) [n:127.0.0.1:51596_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"routeFieldColl",
   [junit4]   2>   "shard":"shard3",
   [junit4]   2>   "core":"routeFieldColl_shard3_replica_n8",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:51593/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 2462812 INFO  (OverseerStateUpdate-98501103224225803-127.0.0.1:51596_solr-n_0000000000) [n:127.0.0.1:51596_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"routeFieldColl",
   [junit4]   2>   "shard":"shard3",
   [junit4]   2>   "core":"routeFieldColl_shard3_replica_n10",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:51595/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 2462818 INFO  (OverseerStateUpdate-98501103224225803-127.0.0.1:51596_solr-n_0000000000) [n:127.0.0.1:51596_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"routeFieldColl",
   [junit4]   2>   "shard":"shard4",
   [junit4]   2>   "core":"routeFieldColl_shard4_replica_n12",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:51596/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 2462821 INFO  (OverseerStateUpdate-98501103224225803-127.0.0.1:51596_solr-n_0000000000) [n:127.0.0.1:51596_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"routeFieldColl",
   [junit4]   2>   "shard":"shard4",
   [junit4]   2>   "core":"routeFieldColl_shard4_replica_n14",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:51594/solr",
   [junit4]   2>   "type":"NRT"} 
   [junit4]   2> 2463044 INFO  (qtp1846104485-19009) [n:127.0.0.1:51595_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=routeFieldColl_shard1_replica_n2&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2463044 INFO  (qtp1846104485-19009) [n:127.0.0.1:51595_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2463051 INFO  (qtp1698732799-19025) [n:127.0.0.1:51596_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node15&collection.configName=conf&newCollection=true&name=routeFieldColl_shard4_replica_n12&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard4&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2463052 INFO  (qtp810559794-19000) [n:127.0.0.1:51593_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node11&collection.configName=conf&newCollection=true&name=routeFieldColl_shard3_replica_n8&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard3&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2463052 INFO  (qtp1846104485-19010) [n:127.0.0.1:51595_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node13&collection.configName=conf&newCollection=true&name=routeFieldColl_shard3_replica_n10&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard3&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2463052 INFO  (qtp1698732799-19025) [n:127.0.0.1:51596_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2463052 INFO  (qtp810559794-19000) [n:127.0.0.1:51593_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2463052 INFO  (qtp810559794-18999) [n:127.0.0.1:51593_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=routeFieldColl_shard1_replica_n1&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2463060 INFO  (qtp1968048865-19016) [n:127.0.0.1:51594_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node9&collection.configName=conf&newCollection=true&name=routeFieldColl_shard2_replica_n6&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2463060 INFO  (qtp1698732799-19026) [n:127.0.0.1:51596_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf&newCollection=true&name=routeFieldColl_shard2_replica_n4&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2463061 INFO  (qtp1968048865-19016) [n:127.0.0.1:51594_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2463060 INFO  (qtp1968048865-19015) [n:127.0.0.1:51594_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node16&collection.configName=conf&newCollection=true&name=routeFieldColl_shard4_replica_n14&action=CREATE&numShards=4&collection=routeFieldColl&shard=shard4&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2463201 INFO  (zkCallback-2436-thread-1-processing-n:127.0.0.1:51595_solr) [n:127.0.0.1:51595_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 2463201 INFO  (zkCallback-2437-thread-1-processing-n:127.0.0.1:51596_solr) [n:127.0.0.1:51596_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/routeFieldColl/state.json] for collection [routeFieldColl] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 2463201 INFO  (zkCallback-2438-thread

[...truncated too long message...]

il.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >   18) Thread[id=19470, name=Connection evictor, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([EC96E5085E5D0EAC]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=19484, name=zkCallback-2472-thread-4, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    2) Thread[id=19457, name=qtp904986691-19457, state=RUNNABLE, group=TGRP-DistribCursorPagingTest]
   [junit4]    >         at sun.nio.ch.KQueueArrayWrapper.interrupt(Native Method)
   [junit4]    >         at sun.nio.ch.KQueueArrayWrapper.interrupt(KQueueArrayWrapper.java:203)
   [junit4]    >         at sun.nio.ch.KQueueSelectorImpl.wakeup(KQueueSelectorImpl.java:257)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
   [junit4]    >         at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:116)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]    >         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
   [junit4]    >         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    3) Thread[id=19474, name=TEST-DistribCursorPagingTest.test-seed#[EC96E5085E5D0EAC]-SendThread(127.0.0.1:51812), state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
   [junit4]    >    4) Thread[id=19480, name=zkCallback-2472-thread-2, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    5) Thread[id=19476, name=zkCallback-2472-thread-1, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    6) Thread[id=19485, name=zkCallback-2472-thread-5, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    7) Thread[id=19483, name=zkCallback-2472-thread-3, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
   [junit4]    >         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
   [junit4]    >         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    >    8) Thread[id=19465, name=org.eclipse.jetty.server.session.HashSessionManager@2eb2f1fcTimer, state=TIMED_WAITING, group=TGRP-DistribCursorPagingTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
   [junit4]    >         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([EC96E5085E5D0EAC]:0)
   [junit4] Completed [505/729 (2!)] on J0 in 28.06s, 1 test, 3 errors <<< FAILURES!

[...truncated 44051 lines...]

Mime
View raw message