lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Policeman Jenkins Server <jenk...@thetaphi.de>
Subject [JENKINS] Lucene-Solr-6.6-Linux (64bit/jdk1.8.0_144) - Build # 81 - Still Unstable!
Date Tue, 22 Aug 2017 21:14:07 GMT
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-6.6-Linux/81/
Java: 64bit/jdk1.8.0_144 -XX:+UseCompressedOops -XX:+UseG1GC

2 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsRecoveryZkTest

Error Message:
ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.HdfsTransactionLog  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)  at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)  at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:203)  at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:153)  at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:110)  at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:108)  at sun.reflect.GeneratedConstructorAccessor189.newInstance(Unknown Source)  at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)  at java.lang.reflect.Constructor.newInstance(Constructor.java:423)  at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:760)  at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:822)  at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1088)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:947)  at org.apache.solr.core.SolrCore.<init>(SolrCore.java:830)  at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:922)  at org.apache.solr.core.CoreContainer.lambda$load$5(CoreContainer.java:564)  at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)  at java.lang.Thread.run(Thread.java:748)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.HdfsTransactionLog
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
	at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
	at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:203)
	at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:153)
	at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:110)
	at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:108)
	at sun.reflect.GeneratedConstructorAccessor189.newInstance(Unknown Source)
	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
	at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:760)
	at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:822)
	at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1088)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:947)
	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:830)
	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:922)
	at org.apache.solr.core.CoreContainer.lambda$load$5(CoreContainer.java:564)
	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)


	at __randomizedtesting.SeedInfo.seed([8D37D70DE1C2E387]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:302)
	at sun.reflect.GeneratedMethodAccessor108.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:870)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI

Error Message:
Something is broken in the assert for no shards using the same indexDir - probably something was changed in the attributes published in the MBean of SolrCore : {}

Stack Trace:
java.lang.AssertionError: Something is broken in the assert for no shards using the same indexDir - probably something was changed in the attributes published in the MBean of SolrCore : {}
	at __randomizedtesting.SeedInfo.seed([8D37D70DE1C2E387:C542A3B9E7F1CC12]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.checkNoTwoShardsUseTheSameIndexDir(CollectionsAPIDistributedZkTest.java:646)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:524)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11303 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-6.6-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8D37D70DE1C2E387-001/init-core-data-001
   [junit4]   2> 181534 WARN  (SUITE-CollectionsAPIDistributedZkTest-seed#[8D37D70DE1C2E387]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=7 numCloses=7
   [junit4]   2> 181534 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[8D37D70DE1C2E387]-worker) [    ] o.a.s.SolrTestCaseJ4 Using TrieFields
   [junit4]   2> 181535 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[8D37D70DE1C2E387]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 181535 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[8D37D70DE1C2E387]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /home/jenkins/workspace/Lucene-Solr-6.6-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8D37D70DE1C2E387-001/tempDir-001
   [junit4]   2> 181536 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[8D37D70DE1C2E387]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 181536 INFO  (Thread-321) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 181536 INFO  (Thread-321) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 181537 ERROR (Thread-321) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 181636 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[8D37D70DE1C2E387]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:41467
   [junit4]   2> 181648 INFO  (jetty-launcher-198-thread-1) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 181648 INFO  (jetty-launcher-198-thread-3) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 181648 INFO  (jetty-launcher-198-thread-4) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 181649 INFO  (jetty-launcher-198-thread-2) [    ] o.e.j.s.Server jetty-9.3.14.v20161028
   [junit4]   2> 181683 INFO  (jetty-launcher-198-thread-4) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6f358ad4{/solr,null,AVAILABLE}
   [junit4]   2> 181683 INFO  (jetty-launcher-198-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@56dede00{/solr,null,AVAILABLE}
   [junit4]   2> 181683 INFO  (jetty-launcher-198-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1b0629a{/solr,null,AVAILABLE}
   [junit4]   2> 181683 INFO  (jetty-launcher-198-thread-3) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@c7464b2{/solr,null,AVAILABLE}
   [junit4]   2> 181683 INFO  (jetty-launcher-198-thread-4) [    ] o.e.j.s.AbstractConnector Started ServerConnector@767d6471{SSL,[ssl, http/1.1]}{127.0.0.1:๔๓๓๘๕}
   [junit4]   2> 181683 INFO  (jetty-launcher-198-thread-4) [    ] o.e.j.s.Server Started @๑๘๓๑๑๖ms
   [junit4]   2> 181683 INFO  (jetty-launcher-198-thread-4) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43385}
   [junit4]   2> 181684 ERROR (jetty-launcher-198-thread-4) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 181684 INFO  (jetty-launcher-198-thread-4) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 6.6.1
   [junit4]   2> 181684 INFO  (jetty-launcher-198-thread-4) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 181684 INFO  (jetty-launcher-198-thread-4) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 181684 INFO  (jetty-launcher-198-thread-4) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-22T20:32:53.281Z
   [junit4]   2> 181687 INFO  (jetty-launcher-198-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6f193b6a{SSL,[ssl, http/1.1]}{127.0.0.1:๓๓๖๒๙}
   [junit4]   2> 181688 INFO  (jetty-launcher-198-thread-3) [    ] o.e.j.s.AbstractConnector Started ServerConnector@51c4e3d1{SSL,[ssl, http/1.1]}{127.0.0.1:๔๐๒๓๕}
   [junit4]   2> 181688 INFO  (jetty-launcher-198-thread-1) [    ] o.e.j.s.Server Started @๑๘๓๑๒๐ms
   [junit4]   2> 181688 INFO  (jetty-launcher-198-thread-3) [    ] o.e.j.s.Server Started @๑๘๓๑๒๐ms
   [junit4]   2> 181687 INFO  (jetty-launcher-198-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@cec84c6{SSL,[ssl, http/1.1]}{127.0.0.1:๓๗๘๗๗}
   [junit4]   2> 181688 INFO  (jetty-launcher-198-thread-3) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40235}
   [junit4]   2> 181688 INFO  (jetty-launcher-198-thread-2) [    ] o.e.j.s.Server Started @๑๘๓๑๒๐ms
   [junit4]   2> 181688 INFO  (jetty-launcher-198-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=33629}
   [junit4]   2> 181688 INFO  (jetty-launcher-198-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=37877}
   [junit4]   2> 181688 ERROR (jetty-launcher-198-thread-2) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 181688 ERROR (jetty-launcher-198-thread-1) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 181688 INFO  (jetty-launcher-198-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 6.6.1
   [junit4]   2> 181688 INFO  (jetty-launcher-198-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 6.6.1
   [junit4]   2> 181688 INFO  (jetty-launcher-198-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 181688 INFO  (jetty-launcher-198-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 181688 INFO  (jetty-launcher-198-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 181688 INFO  (jetty-launcher-198-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 181688 INFO  (jetty-launcher-198-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-22T20:32:53.285Z
   [junit4]   2> 181688 ERROR (jetty-launcher-198-thread-3) [    ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 181688 INFO  (jetty-launcher-198-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-22T20:32:53.285Z
   [junit4]   2> 181691 INFO  (jetty-launcher-198-thread-3) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 6.6.1
   [junit4]   2> 181691 INFO  (jetty-launcher-198-thread-3) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 181691 INFO  (jetty-launcher-198-thread-3) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 181691 INFO  (jetty-launcher-198-thread-3) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-08-22T20:32:53.288Z
   [junit4]   2> 181691 INFO  (jetty-launcher-198-thread-4) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 181695 INFO  (jetty-launcher-198-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 181695 INFO  (jetty-launcher-198-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 181695 INFO  (jetty-launcher-198-thread-3) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 181698 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e0ba5dd8e0003, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 181698 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15e0ba5dd8e0004, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 181699 INFO  (jetty-launcher-198-thread-4) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 181700 INFO  (jetty-launcher-198-thread-2) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 181700 INFO  (jetty-launcher-198-thread-4) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41467/solr
   [junit4]   2> 181700 INFO  (jetty-launcher-198-thread-3) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 181702 INFO  (jetty-launcher-198-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41467/solr
   [junit4]   2> 181703 INFO  (jetty-launcher-198-thread-3) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41467/solr
   [junit4]   2> 181703 INFO  (jetty-launcher-198-thread-1) [    ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 181707 INFO  (jetty-launcher-198-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:41467/solr
   [junit4]   2> 181714 INFO  (jetty-launcher-198-thread-4) [n:127.0.0.1:43385_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 181715 INFO  (jetty-launcher-198-thread-3) [n:127.0.0.1:40235_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 181715 INFO  (jetty-launcher-198-thread-2) [n:127.0.0.1:37877_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 181715 INFO  (jetty-launcher-198-thread-1) [n:127.0.0.1:33629_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 181715 INFO  (jetty-launcher-198-thread-4) [n:127.0.0.1:43385_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43385_solr
   [junit4]   2> 181715 INFO  (jetty-launcher-198-thread-1) [n:127.0.0.1:33629_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33629_solr
   [junit4]   2> 181715 INFO  (jetty-launcher-198-thread-3) [n:127.0.0.1:40235_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40235_solr
   [junit4]   2> 181715 INFO  (jetty-launcher-198-thread-2) [n:127.0.0.1:37877_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37877_solr
   [junit4]   2> 181715 INFO  (jetty-launcher-198-thread-4) [n:127.0.0.1:43385_solr    ] o.a.s.c.Overseer Overseer (id=98529048863309831-127.0.0.1:43385_solr-n_0000000000) starting
   [junit4]   2> 181716 INFO  (zkCallback-212-thread-1-processing-n:127.0.0.1:43385_solr) [n:127.0.0.1:43385_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 181716 INFO  (zkCallback-219-thread-1-processing-n:127.0.0.1:40235_solr) [n:127.0.0.1:40235_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 181719 INFO  (zkCallback-220-thread-1-processing-n:127.0.0.1:33629_solr) [n:127.0.0.1:33629_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 181719 INFO  (zkCallback-217-thread-1-processing-n:127.0.0.1:37877_solr) [n:127.0.0.1:37877_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 181728 INFO  (jetty-launcher-198-thread-4) [n:127.0.0.1:43385_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43385_solr
   [junit4]   2> 181729 INFO  (zkCallback-217-thread-1-processing-n:127.0.0.1:37877_solr) [n:127.0.0.1:37877_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 181729 INFO  (zkCallback-219-thread-1-processing-n:127.0.0.1:40235_solr) [n:127.0.0.1:40235_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 181735 INFO  (zkCallback-220-thread-1-processing-n:127.0.0.1:33629_solr) [n:127.0.0.1:33629_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 181735 INFO  (zkCallback-212-thread-2-processing-n:127.0.0.1:43385_solr) [n:127.0.0.1:43385_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 181758 INFO  (jetty-launcher-198-thread-2) [n:127.0.0.1:37877_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-6.6-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8D37D70DE1C2E387-001/tempDir-001/node2/.
   [junit4]   2> 181758 INFO  (jetty-launcher-198-thread-3) [n:127.0.0.1:40235_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-6.6-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8D37D70DE1C2E387-001/tempDir-001/node3/.
   [junit4]   2> 181772 INFO  (jetty-launcher-198-thread-1) [n:127.0.0.1:33629_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-6.6-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8D37D70DE1C2E387-001/tempDir-001/node1/.
   [junit4]   2> 181836 INFO  (jetty-launcher-198-thread-4) [n:127.0.0.1:43385_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-6.6-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8D37D70DE1C2E387-001/tempDir-001/node4/.
   [junit4]   2> 181843 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[8D37D70DE1C2E387]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 181843 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[8D37D70DE1C2E387]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:41467/solr ready
   [junit4]   2> 181855 INFO  (TEST-CollectionsAPIDistributedZkTest.testMaxNodesPerShard-seed#[8D37D70DE1C2E387]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testMaxNodesPerShard
   [junit4]   2> 181856 INFO  (TEST-CollectionsAPIDistributedZkTest.testMaxNodesPerShard-seed#[8D37D70DE1C2E387]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 181919 INFO  (qtp1097634832-1362) [n:127.0.0.1:37877_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&name=oversharded&action=CREATE&numShards=3&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 181921 INFO  (OverseerThreadFactory-664-thread-1-processing-n:127.0.0.1:43385_solr) [n:127.0.0.1:43385_solr    ] o.a.s.c.CreateCollectionCmd Create collection oversharded
   [junit4]   2> 181921 ERROR (OverseerThreadFactory-664-thread-1-processing-n:127.0.0.1:43385_solr) [n:127.0.0.1:43385_solr    ] o.a.s.c.OverseerCollectionMessageHandler Collection: oversharded operation: create failed:org.apache.solr.common.SolrException: Cannot create collection oversharded. Value of maxShardsPerNode is 1, and the number of nodes currently live or live and part of your createNodeSet is 4. This allows a maximum of 4 to be created. Value of numShards is 3 and value of replicationFactor is 2. This requires 6 shards to be created (higher than the allowed number)
   [junit4]   2> 	at org.apache.solr.cloud.CreateCollectionCmd.call(CreateCollectionCmd.java:155)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:226)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:464)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 181927 INFO  (qtp1097634832-1362) [n:127.0.0.1:37877_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf&name=oversharded&action=CREATE&numShards=3&wt=javabin&version=2} status=400 QTime=7
   [junit4]   2> 181928 INFO  (TEST-CollectionsAPIDistributedZkTest.testMaxNodesPerShard-seed#[8D37D70DE1C2E387]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testMaxNodesPerShard
   [junit4]   2> 181936 INFO  (TEST-CollectionsAPIDistributedZkTest.deleteCollectionRemovesStaleZkCollectionsNode-seed#[8D37D70DE1C2E387]) [    ] o.a.s.SolrTestCaseJ4 ###Starting deleteCollectionRemovesStaleZkCollectionsNode
   [junit4]   2> 181937 INFO  (TEST-CollectionsAPIDistributedZkTest.deleteCollectionRemovesStaleZkCollectionsNode-seed#[8D37D70DE1C2E387]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 181940 INFO  (qtp1097634832-1382) [n:127.0.0.1:37877_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=out_of_sync_collection&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 181941 INFO  (OverseerCollectionConfigSetProcessor-98529048863309831-127.0.0.1:43385_solr-n_0000000000) [n:127.0.0.1:43385_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 181943 INFO  (qtp1097634832-1382) [n:127.0.0.1:37877_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=out_of_sync_collection&action=DELETE&wt=javabin&version=2} status=0 QTime=2
   [junit4]   2> 181945 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 181945 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 181946 INFO  (TEST-CollectionsAPIDistributedZkTest.deleteCollectionRemovesStaleZkCollectionsNode-seed#[8D37D70DE1C2E387]) [    ] o.a.s.SolrTestCaseJ4 ###Ending deleteCollectionRemovesStaleZkCollectionsNode
   [junit4]   2> 181957 INFO  (TEST-CollectionsAPIDistributedZkTest.testCreationAndDeletion-seed#[8D37D70DE1C2E387]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testCreationAndDeletion
   [junit4]   2> 181957 INFO  (TEST-CollectionsAPIDistributedZkTest.testCreationAndDeletion-seed#[8D37D70DE1C2E387]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 181959 INFO  (qtp1097634832-1388) [n:127.0.0.1:37877_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=created_and_deleted&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 181961 INFO  (OverseerThreadFactory-664-thread-3-processing-n:127.0.0.1:43385_solr) [n:127.0.0.1:43385_solr    ] o.a.s.c.CreateCollectionCmd Create collection created_and_deleted
   [junit4]   2> 181961 INFO  (OverseerCollectionConfigSetProcessor-98529048863309831-127.0.0.1:43385_solr-n_0000000000) [n:127.0.0.1:43385_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 182118 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=created_and_deleted_shard1_replica1&action=CREATE&numShards=1&collection=created_and_deleted&shard=shard1&wt=javabin&version=2
   [junit4]   2> 182118 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 182225 INFO  (zkCallback-212-thread-2-processing-n:127.0.0.1:43385_solr) [n:127.0.0.1:43385_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/created_and_deleted/state.json] for collection [created_and_deleted] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 183132 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.6.1
   [junit4]   2> 183138 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.s.IndexSchema [created_and_deleted_shard1_replica1] Schema name=minimal
   [junit4]   2> 183141 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 183141 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'created_and_deleted_shard1_replica1' using configuration from collection created_and_deleted, trusted=true
   [junit4]   2> 183141 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 183141 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.c.SolrCore [[created_and_deleted_shard1_replica1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-6.6-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8D37D70DE1C2E387-001/tempDir-001/node4/created_and_deleted_shard1_replica1], dataDir=[/home/jenkins/workspace/Lucene-Solr-6.6-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8D37D70DE1C2E387-001/tempDir-001/node4/./created_and_deleted_shard1_replica1/data/]
   [junit4]   2> 183177 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 183177 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 183178 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 183178 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 183179 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@57ca56d0[created_and_deleted_shard1_replica1] main]
   [junit4]   2> 183179 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 183180 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 183180 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 183180 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576464783535767552
   [junit4]   2> 183180 INFO  (searcherExecutor-673-thread-1-processing-n:127.0.0.1:43385_solr x:created_and_deleted_shard1_replica1 s:shard1 c:created_and_deleted) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.c.SolrCore [created_and_deleted_shard1_replica1] Registered new searcher Searcher@57ca56d0[created_and_deleted_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 183183 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 183183 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 183183 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:43385/solr/created_and_deleted_shard1_replica1/
   [junit4]   2> 183183 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 183183 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.c.SyncStrategy https://127.0.0.1:43385/solr/created_and_deleted_shard1_replica1/ has no replicas
   [junit4]   2> 183183 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 183184 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:43385/solr/created_and_deleted_shard1_replica1/ shard1
   [junit4]   2> 183286 INFO  (zkCallback-212-thread-2-processing-n:127.0.0.1:43385_solr) [n:127.0.0.1:43385_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/created_and_deleted/state.json] for collection [created_and_deleted] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 183335 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 183337 INFO  (qtp2010624333-1363) [n:127.0.0.1:43385_solr c:created_and_deleted s:shard1  x:created_and_deleted_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=created_and_deleted_shard1_replica1&action=CREATE&numShards=1&collection=created_and_deleted&shard=shard1&wt=javabin&version=2} status=0 QTime=1219
   [junit4]   2> 183340 INFO  (qtp1097634832-1388) [n:127.0.0.1:37877_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 183439 INFO  (zkCallback-212-thread-2-processing-n:127.0.0.1:43385_solr) [n:127.0.0.1:43385_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/created_and_deleted/state.json] for collection [created_and_deleted] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 183963 INFO  (OverseerCollectionConfigSetProcessor-98529048863309831-127.0.0.1:43385_solr-n_0000000000) [n:127.0.0.1:43385_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 184341 INFO  (qtp1097634832-1388) [n:127.0.0.1:37877_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf&name=created_and_deleted&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2381
   [junit4]   2> 184344 INFO  (qtp1097634832-1362) [n:127.0.0.1:37877_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 184344 INFO  (qtp1097634832-1362) [n:127.0.0.1:37877_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 184347 INFO  (qtp1097634832-1382) [n:127.0.0.1:37877_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=created_and_deleted&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 184350 INFO  (OverseerThreadFactory-664-thread-4-processing-n:127.0.0.1:43385_solr) [n:127.0.0.1:43385_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 184352 INFO  (qtp2010624333-1389) [n:127.0.0.1:43385_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.created_and_deleted.shard1.replica1
   [junit4]   2> 184353 INFO  (qtp2010624333-1389) [n:127.0.0.1:43385_solr    ] o.a.s.c.SolrCore [created_and_deleted_shard1_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@831db48
   [junit4]   2> 184354 INFO  (qtp2010624333-1389) [n:127.0.0.1:43385_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.created_and_deleted.shard1.replica1
   [junit4]   2> 184355 INFO  (qtp2010624333-1389) [n:127.0.0.1:43385_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=created_and_deleted_shard1_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=3
   [junit4]   2> 185061 INFO  (qtp1097634832-1382) [n:127.0.0.1:37877_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=created_and_deleted&action=DELETE&wt=javabin&version=2} status=0 QTime=713
   [junit4]   2> 185064 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 185064 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 185065 INFO  (TEST-CollectionsAPIDistributedZkTest.testCreationAndDeletion-seed#[8D37D70DE1C2E387]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testCreationAndDeletion
   [junit4]   2> 185086 INFO  (TEST-CollectionsAPIDistributedZkTest.testBadActionNames-seed#[8D37D70DE1C2E387]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testBadActionNames
   [junit4]   2> 185086 INFO  (TEST-CollectionsAPIDistributedZkTest.testBadActionNames-seed#[8D37D70DE1C2E387]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 185089 ERROR (qtp1097634832-1388) [n:127.0.0.1:37877_solr    ] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Unknown action: BADACTION
   [junit4]   2> 	at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:210)
   [junit4]   2> 	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:173)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:748)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:729)
   [junit4]   2> 	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:510)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:361)
   [junit4]   2> 	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:305)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:136)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1699)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
   [junit4]   2> 	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
   [junit4]   2> 	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)
   [junit4]   2> 	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
   [junit4]   2> 	at org.eclipse.jetty.server.Server.handle(Server.java:534)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
   [junit4]   2> 	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]   2> 	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:202)
   [junit4]   2> 	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
   [junit4]   2> 	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
   [junit4]   2> 	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
   [junit4]   2> 	at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> 185090 INFO  (qtp1097634832-1388) [n:127.0.0.1:37877_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=badactioncollection&action=BADACTION&numShards=2&wt=javabin&version=2} status=400 QTime=1
   [junit4]   2> 185091 INFO  (TEST-CollectionsAPIDistributedZkTest.testBadActionNames-seed#[8D37D70DE1C2E387]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testBadActionNames
   [junit4]   2> 185103 INFO  (TEST-CollectionsAPIDistributedZkTest.testCoresAreDistributedAcrossNodes-seed#[8D37D70DE1C2E387]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testCoresAreDistributedAcrossNodes
   [junit4]   2> 185103 INFO  (TEST-CollectionsAPIDistributedZkTest.testCoresAreDistributedAcrossNodes-seed#[8D37D70DE1C2E387]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 185105 INFO  (qtp1097634832-1362) [n:127.0.0.1:37877_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&name=nodes_used_collection&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 185106 INFO  (OverseerThreadFactory-664-thread-5-processing-n:127.0.0.1:43385_solr) [n:127.0.0.1:43385_solr    ] o.a.s.c.CreateCollectionCmd Create collection nodes_used_collection
   [junit4]   2> 185106 INFO  (OverseerCollectionConfigSetProcessor-98529048863309831-127.0.0.1:43385_solr-n_0000000000) [n:127.0.0.1:43385_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 185320 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=nodes_used_collection_shard1_replica2&action=CREATE&numShards=2&collection=nodes_used_collection&shard=shard1&wt=javabin&version=2
   [junit4]   2> 185331 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=nodes_used_collection_shard2_replica2&action=CREATE&numShards=2&collection=nodes_used_collection&shard=shard2&wt=javabin&version=2
   [junit4]   2> 185332 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 185389 INFO  (qtp989994946-1364) [n:127.0.0.1:40235_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=nodes_used_collection_shard1_replica1&action=CREATE&numShards=2&collection=nodes_used_collection&shard=shard1&wt=javabin&version=2
   [junit4]   2> 185390 INFO  (qtp989994946-1364) [n:127.0.0.1:40235_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 185396 INFO  (qtp1566531690-1360) [n:127.0.0.1:33629_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf&newCollection=true&name=nodes_used_collection_shard2_replica1&action=CREATE&numShards=2&collection=nodes_used_collection&shard=shard2&wt=javabin&version=2
   [junit4]   2> 185397 INFO  (qtp1566531690-1360) [n:127.0.0.1:33629_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 185501 INFO  (zkCallback-217-thread-1-processing-n:127.0.0.1:37877_solr) [n:127.0.0.1:37877_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 185501 INFO  (zkCallback-220-thread-1-processing-n:127.0.0.1:33629_solr) [n:127.0.0.1:33629_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 185501 INFO  (zkCallback-212-thread-2-processing-n:127.0.0.1:43385_solr) [n:127.0.0.1:43385_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 185501 INFO  (zkCallback-219-thread-1-processing-n:127.0.0.1:40235_solr) [n:127.0.0.1:40235_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 186332 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.6.1
   [junit4]   2> 186336 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.s.IndexSchema [nodes_used_collection_shard1_replica2] Schema name=minimal
   [junit4]   2> 186338 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 186338 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.c.CoreContainer Creating SolrCore 'nodes_used_collection_shard1_replica2' using configuration from collection nodes_used_collection, trusted=true
   [junit4]   2> 186338 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 186338 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.c.SolrCore [[nodes_used_collection_shard1_replica2] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-6.6-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8D37D70DE1C2E387-001/tempDir-001/node4/nodes_used_collection_shard1_replica2], dataDir=[/home/jenkins/workspace/Lucene-Solr-6.6-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8D37D70DE1C2E387-001/tempDir-001/node4/./nodes_used_collection_shard1_replica2/data/]
   [junit4]   2> 186339 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.6.1
   [junit4]   2> 186342 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.s.IndexSchema [nodes_used_collection_shard2_replica2] Schema name=minimal
   [junit4]   2> 186343 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 186343 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.c.CoreContainer Creating SolrCore 'nodes_used_collection_shard2_replica2' using configuration from collection nodes_used_collection, trusted=true
   [junit4]   2> 186344 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 186344 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.c.SolrCore [[nodes_used_collection_shard2_replica2] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-6.6-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8D37D70DE1C2E387-001/tempDir-001/node2/nodes_used_collection_shard2_replica2], dataDir=[/home/jenkins/workspace/Lucene-Solr-6.6-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8D37D70DE1C2E387-001/tempDir-001/node2/./nodes_used_collection_shard2_replica2/data/]
   [junit4]   2> 186360 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 186360 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 186361 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 186361 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 186361 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 186361 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 186361 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@18fe09da[nodes_used_collection_shard1_replica2] main]
   [junit4]   2> 186362 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 186362 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 186362 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 186362 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 186362 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 186362 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.s.SolrIndexSearcher Opening [Searcher@6ff805df[nodes_used_collection_shard2_replica2] main]
   [junit4]   2> 186363 INFO  (searcherExecutor-678-thread-1-processing-n:127.0.0.1:43385_solr x:nodes_used_collection_shard1_replica2 s:shard1 c:nodes_used_collection) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.c.SolrCore [nodes_used_collection_shard1_replica2] Registered new searcher Searcher@18fe09da[nodes_used_collection_shard1_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 186363 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576464786873384960
   [junit4]   2> 186363 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 186363 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 186363 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 186364 INFO  (searcherExecutor-679-thread-1-processing-n:127.0.0.1:37877_solr x:nodes_used_collection_shard2_replica2 s:shard2 c:nodes_used_collection) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.c.SolrCore [nodes_used_collection_shard2_replica2] Registered new searcher Searcher@6ff805df[nodes_used_collection_shard2_replica2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 186364 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576464786874433536
   [junit4]   2> 186365 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 186366 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=9999ms
   [junit4]   2> 186398 INFO  (qtp989994946-1364) [n:127.0.0.1:40235_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.6.1
   [junit4]   2> 186401 INFO  (qtp989994946-1364) [n:127.0.0.1:40235_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica1] o.a.s.s.IndexSchema [nodes_used_collection_shard1_replica1] Schema name=minimal
   [junit4]   2> 186402 INFO  (qtp989994946-1364) [n:127.0.0.1:40235_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 186402 INFO  (qtp989994946-1364) [n:127.0.0.1:40235_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'nodes_used_collection_shard1_replica1' using configuration from collection nodes_used_collection, trusted=true
   [junit4]   2> 186402 INFO  (qtp989994946-1364) [n:127.0.0.1:40235_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 186402 INFO  (qtp989994946-1364) [n:127.0.0.1:40235_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica1] o.a.s.c.SolrCore [[nodes_used_collection_shard1_replica1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-6.6-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8D37D70DE1C2E387-001/tempDir-001/node3/nodes_used_collection_shard1_replica1], dataDir=[/home/jenkins/workspace/Lucene-Solr-6.6-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8D37D70DE1C2E387-001/tempDir-001/node3/./nodes_used_collection_shard1_replica1/data/]
   [junit4]   2> 186403 INFO  (qtp1566531690-1360) [n:127.0.0.1:33629_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 6.6.1
   [junit4]   2> 186420 INFO  (qtp1566531690-1360) [n:127.0.0.1:33629_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica1] o.a.s.s.IndexSchema [nodes_used_collection_shard2_replica1] Schema name=minimal
   [junit4]   2> 186420 INFO  (qtp1566531690-1360) [n:127.0.0.1:33629_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 186421 INFO  (qtp1566531690-1360) [n:127.0.0.1:33629_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica1] o.a.s.c.CoreContainer Creating SolrCore 'nodes_used_collection_shard2_replica1' using configuration from collection nodes_used_collection, trusted=true
   [junit4]   2> 186421 INFO  (qtp1566531690-1360) [n:127.0.0.1:33629_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 186421 INFO  (qtp1566531690-1360) [n:127.0.0.1:33629_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica1] o.a.s.c.SolrCore [[nodes_used_collection_shard2_replica1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-6.6-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8D37D70DE1C2E387-001/tempDir-001/node1/nodes_used_collection_shard2_replica1], dataDir=[/home/jenkins/workspace/Lucene-Solr-6.6-Linux/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8D37D70DE1C2E387-001/tempDir-001/node1/./nodes_used_collection_shard2_replica1/data/]
   [junit4]   2> 186436 INFO  (qtp989994946-1364) [n:127.0.0.1:40235_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 186436 INFO  (qtp989994946-1364) [n:127.0.0.1:40235_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 186436 INFO  (qtp989994946-1364) [n:127.0.0.1:40235_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 186436 INFO  (qtp989994946-1364) [n:127.0.0.1:40235_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 186437 INFO  (qtp989994946-1364) [n:127.0.0.1:40235_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7a7d2ccc[nodes_used_collection_shard1_replica1] main]
   [junit4]   2> 186437 INFO  (qtp989994946-1364) [n:127.0.0.1:40235_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 186438 INFO  (qtp989994946-1364) [n:127.0.0.1:40235_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 186438 INFO  (qtp989994946-1364) [n:127.0.0.1:40235_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 186438 INFO  (searcherExecutor-688-thread-1-processing-n:127.0.0.1:40235_solr x:nodes_used_collection_shard1_replica1 s:shard1 c:nodes_used_collection) [n:127.0.0.1:40235_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica1] o.a.s.c.SolrCore [nodes_used_collection_shard1_replica1] Registered new searcher Searcher@7a7d2ccc[nodes_used_collection_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 186438 INFO  (qtp989994946-1364) [n:127.0.0.1:40235_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576464786952028160
   [junit4]   2> 186441 INFO  (qtp1566531690-1360) [n:127.0.0.1:33629_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 186441 INFO  (qtp1566531690-1360) [n:127.0.0.1:33629_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 186442 INFO  (qtp1566531690-1360) [n:127.0.0.1:33629_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 186442 INFO  (qtp1566531690-1360) [n:127.0.0.1:33629_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 186443 INFO  (qtp1566531690-1360) [n:127.0.0.1:33629_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5d9b7be5[nodes_used_collection_shard2_replica1] main]
   [junit4]   2> 186443 INFO  (qtp1566531690-1360) [n:127.0.0.1:33629_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 186443 INFO  (qtp1566531690-1360) [n:127.0.0.1:33629_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 186443 INFO  (qtp1566531690-1360) [n:127.0.0.1:33629_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 186443 INFO  (qtp1566531690-1360) [n:127.0.0.1:33629_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1576464786957271040
   [junit4]   2> 186444 INFO  (searcherExecutor-689-thread-1-processing-n:127.0.0.1:33629_solr x:nodes_used_collection_shard2_replica1 s:shard2 c:nodes_used_collection) [n:127.0.0.1:33629_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica1] o.a.s.c.SolrCore [nodes_used_collection_shard2_replica1] Registered new searcher Searcher@5d9b7be5[nodes_used_collection_shard2_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 186467 INFO  (zkCallback-219-thread-1-processing-n:127.0.0.1:40235_solr) [n:127.0.0.1:40235_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 186467 INFO  (zkCallback-220-thread-1-processing-n:127.0.0.1:33629_solr) [n:127.0.0.1:33629_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 186467 INFO  (zkCallback-212-thread-2-processing-n:127.0.0.1:43385_solr) [n:127.0.0.1:43385_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 186467 INFO  (zkCallback-217-thread-1-processing-n:127.0.0.1:37877_solr) [n:127.0.0.1:37877_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 186866 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 186866 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 186866 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:43385/solr/nodes_used_collection_shard1_replica2/
   [junit4]   2> 186866 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.u.PeerSync PeerSync: core=nodes_used_collection_shard1_replica2 url=https://127.0.0.1:43385/solr START replicas=[https://127.0.0.1:40235/solr/nodes_used_collection_shard1_replica1/] nUpdates=100
   [junit4]   2> 186867 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.u.PeerSync PeerSync: core=nodes_used_collection_shard1_replica2 url=https://127.0.0.1:43385/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 186867 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 186867 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 186867 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:37877/solr/nodes_used_collection_shard2_replica2/
   [junit4]   2> 186868 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.u.PeerSync PeerSync: core=nodes_used_collection_shard2_replica2 url=https://127.0.0.1:37877/solr START replicas=[https://127.0.0.1:33629/solr/nodes_used_collection_shard2_replica1/] nUpdates=100
   [junit4]   2> 186870 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.u.PeerSync PeerSync: core=nodes_used_collection_shard2_replica2 url=https://127.0.0.1:37877/solr DONE.  We have no versions.  sync failed.
   [junit4]   2> 186883 INFO  (qtp989994946-1367) [n:127.0.0.1:40235_solr c:nodes_used_collection s:shard1 r:core_node3 x:nodes_used_collection_shard1_replica1] o.a.s.c.S.Request [nodes_used_collection_shard1_replica1]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=1
   [junit4]   2> 186885 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 186885 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 186885 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 186886 INFO  (qtp1566531690-1381) [n:127.0.0.1:33629_solr c:nodes_used_collection s:shard2 r:core_node4 x:nodes_used_collection_shard2_replica1] o.a.s.c.S.Request [nodes_used_collection_shard2_replica1]  webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 186887 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 186887 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 186887 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 186891 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:43385/solr/nodes_used_collection_shard1_replica2/ shard1
   [junit4]   2> 186891 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:37877/solr/nodes_used_collection_shard2_replica2/ shard2
   [junit4]   2> 186995 INFO  (zkCallback-217-thread-1-processing-n:127.0.0.1:37877_solr) [n:127.0.0.1:37877_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 186995 INFO  (zkCallback-220-thread-1-processing-n:127.0.0.1:33629_solr) [n:127.0.0.1:33629_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 186995 INFO  (zkCallback-212-thread-2-processing-n:127.0.0.1:43385_solr) [n:127.0.0.1:43385_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 186995 INFO  (zkCallback-219-thread-1-processing-n:127.0.0.1:40235_solr) [n:127.0.0.1:40235_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 187042 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 187042 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 187043 INFO  (qtp2010624333-1391) [n:127.0.0.1:43385_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=nodes_used_collection_shard1_replica2&action=CREATE&numShards=2&collection=nodes_used_collection&shard=shard1&wt=javabin&version=2} status=0 QTime=1723
   [junit4]   2> 187044 INFO  (qtp1097634832-1384) [n:127.0.0.1:37877_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=nodes_used_collection_shard2_replica2&action=CREATE&numShards=2&collection=nodes_used_collection&shard=shard2&wt=javabin&version=2} status=0 QTime=1713
   [junit4]   2> 187147 INFO  (zkCallback-219-thread-1-processing-n:127.0.0.1:40235_solr) [n:127.0.0.1:40235_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 187147 INFO  (zkCallback-212-thread-2-processing-n:127.0.0.1:43385_solr) [n:127.0.0.1:43385_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 187147 INFO  (zkCallback-220-thread-1-processing-n:127.0.0.1:33629_solr) [n:127.0.0.1:33629_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 187147 INFO  (zkCallback-217-thread-1-processing-n:127.0.0.1:37877_solr) [n:127.0.0.1:37877_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 187443 INFO  (qtp989994946-1364) [n:127.0.0.1:40235_solr c:nodes_used_collection s:shard1  x:nodes_used_collection_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=nodes_used_collection_shard1_replica1&action=CREATE&numShards=2&collection=nodes_used_collection&shard=shard1&wt=javabin&version=2} status=0 QTime=2056
   [junit4]   2> 187448 INFO  (qtp1566531690-1360) [n:127.0.0.1:33629_solr c:nodes_used_collection s:shard2  x:nodes_used_collection_shard2_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf&newCollection=true&name=nodes_used_collection_shard2_replica1&action=CREATE&numShards=2&collection=nodes_used_collection&shard=shard2&wt=javabin&version=2} status=0 QTime=2052
   [junit4]   2> 187452 INFO  (qtp1097634832-1362) [n:127.0.0.1:37877_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 187552 INFO  (zkCallback-219-thread-1-processing-n:127.0.0.1:40235_solr) [n:127.0.0.1:40235_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 187552 INFO  (zkCallback-217-thread-1-processing-n:127.0.0.1:37877_solr) [n:127.0.0.1:37877_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 187552 INFO  (zkCallback-212-thread-2-processing-n:127.0.0.1:43385_solr) [n:127.0.0.1:43385_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 187552 INFO  (zkCallback-220-thread-1-processing-n:127.0.0.1:33629_solr) [n:127.0.0.1:33629_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
   [junit4]   2> 188452 INFO  (qtp1097634832-1362) [n:127.0.0.1:37877_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf&name=nodes_used_collection&action=CREATE&numShards=2&wt=javabin&version=2} status=0 QTime=3346
   [junit4]   2> 188454 INFO  (TEST-CollectionsAPIDistributedZkTest.testCoresAreDistributedAcrossNodes-seed#[8D37D70DE1C2E387]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testCoresAreDistributedAcrossNodes
   [junit4]   2> 188489 INFO  (TEST-CollectionsAPIDistributedZkTest.testDeleteNonExistentCollection-seed#[8D37D70DE1C2E387]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDeleteNonExistentCollection
   [junit4]   2> 188491 INFO  (TEST-CollectionsAPIDistributedZkTest.testDeleteNonExistentCollection-seed#[8D37D70DE1C2E387]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 188494 INFO  (qtp1097634832-1382) [n:127.0.0.1:37877_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=nodes_used_collection&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 188497 INFO  (OverseerThreadFactory-664-thread-5-processing-n:127.0.0.1:43385_solr) [n:127.0.0.1:43385_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 188497 INFO  (OverseerCollectionConfigSetProcessor-98529048863309831-127.0.0.1:43385_solr-n_0000000000) [n:127.0.0.1:43385_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000008 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 188501 INFO  (qtp1097634832-1388) [n:127.0.0.1:37877_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.nodes_used_collection.shard2.replica2
   [junit4]   2> 188501 INFO  (qtp2010624333-1392) [n:127.0.0.1:43385_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.nodes_used_collection.shard1.replica2
   [junit4]   2> 188501 INFO  (qtp989994946-1380) [n:127.0.0.1:40235_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.nodes_used_collection.shard1.replica1
   [junit4]   2> 188503 INFO  (qtp1097634832-1388) [n:127.0.0.1:37877_solr    ] o.a.s.c.SolrCore [nodes_used_collection_shard2_replica2]  CLOSING SolrCore org.apache.solr.core.SolrCore@7283929a
   [junit4]   2> 188503 INFO  (qtp2010624333-1392) [n:127.0.0.1:43385_solr    ] o.a.s.c.SolrCore [nodes_used_collection_shard1_replica2]  CLOSING SolrCore org.apache.solr.core.SolrCore@423c7058
   [junit4]   2> 188503 INFO  (qtp989994946-1380) [n:127.0.0.1:40235_solr    ] o.a.s.c.SolrCore [nodes_used_collection_shard1_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@218b5793
   [junit4]   2> 188510 INFO  (qtp1566531690-1377) [n:127.0.0.1:33629_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.nodes_used_collection.shard2.replica1
   [junit4]   2> 188510 INFO  (qtp1097634832-1388) [n:127.0.0.1:37877_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.nodes_used_collection.shard2.replica2
   [junit4]   2> 188510 INFO  (qtp1566531690-1377) [n:127.0.0.1:33629_solr    ] o.a.s.c.SolrCore [nodes_used_collection_shard2_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@14d03a0a
   [junit4]   2> 188510 INFO  (qtp2010624333-1392) [n:127.0.0.1:43385_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.nodes_used_collection.shard1.replica2
   [junit4]   2> 188511 INFO  (qtp989994946-1380) [n:127.0.0.1:40235_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.nodes_used_collection.shard1.replica1
   [junit4]   2> 188513 INFO  (qtp1566531690-1377) [n:127.0.0.1:33629_solr    ] o.a.s.m.SolrMetricManager Closing metric reporters for: solr.core.nodes_used_collection.shard2.replica1
   [junit4]   2> 188515 INFO  (qtp989994946-1380) [n:127.0.0.1:40235_solr    ] o.a.s.c.ShardLeaderElectionContextBase No version found for ephemeral leader parent node, won't remove previous leader registration.
   [junit4]   2> 188515 WARN  (zkCallback-219-thread-1-processing-n:127.0.0.1:40235_solr) [n:127.0.0.1:40235_solr    ] o.a.s.c.LeaderElector Our node is no longer in line to be leader
   [junit4]   2> 188517 WARN  (zkCallback-220-thread-1-processing-n:127.0.0.1:33629_solr) [n:127.0.0.1:33629_solr    ] o.a.s.c.LeaderElector 
   [junit4]   2> org.apache.solr.common.SolrException: SolrCore not found:nodes_used_collection_shard2_replica1 in []
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:294)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:56)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:348)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.lambda$process$0(SolrZkClient.java:269)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]   2> 	at j

[...truncated too long message...]

2> 
   [junit4]   2> SUITE-HdfsRecoveryZkTest-seed#[8D37D70DE1C2E387]-worker:
   [junit4]   2>     java.lang.Thread.dumpThreads(Native Method)
   [junit4]   2>     java.lang.Thread.getAllStackTraces(Thread.java:1610)
   [junit4]   2>     org.apache.solr.SolrTestCaseJ4.clearObjectTrackerAndCheckEmpty(SolrTestCaseJ4.java:355)
   [junit4]   2>     org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:301)
   [junit4]   2>     [...sun.*, java.lang.reflect.*, com.carrotsearch.randomizedtesting.*]
   [junit4]   2>     org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>     [...com.carrotsearch.randomizedtesting.*]
   [junit4]   2>     org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>     [...com.carrotsearch.randomizedtesting.*]
   [junit4]   2>     org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>     org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>     org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>     org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>     [...com.carrotsearch.randomizedtesting.*]
   [junit4]   2>     java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> main:
   [junit4]   2>     java.lang.Object.wait(Native Method)
   [junit4]   2>     java.lang.Thread.join(Thread.java:1252)
   [junit4]   2>     java.lang.Thread.join(Thread.java:1326)
   [junit4]   2>     [...com.carrotsearch.randomizedtesting.*]
   [junit4]   2>     com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:244)
   [junit4]   2>     com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:355)
   [junit4]   2>     com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:13)
   [junit4]   2> 
   [junit4]   2> JUnit4-serializer-daemon:
   [junit4]   2>     java.lang.Thread.sleep(Native Method)
   [junit4]   2>     com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:50)
   [junit4]   2> 
   [junit4]   2> LeaseRenewer:jenkins@localhost.localdomain:43481:
   [junit4]   2>     java.lang.Thread.sleep(Native Method)
   [junit4]   2>     org.apache.hadoop.hdfs.LeaseRenewer.run(LeaseRenewer.java:444)
   [junit4]   2>     org.apache.hadoop.hdfs.LeaseRenewer.access$700(LeaseRenewer.java:71)
   [junit4]   2>     org.apache.hadoop.hdfs.LeaseRenewer$1.run(LeaseRenewer.java:304)
   [junit4]   2>     java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 
   [junit4]   2> TimeLimitedCollector timer thread:
   [junit4]   2>     java.lang.Thread.sleep(Native Method)
   [junit4]   2>     org.apache.lucene.search.TimeLimitingCollector$TimerThread.run(TimeLimitingCollector.java:261)
   [junit4]   2> 
   [junit4]   2> 1577648 INFO  (SUITE-HdfsRecoveryZkTest-seed#[8D37D70DE1C2E387]-worker) [    ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
   [junit4]   2> NOTE: test params are: codec=Lucene62, sim=RandomSimilarity(queryNorm=false,coord=no): {}, locale=en-US, timezone=Pacific/Chatham
   [junit4]   2> NOTE: Linux 4.10.0-27-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=8,threads=2,free=119217200,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [TestSSLRandomization, TestRemoteStreaming, ClusterStateTest, CoreAdminCreateDiscoverTest, ConjunctionSolrSpellCheckerTest, TestRecoveryHdfs, TlogReplayBufferedWhileIndexingTest, TestSolrCoreParser, TestRandomCollapseQParserPlugin, TestReplicaProperties, DistributedFacetExistsSmallTest, HighlighterMaxOffsetTest, RequiredFieldsTest, TestStressRecovery, TestBinaryResponseWriter, CollectionsAPIAsyncDistributedZkTest, TestJmxIntegration, TestPKIAuthenticationPlugin, TestSolrCoreProperties, TestJettySolrRunner, TestCoreAdminApis, TestPerFieldSimilarity, JsonValidatorTest, ShufflingReplicaListTransformerTest, TestConfigSetProperties, TestIBSimilarityFactory, TestMinMaxOnMultiValuedField, AliasIntegrationTest, TestLocalFSCloudBackupRestore, SolrCoreTest, HdfsBasicDistributedZk2Test, TestDownShardTolerantSearch, TestAtomicUpdateErrorCases, TestSubQueryTransformerCrossCore, DocumentAnalysisRequestHandlerTest, TestInfoStreamLogging, TestSimpleQParserPlugin, BaseCdcrDistributedZkTest, TestGeoJSONResponseWriter, RecoveryAfterSoftCommitTest, BlockJoinFacetSimpleTest, TestUniqueKeyFieldResource, TolerantUpdateProcessorTest, TestZkAclsWithHadoopAuth, ExplicitHLLTest, TestSchemaResource, ScriptEngineTest, RegexBoostProcessorTest, TestValueSourceCache, SolrRequestParserTest, XsltUpdateRequestHandlerTest, SolrMetricReporterTest, TestRandomFaceting, HdfsRecoverLeaseTest, VersionInfoTest, TestDocSet, HdfsChaosMonkeySafeLeaderTest, PolyFieldTest, ManagedSchemaRoundRobinCloudTest, SignatureUpdateProcessorFactoryTest, BadCopyFieldTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestQuerySenderNoQuery, UUIDFieldTest, ChaosMonkeyNothingIsSafeTest, SynonymTokenizerTest, TestCustomStream, TestBackupRepositoryFactory, TestDocTermOrds, TestAuthenticationFramework, TestPointFields, ResourceLoaderTest, TestInPlaceUpdatesDistrib, TestCSVLoader, SSLMigrationTest, TestCloudRecovery, FileBasedSpellCheckerTest, DataDrivenBlockJoinTest, TestClassicSimilarityFactory, ExitableDirectoryReaderTest, TestFunctionQuery, TestLazyCores, TestFilteredDocIdSet, TestNestedDocsSort, QueryElevationComponentTest, DocValuesNotIndexedTest, TestConfigSetsAPI, ExternalFileFieldSortTest, TestSolrDeletionPolicy2, DistributedSpellCheckComponentTest, TestFieldCacheSanityChecker, TestFoldingMultitermQuery, LeaderElectionContextKeyTest, TestLeaderElectionWithEmptyReplica, ConfigureRecoveryStrategyTest, NumericFieldsTest, TestNumericTerms32, SolrSlf4jReporterTest, TestClassNameShortening, TestFieldCacheVsDocValues, TestFaceting, TestManagedSchema, SyncSliceTest, TestDistributedSearch, TestBlendedInfixSuggestions, OverseerStatusTest, TestStressLucene, TestCloudJSONFacetJoinDomain, BasicDistributedZkTest, MBeansHandlerTest, TestPseudoReturnFields, BasicZkTest, HdfsUnloadDistributedZkTest, DistributedFacetPivotSmallAdvancedTest, OutputWriterTest, TestRandomDVFaceting, AutoCommitTest, TestHashPartitioner, TestReload, DistributedTermsComponentTest, TestCoreContainer, StatsComponentTest, TestFiltering, DistanceFunctionTest, CacheHeaderTest, LukeRequestHandlerTest, XmlUpdateRequestHandlerTest, TestSolrQueryParser, UpdateRequestProcessorFactoryTest, TestAnalyzedSuggestions, TestLFUCache, ActionThrottleTest, CdcrBootstrapTest, DeleteStatusTest, DistribJoinFromCollectionTest, LeaderFailoverAfterPartitionTest, MoveReplicaTest, OverseerCollectionConfigSetProcessorTest, OverseerModifyCollectionTest, PeerSyncReplicationTest, RemoteQueryErrorTest, ReplicationFactorTest, RollingRestartTest, SolrCLIZkUtilsTest, SolrXmlInZkTest, TestConfigSetsAPIZkFailure, TestCryptoKeys, TestLeaderElectionZkExpiry, TestRandomFlRTGCloud, HdfsCollectionsAPIDistributedZkTest, HdfsRecoveryZkTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=HdfsRecoveryZkTest -Dtests.seed=8D37D70DE1C2E387 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=en-US -Dtests.timezone=Pacific/Chatham -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J0 | HdfsRecoveryZkTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [HdfsTransactionLog]
   [junit4]    > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.update.HdfsTransactionLog
   [junit4]    > 	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
   [junit4]    > 	at org.apache.solr.update.HdfsTransactionLog.<init>(HdfsTransactionLog.java:130)
   [junit4]    > 	at org.apache.solr.update.HdfsUpdateLog.init(HdfsUpdateLog.java:203)
   [junit4]    > 	at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:153)
   [junit4]    > 	at org.apache.solr.update.UpdateHandler.<init>(UpdateHandler.java:110)
   [junit4]    > 	at org.apache.solr.update.DirectUpdateHandler2.<init>(DirectUpdateHandler2.java:108)
   [junit4]    > 	at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
   [junit4]    > 	at org.apache.solr.core.SolrCore.createInstance(SolrCore.java:760)
   [junit4]    > 	at org.apache.solr.core.SolrCore.createUpdateHandler(SolrCore.java:822)
   [junit4]    > 	at org.apache.solr.core.SolrCore.initUpdateHandler(SolrCore.java:1088)
   [junit4]    > 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:947)
   [junit4]    > 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:830)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:922)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.lambda$load$5(CoreContainer.java:564)
   [junit4]    > 	at com.codahale.metrics.InstrumentedExecutorService$InstrumentedCallable.call(InstrumentedExecutorService.java:197)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([8D37D70DE1C2E387]:0)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:302)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4] Completed [544/713 (2!)] on J0 in 41.51s, 1 test, 1 failure <<< FAILURES!

[...truncated 44018 lines...]

Mime
View raw message