lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Apache Jenkins Server <jenk...@builds.apache.org>
Subject [JENKINS] Lucene-Solr-BadApples-NightlyTests-7.x - Build # 33 - Failure
Date Tue, 30 Oct 2018 22:34:00 GMT
Build: https://builds.apache.org/job/Lucene-Solr-BadApples-NightlyTests-7.x/33/

11 tests failed.
FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:36095/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:36095/collection1
	at __randomizedtesting.SeedInfo.seed([27819D609D1F050:8A2C260CA72D9DA8]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1107)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:504)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:479)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1590)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:215)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1010)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:985)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:171)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
	... 53 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
4 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:     1) Thread[id=11700, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[27819D609D1F050]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)    2) Thread[id=11701, name=zkConnectionManagerCallback-3392-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    3) Thread[id=11699, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[27819D609D1F050]-SendThread(127.0.0.1:33747), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)    4) Thread[id=11698, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 4 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 
   1) Thread[id=11700, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[27819D609D1F050]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
   2) Thread[id=11701, name=zkConnectionManagerCallback-3392-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   3) Thread[id=11699, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[27819D609D1F050]-SendThread(127.0.0.1:33747), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
   4) Thread[id=11698, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([27819D609D1F050]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=11701, name=zkConnectionManagerCallback-3392-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    2) Thread[id=11699, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[27819D609D1F050]-SendThread(127.0.0.1:33747), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=11701, name=zkConnectionManagerCallback-3392-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   2) Thread[id=11699, name=TEST-ChaosMonkeyNothingIsSafeTest.test-seed#[27819D609D1F050]-SendThread(127.0.0.1:33747), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054)
	at __randomizedtesting.SeedInfo.seed([27819D609D1F050]:0)


FAILED:  org.apache.solr.cloud.api.collections.TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete

Error Message:
Error from server at http://127.0.0.1:43320/solr/testcollection_shard1_replica_n3: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/> <title>Error 404 Can not find: /solr/testcollection_shard1_replica_n3/update</title> </head> <body><h2>HTTP ERROR 404</h2> <p>Problem accessing /solr/testcollection_shard1_replica_n3/update. Reason: <pre>    Can not find: /solr/testcollection_shard1_replica_n3/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.11.v20180605</a><hr/>  </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at http://127.0.0.1:43320/solr/testcollection_shard1_replica_n3: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
<title>Error 404 Can not find: /solr/testcollection_shard1_replica_n3/update</title>
</head>
<body><h2>HTTP ERROR 404</h2>
<p>Problem accessing /solr/testcollection_shard1_replica_n3/update. Reason:
<pre>    Can not find: /solr/testcollection_shard1_replica_n3/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.11.v20180605</a><hr/>

</body>
</html>

	at __randomizedtesting.SeedInfo.seed([27819D609D1F050:A182B7738E391AF5]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:551)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1016)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:237)
	at org.apache.solr.cloud.api.collections.TestCollectionsAPIViaSolrCloudCluster.testCollectionCreateSearchDelete(TestCollectionsAPIViaSolrCloudCluster.java:170)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:43320/solr/testcollection_shard1_replica_n3: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
<title>Error 404 Can not find: /solr/testcollection_shard1_replica_n3/update</title>
</head>
<body><h2>HTTP ERROR 404</h2>
<p>Problem accessing /solr/testcollection_shard1_replica_n3/update. Reason:
<pre>    Can not find: /solr/testcollection_shard1_replica_n3/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.11.v20180605</a><hr/>

</body>
</html>

	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:607)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:528)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	... 1 more


FAILED:  org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest.testInactiveShardCleanup

Error Message:
missing cleanup event: [CapturedEvent{timestamp=2072443178755382, stage=STARTED, actionName='null', event={   "id":"75ce047444daaT5p58qo0hbelg2vmyg1fy6rtxb",   "source":".scheduled_maintenance",   "eventTime":2072443175062954,   "eventType":"SCHEDULED",   "properties":{     "actualEventTime":1540937657402,     "_enqueue_time_":2072443176907937}}, context={}, config={   "trigger":".scheduled_maintenance",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "beforeAction":"inactive_shard_plan",   "afterAction":"inactive_shard_plan",   "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072443180737590, stage=BEFORE_ACTION, actionName='inactive_shard_plan', event={   "id":"75ce047444daaT5p58qo0hbelg2vmyg1fy6rtxb",   "source":".scheduled_maintenance",   "eventTime":2072443175062954,   "eventType":"SCHEDULED",   "properties":{     "actualEventTime":1540937657402,     "_enqueue_time_":2072443176907937}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance}, config={   "trigger":".scheduled_maintenance",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "beforeAction":"inactive_shard_plan",   "afterAction":"inactive_shard_plan",   "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072443198284716, stage=AFTER_ACTION, actionName='inactive_shard_plan', event={   "id":"75ce047444daaT5p58qo0hbelg2vmyg1fy6rtxb",   "source":".scheduled_maintenance",   "eventTime":2072443175062954,   "eventType":"SCHEDULED",   "properties":{     "actualEventTime":1540937657402,     "_enqueue_time_":2072443176907937}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance, properties.inactive_shard_plan={staleLocks={ScheduledMaintenanceTriggerTest_collection1/staleShard-splitting={stateTimestamp=1540764857375462512, currentTimeNs=1540937657424373220, deltaSec=172800, ttlSec=20}}}, properties.AFTER_ACTION=[inactive_shard_plan, execute_plan, test]}, config={   "trigger":".scheduled_maintenance",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "beforeAction":"inactive_shard_plan",   "afterAction":"inactive_shard_plan",   "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072443201434624, stage=SUCCEEDED, actionName='null', event={   "id":"75ce047444daaT5p58qo0hbelg2vmyg1fy6rtxb",   "source":".scheduled_maintenance",   "eventTime":2072443175062954,   "eventType":"SCHEDULED",   "properties":{     "actualEventTime":1540937657402,     "_enqueue_time_":2072443176907937}}, context={}, config={   "trigger":".scheduled_maintenance",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "beforeAction":"inactive_shard_plan",   "afterAction":"inactive_shard_plan",   "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072448204734725, stage=STARTED, actionName='null', event={   "id":"75ce1730008abT5p58qo0hbelg2vmyg1fy6rtxh",   "source":".scheduled_maintenance",   "eventTime":2072448203753643,   "eventType":"SCHEDULED",   "properties":{     "actualEventTime":1540937662431,     "_enqueue_time_":2072448203980419}}, context={}, config={   "trigger":".scheduled_maintenance",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "beforeAction":"inactive_shard_plan",   "afterAction":"inactive_shard_plan",   "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072448205469030, stage=BEFORE_ACTION, actionName='inactive_shard_plan', event={   "id":"75ce1730008abT5p58qo0hbelg2vmyg1fy6rtxh",   "source":".scheduled_maintenance",   "eventTime":2072448203753643,   "eventType":"SCHEDULED",   "properties":{     "actualEventTime":1540937662431,     "_enqueue_time_":2072448203980419}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance}, config={   "trigger":".scheduled_maintenance",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "beforeAction":"inactive_shard_plan",   "afterAction":"inactive_shard_plan",   "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072448209499488, stage=AFTER_ACTION, actionName='inactive_shard_plan', event={   "id":"75ce1730008abT5p58qo0hbelg2vmyg1fy6rtxh",   "source":".scheduled_maintenance",   "eventTime":2072448203753643,   "eventType":"SCHEDULED",   "properties":{     "actualEventTime":1540937662431,     "_enqueue_time_":2072448203980419}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance, properties.AFTER_ACTION=[inactive_shard_plan, execute_plan, test]}, config={   "trigger":".scheduled_maintenance",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "beforeAction":"inactive_shard_plan",   "afterAction":"inactive_shard_plan",   "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072448210982445, stage=SUCCEEDED, actionName='null', event={   "id":"75ce1730008abT5p58qo0hbelg2vmyg1fy6rtxh",   "source":".scheduled_maintenance",   "eventTime":2072448203753643,   "eventType":"SCHEDULED",   "properties":{     "actualEventTime":1540937662431,     "_enqueue_time_":2072448203980419}}, context={}, config={   "trigger":".scheduled_maintenance",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "beforeAction":"inactive_shard_plan",   "afterAction":"inactive_shard_plan",   "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072453212386765, stage=STARTED, actionName='null', event={   "id":"75ce29d7bdb1bT5p58qo0hbelg2vmyg1fy6rtxi",   "source":".scheduled_maintenance",   "eventTime":2072453211478811,   "eventType":"SCHEDULED",   "properties":{     "actualEventTime":1540937667439,     "_enqueue_time_":2072453211731127}}, context={}, config={   "trigger":".scheduled_maintenance",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "beforeAction":"inactive_shard_plan",   "afterAction":"inactive_shard_plan",   "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072453213083124, stage=BEFORE_ACTION, actionName='inactive_shard_plan', event={   "id":"75ce29d7bdb1bT5p58qo0hbelg2vmyg1fy6rtxi",   "source":".scheduled_maintenance",   "eventTime":2072453211478811,   "eventType":"SCHEDULED",   "properties":{     "actualEventTime":1540937667439,     "_enqueue_time_":2072453211731127}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance}, config={   "trigger":".scheduled_maintenance",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "beforeAction":"inactive_shard_plan",   "afterAction":"inactive_shard_plan",   "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072453215766094, stage=AFTER_ACTION, actionName='inactive_shard_plan', event={   "id":"75ce29d7bdb1bT5p58qo0hbelg2vmyg1fy6rtxi",   "source":".scheduled_maintenance",   "eventTime":2072453211478811,   "eventType":"SCHEDULED",   "properties":{     "actualEventTime":1540937667439,     "_enqueue_time_":2072453211731127}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance, properties.AFTER_ACTION=[inactive_shard_plan, execute_plan, test]}, config={   "trigger":".scheduled_maintenance",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "beforeAction":"inactive_shard_plan",   "afterAction":"inactive_shard_plan",   "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072453217129932, stage=SUCCEEDED, actionName='null', event={   "id":"75ce29d7bdb1bT5p58qo0hbelg2vmyg1fy6rtxi",   "source":".scheduled_maintenance",   "eventTime":2072453211478811,   "eventType":"SCHEDULED",   "properties":{     "actualEventTime":1540937667439,     "_enqueue_time_":2072453211731127}}, context={}, config={   "trigger":".scheduled_maintenance",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "beforeAction":"inactive_shard_plan",   "afterAction":"inactive_shard_plan",   "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072458218763335, stage=STARTED, actionName='null', event={   "id":"75ce3c7e16cf7T5p58qo0hbelg2vmyg1fy6rtxj",   "source":".scheduled_maintenance",   "eventTime":2072458217745655,   "eventType":"SCHEDULED",   "properties":{     "actualEventTime":1540937672445,     "_enqueue_time_":2072458218015497}}, context={}, config={   "trigger":".scheduled_maintenance",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "beforeAction":"inactive_shard_plan",   "afterAction":"inactive_shard_plan",   "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072458219652691, stage=BEFORE_ACTION, actionName='inactive_shard_plan', event={   "id":"75ce3c7e16cf7T5p58qo0hbelg2vmyg1fy6rtxj",   "source":".scheduled_maintenance",   "eventTime":2072458217745655,   "eventType":"SCHEDULED",   "properties":{     "actualEventTime":1540937672445,     "_enqueue_time_":2072458218015497}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance}, config={   "trigger":".scheduled_maintenance",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "beforeAction":"inactive_shard_plan",   "afterAction":"inactive_shard_plan",   "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072458222208788, stage=AFTER_ACTION, actionName='inactive_shard_plan', event={   "id":"75ce3c7e16cf7T5p58qo0hbelg2vmyg1fy6rtxj",   "source":".scheduled_maintenance",   "eventTime":2072458217745655,   "eventType":"SCHEDULED",   "properties":{     "actualEventTime":1540937672445,     "_enqueue_time_":2072458218015497}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance, properties.AFTER_ACTION=[inactive_shard_plan, execute_plan, test]}, config={   "trigger":".scheduled_maintenance",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "beforeAction":"inactive_shard_plan",   "afterAction":"inactive_shard_plan",   "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072458223619204, stage=SUCCEEDED, actionName='null', event={   "id":"75ce3c7e16cf7T5p58qo0hbelg2vmyg1fy6rtxj",   "source":".scheduled_maintenance",   "eventTime":2072458217745655,   "eventType":"SCHEDULED",   "properties":{     "actualEventTime":1540937672445,     "_enqueue_time_":2072458218015497}}, context={}, config={   "trigger":".scheduled_maintenance",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "beforeAction":"inactive_shard_plan",   "afterAction":"inactive_shard_plan",   "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072463225175175, stage=STARTED, actionName='null', event={   "id":"75ce4f249e208T5p58qo0hbelg2vmyg1fy6rtxk",   "source":".scheduled_maintenance",   "eventTime":2072463224201736,   "eventType":"SCHEDULED",   "properties":{     "actualEventTime":1540937677451,     "_enqueue_time_":2072463224445698}}, context={}, config={   "trigger":".scheduled_maintenance",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "beforeAction":"inactive_shard_plan",   "afterAction":"inactive_shard_plan",   "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072463225893312, stage=BEFORE_ACTION, actionName='inactive_shard_plan', event={   "id":"75ce4f249e208T5p58qo0hbelg2vmyg1fy6rtxk",   "source":".scheduled_maintenance",   "eventTime":2072463224201736,   "eventType":"SCHEDULED",   "properties":{     "actualEventTime":1540937677451,     "_enqueue_time_":2072463224445698}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance}, config={   "trigger":".scheduled_maintenance",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "beforeAction":"inactive_shard_plan",   "afterAction":"inactive_shard_plan",   "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072463228827718, stage=AFTER_ACTION, actionName='inactive_shard_plan', event={   "id":"75ce4f249e208T5p58qo0hbelg2vmyg1fy6rtxk",   "source":".scheduled_maintenance",   "eventTime":2072463224201736,   "eventType":"SCHEDULED",   "properties":{     "actualEventTime":1540937677451,     "_enqueue_time_":2072463224445698}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance, properties.AFTER_ACTION=[inactive_shard_plan, execute_plan, test]}, config={   "trigger":".scheduled_maintenance",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "beforeAction":"inactive_shard_plan",   "afterAction":"inactive_shard_plan",   "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072463230727086, stage=SUCCEEDED, actionName='null', event={   "id":"75ce4f249e208T5p58qo0hbelg2vmyg1fy6rtxk",   "source":".scheduled_maintenance",   "eventTime":2072463224201736,   "eventType":"SCHEDULED",   "properties":{     "actualEventTime":1540937677451,     "_enqueue_time_":2072463224445698}}, context={}, config={   "trigger":".scheduled_maintenance",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "beforeAction":"inactive_shard_plan",   "afterAction":"inactive_shard_plan",   "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072468232425669, stage=STARTED, actionName='null', event={   "id":"75ce61cbc5d7aT5p58qo0hbelg2vmyg1fy6rtxl",   "source":".scheduled_maintenance",   "eventTime":2072468231314810,   "eventType":"SCHEDULED",   "properties":{     "actualEventTime":1540937682458,     "_enqueue_time_":2072468231602025}}, context={}, config={   "trigger":".scheduled_maintenance",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "beforeAction":"inactive_shard_plan",   "afterAction":"inactive_shard_plan",   "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072468233201881, stage=BEFORE_ACTION, actionName='inactive_shard_plan', event={   "id":"75ce61cbc5d7aT5p58qo0hbelg2vmyg1fy6rtxl",   "source":".scheduled_maintenance",   "eventTime":2072468231314810,   "eventType":"SCHEDULED",   "properties":{     "actualEventTime":1540937682458,     "_enqueue_time_":2072468231602025}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance}, config={   "trigger":".scheduled_maintenance",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "beforeAction":"inactive_shard_plan",   "afterAction":"inactive_shard_plan",   "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072468235525545, stage=AFTER_ACTION, actionName='inactive_shard_plan', event={   "id":"75ce61cbc5d7aT5p58qo0hbelg2vmyg1fy6rtxl",   "source":".scheduled_maintenance",   "eventTime":2072468231314810,   "eventType":"SCHEDULED",   "properties":{     "actualEventTime":1540937682458,     "_enqueue_time_":2072468231602025}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance, properties.inactive_shard_plan={staleLocks={ScheduledMaintenanceTriggerTest_collection1/shard1-splitting={stateTimestamp=1540937657422126363, currentTimeNs=1540937682462216654, deltaSec=25, ttlSec=20}}}, properties.AFTER_ACTION=[inactive_shard_plan, execute_plan, test]}, config={   "trigger":".scheduled_maintenance",   "stage":[     "STARTED",     "ABORTED",     "SUCCEEDED",     "FAILED"],   "beforeAction":"inactive_shard_plan",   "afterAction":"inactive_shard_plan",   "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}]

Stack Trace:
java.lang.AssertionError: missing cleanup event: [CapturedEvent{timestamp=2072443178755382, stage=STARTED, actionName='null', event={
  "id":"75ce047444daaT5p58qo0hbelg2vmyg1fy6rtxb",
  "source":".scheduled_maintenance",
  "eventTime":2072443175062954,
  "eventType":"SCHEDULED",
  "properties":{
    "actualEventTime":1540937657402,
    "_enqueue_time_":2072443176907937}}, context={}, config={
  "trigger":".scheduled_maintenance",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "beforeAction":"inactive_shard_plan",
  "afterAction":"inactive_shard_plan",
  "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072443180737590, stage=BEFORE_ACTION, actionName='inactive_shard_plan', event={
  "id":"75ce047444daaT5p58qo0hbelg2vmyg1fy6rtxb",
  "source":".scheduled_maintenance",
  "eventTime":2072443175062954,
  "eventType":"SCHEDULED",
  "properties":{
    "actualEventTime":1540937657402,
    "_enqueue_time_":2072443176907937}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance}, config={
  "trigger":".scheduled_maintenance",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "beforeAction":"inactive_shard_plan",
  "afterAction":"inactive_shard_plan",
  "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072443198284716, stage=AFTER_ACTION, actionName='inactive_shard_plan', event={
  "id":"75ce047444daaT5p58qo0hbelg2vmyg1fy6rtxb",
  "source":".scheduled_maintenance",
  "eventTime":2072443175062954,
  "eventType":"SCHEDULED",
  "properties":{
    "actualEventTime":1540937657402,
    "_enqueue_time_":2072443176907937}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance, properties.inactive_shard_plan={staleLocks={ScheduledMaintenanceTriggerTest_collection1/staleShard-splitting={stateTimestamp=1540764857375462512, currentTimeNs=1540937657424373220, deltaSec=172800, ttlSec=20}}}, properties.AFTER_ACTION=[inactive_shard_plan, execute_plan, test]}, config={
  "trigger":".scheduled_maintenance",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "beforeAction":"inactive_shard_plan",
  "afterAction":"inactive_shard_plan",
  "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072443201434624, stage=SUCCEEDED, actionName='null', event={
  "id":"75ce047444daaT5p58qo0hbelg2vmyg1fy6rtxb",
  "source":".scheduled_maintenance",
  "eventTime":2072443175062954,
  "eventType":"SCHEDULED",
  "properties":{
    "actualEventTime":1540937657402,
    "_enqueue_time_":2072443176907937}}, context={}, config={
  "trigger":".scheduled_maintenance",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "beforeAction":"inactive_shard_plan",
  "afterAction":"inactive_shard_plan",
  "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072448204734725, stage=STARTED, actionName='null', event={
  "id":"75ce1730008abT5p58qo0hbelg2vmyg1fy6rtxh",
  "source":".scheduled_maintenance",
  "eventTime":2072448203753643,
  "eventType":"SCHEDULED",
  "properties":{
    "actualEventTime":1540937662431,
    "_enqueue_time_":2072448203980419}}, context={}, config={
  "trigger":".scheduled_maintenance",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "beforeAction":"inactive_shard_plan",
  "afterAction":"inactive_shard_plan",
  "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072448205469030, stage=BEFORE_ACTION, actionName='inactive_shard_plan', event={
  "id":"75ce1730008abT5p58qo0hbelg2vmyg1fy6rtxh",
  "source":".scheduled_maintenance",
  "eventTime":2072448203753643,
  "eventType":"SCHEDULED",
  "properties":{
    "actualEventTime":1540937662431,
    "_enqueue_time_":2072448203980419}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance}, config={
  "trigger":".scheduled_maintenance",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "beforeAction":"inactive_shard_plan",
  "afterAction":"inactive_shard_plan",
  "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072448209499488, stage=AFTER_ACTION, actionName='inactive_shard_plan', event={
  "id":"75ce1730008abT5p58qo0hbelg2vmyg1fy6rtxh",
  "source":".scheduled_maintenance",
  "eventTime":2072448203753643,
  "eventType":"SCHEDULED",
  "properties":{
    "actualEventTime":1540937662431,
    "_enqueue_time_":2072448203980419}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance, properties.AFTER_ACTION=[inactive_shard_plan, execute_plan, test]}, config={
  "trigger":".scheduled_maintenance",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "beforeAction":"inactive_shard_plan",
  "afterAction":"inactive_shard_plan",
  "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072448210982445, stage=SUCCEEDED, actionName='null', event={
  "id":"75ce1730008abT5p58qo0hbelg2vmyg1fy6rtxh",
  "source":".scheduled_maintenance",
  "eventTime":2072448203753643,
  "eventType":"SCHEDULED",
  "properties":{
    "actualEventTime":1540937662431,
    "_enqueue_time_":2072448203980419}}, context={}, config={
  "trigger":".scheduled_maintenance",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "beforeAction":"inactive_shard_plan",
  "afterAction":"inactive_shard_plan",
  "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072453212386765, stage=STARTED, actionName='null', event={
  "id":"75ce29d7bdb1bT5p58qo0hbelg2vmyg1fy6rtxi",
  "source":".scheduled_maintenance",
  "eventTime":2072453211478811,
  "eventType":"SCHEDULED",
  "properties":{
    "actualEventTime":1540937667439,
    "_enqueue_time_":2072453211731127}}, context={}, config={
  "trigger":".scheduled_maintenance",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "beforeAction":"inactive_shard_plan",
  "afterAction":"inactive_shard_plan",
  "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072453213083124, stage=BEFORE_ACTION, actionName='inactive_shard_plan', event={
  "id":"75ce29d7bdb1bT5p58qo0hbelg2vmyg1fy6rtxi",
  "source":".scheduled_maintenance",
  "eventTime":2072453211478811,
  "eventType":"SCHEDULED",
  "properties":{
    "actualEventTime":1540937667439,
    "_enqueue_time_":2072453211731127}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance}, config={
  "trigger":".scheduled_maintenance",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "beforeAction":"inactive_shard_plan",
  "afterAction":"inactive_shard_plan",
  "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072453215766094, stage=AFTER_ACTION, actionName='inactive_shard_plan', event={
  "id":"75ce29d7bdb1bT5p58qo0hbelg2vmyg1fy6rtxi",
  "source":".scheduled_maintenance",
  "eventTime":2072453211478811,
  "eventType":"SCHEDULED",
  "properties":{
    "actualEventTime":1540937667439,
    "_enqueue_time_":2072453211731127}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance, properties.AFTER_ACTION=[inactive_shard_plan, execute_plan, test]}, config={
  "trigger":".scheduled_maintenance",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "beforeAction":"inactive_shard_plan",
  "afterAction":"inactive_shard_plan",
  "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072453217129932, stage=SUCCEEDED, actionName='null', event={
  "id":"75ce29d7bdb1bT5p58qo0hbelg2vmyg1fy6rtxi",
  "source":".scheduled_maintenance",
  "eventTime":2072453211478811,
  "eventType":"SCHEDULED",
  "properties":{
    "actualEventTime":1540937667439,
    "_enqueue_time_":2072453211731127}}, context={}, config={
  "trigger":".scheduled_maintenance",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "beforeAction":"inactive_shard_plan",
  "afterAction":"inactive_shard_plan",
  "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072458218763335, stage=STARTED, actionName='null', event={
  "id":"75ce3c7e16cf7T5p58qo0hbelg2vmyg1fy6rtxj",
  "source":".scheduled_maintenance",
  "eventTime":2072458217745655,
  "eventType":"SCHEDULED",
  "properties":{
    "actualEventTime":1540937672445,
    "_enqueue_time_":2072458218015497}}, context={}, config={
  "trigger":".scheduled_maintenance",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "beforeAction":"inactive_shard_plan",
  "afterAction":"inactive_shard_plan",
  "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072458219652691, stage=BEFORE_ACTION, actionName='inactive_shard_plan', event={
  "id":"75ce3c7e16cf7T5p58qo0hbelg2vmyg1fy6rtxj",
  "source":".scheduled_maintenance",
  "eventTime":2072458217745655,
  "eventType":"SCHEDULED",
  "properties":{
    "actualEventTime":1540937672445,
    "_enqueue_time_":2072458218015497}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance}, config={
  "trigger":".scheduled_maintenance",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "beforeAction":"inactive_shard_plan",
  "afterAction":"inactive_shard_plan",
  "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072458222208788, stage=AFTER_ACTION, actionName='inactive_shard_plan', event={
  "id":"75ce3c7e16cf7T5p58qo0hbelg2vmyg1fy6rtxj",
  "source":".scheduled_maintenance",
  "eventTime":2072458217745655,
  "eventType":"SCHEDULED",
  "properties":{
    "actualEventTime":1540937672445,
    "_enqueue_time_":2072458218015497}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance, properties.AFTER_ACTION=[inactive_shard_plan, execute_plan, test]}, config={
  "trigger":".scheduled_maintenance",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "beforeAction":"inactive_shard_plan",
  "afterAction":"inactive_shard_plan",
  "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072458223619204, stage=SUCCEEDED, actionName='null', event={
  "id":"75ce3c7e16cf7T5p58qo0hbelg2vmyg1fy6rtxj",
  "source":".scheduled_maintenance",
  "eventTime":2072458217745655,
  "eventType":"SCHEDULED",
  "properties":{
    "actualEventTime":1540937672445,
    "_enqueue_time_":2072458218015497}}, context={}, config={
  "trigger":".scheduled_maintenance",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "beforeAction":"inactive_shard_plan",
  "afterAction":"inactive_shard_plan",
  "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072463225175175, stage=STARTED, actionName='null', event={
  "id":"75ce4f249e208T5p58qo0hbelg2vmyg1fy6rtxk",
  "source":".scheduled_maintenance",
  "eventTime":2072463224201736,
  "eventType":"SCHEDULED",
  "properties":{
    "actualEventTime":1540937677451,
    "_enqueue_time_":2072463224445698}}, context={}, config={
  "trigger":".scheduled_maintenance",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "beforeAction":"inactive_shard_plan",
  "afterAction":"inactive_shard_plan",
  "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072463225893312, stage=BEFORE_ACTION, actionName='inactive_shard_plan', event={
  "id":"75ce4f249e208T5p58qo0hbelg2vmyg1fy6rtxk",
  "source":".scheduled_maintenance",
  "eventTime":2072463224201736,
  "eventType":"SCHEDULED",
  "properties":{
    "actualEventTime":1540937677451,
    "_enqueue_time_":2072463224445698}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance}, config={
  "trigger":".scheduled_maintenance",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "beforeAction":"inactive_shard_plan",
  "afterAction":"inactive_shard_plan",
  "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072463228827718, stage=AFTER_ACTION, actionName='inactive_shard_plan', event={
  "id":"75ce4f249e208T5p58qo0hbelg2vmyg1fy6rtxk",
  "source":".scheduled_maintenance",
  "eventTime":2072463224201736,
  "eventType":"SCHEDULED",
  "properties":{
    "actualEventTime":1540937677451,
    "_enqueue_time_":2072463224445698}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance, properties.AFTER_ACTION=[inactive_shard_plan, execute_plan, test]}, config={
  "trigger":".scheduled_maintenance",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "beforeAction":"inactive_shard_plan",
  "afterAction":"inactive_shard_plan",
  "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072463230727086, stage=SUCCEEDED, actionName='null', event={
  "id":"75ce4f249e208T5p58qo0hbelg2vmyg1fy6rtxk",
  "source":".scheduled_maintenance",
  "eventTime":2072463224201736,
  "eventType":"SCHEDULED",
  "properties":{
    "actualEventTime":1540937677451,
    "_enqueue_time_":2072463224445698}}, context={}, config={
  "trigger":".scheduled_maintenance",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "beforeAction":"inactive_shard_plan",
  "afterAction":"inactive_shard_plan",
  "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072468232425669, stage=STARTED, actionName='null', event={
  "id":"75ce61cbc5d7aT5p58qo0hbelg2vmyg1fy6rtxl",
  "source":".scheduled_maintenance",
  "eventTime":2072468231314810,
  "eventType":"SCHEDULED",
  "properties":{
    "actualEventTime":1540937682458,
    "_enqueue_time_":2072468231602025}}, context={}, config={
  "trigger":".scheduled_maintenance",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "beforeAction":"inactive_shard_plan",
  "afterAction":"inactive_shard_plan",
  "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072468233201881, stage=BEFORE_ACTION, actionName='inactive_shard_plan', event={
  "id":"75ce61cbc5d7aT5p58qo0hbelg2vmyg1fy6rtxl",
  "source":".scheduled_maintenance",
  "eventTime":2072468231314810,
  "eventType":"SCHEDULED",
  "properties":{
    "actualEventTime":1540937682458,
    "_enqueue_time_":2072468231602025}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance}, config={
  "trigger":".scheduled_maintenance",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "beforeAction":"inactive_shard_plan",
  "afterAction":"inactive_shard_plan",
  "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}, CapturedEvent{timestamp=2072468235525545, stage=AFTER_ACTION, actionName='inactive_shard_plan', event={
  "id":"75ce61cbc5d7aT5p58qo0hbelg2vmyg1fy6rtxl",
  "source":".scheduled_maintenance",
  "eventTime":2072468231314810,
  "eventType":"SCHEDULED",
  "properties":{
    "actualEventTime":1540937682458,
    "_enqueue_time_":2072468231602025}}, context={properties.BEFORE_ACTION=[inactive_shard_plan, execute_plan, test], source=.scheduled_maintenance, properties.inactive_shard_plan={staleLocks={ScheduledMaintenanceTriggerTest_collection1/shard1-splitting={stateTimestamp=1540937657422126363, currentTimeNs=1540937682462216654, deltaSec=25, ttlSec=20}}}, properties.AFTER_ACTION=[inactive_shard_plan, execute_plan, test]}, config={
  "trigger":".scheduled_maintenance",
  "stage":[
    "STARTED",
    "ABORTED",
    "SUCCEEDED",
    "FAILED"],
  "beforeAction":"inactive_shard_plan",
  "afterAction":"inactive_shard_plan",
  "class":"org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest$CapturingTriggerListener"}, message='null'}]
	at __randomizedtesting.SeedInfo.seed([27819D609D1F050:1F54D9A46892D75B]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.cloud.autoscaling.ScheduledMaintenanceTriggerTest.testInactiveShardCleanup(ScheduledMaintenanceTriggerTest.java:253)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:38640/x_okt/i/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:38640/x_okt/i/collection1
	at __randomizedtesting.SeedInfo.seed([27819D609D1F050:8A2C260CA72D9DA8]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:654)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1107)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:504)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:479)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1590)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:215)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1010)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:985)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:171)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
	... 53 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest

Error Message:
4 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest:     1) Thread[id=17993, name=TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[27819D609D1F050]-EventThread, state=WAITING, group=TGRP-HdfsChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)    2) Thread[id=17992, name=TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[27819D609D1F050]-SendThread(127.0.0.1:40136), state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)    3) Thread[id=17994, name=zkConnectionManagerCallback-3466-thread-1, state=WAITING, group=TGRP-HdfsChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)    4) Thread[id=17991, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 4 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest: 
   1) Thread[id=17993, name=TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[27819D609D1F050]-EventThread, state=WAITING, group=TGRP-HdfsChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
   2) Thread[id=17992, name=TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[27819D609D1F050]-SendThread(127.0.0.1:40136), state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
   3) Thread[id=17994, name=zkConnectionManagerCallback-3466-thread-1, state=WAITING, group=TGRP-HdfsChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
   4) Thread[id=17991, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([27819D609D1F050]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=17992, name=TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[27819D609D1F050]-SendThread(127.0.0.1:40136), state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeyNothingIsSafeTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054)    2) Thread[id=17994, name=zkConnectionManagerCallback-3466-thread-1, state=WAITING, group=TGRP-HdfsChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=17992, name=TEST-HdfsChaosMonkeyNothingIsSafeTest.test-seed#[27819D609D1F050]-SendThread(127.0.0.1:40136), state=TIMED_WAITING, group=TGRP-HdfsChaosMonkeyNothingIsSafeTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1054)
   2) Thread[id=17994, name=zkConnectionManagerCallback-3466-thread-1, state=WAITING, group=TGRP-HdfsChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([27819D609D1F050]:0)


FAILED:  org.apache.solr.cloud.hdfs.HdfsRestartWhileUpdatingTest.test

Error Message:
There are still nodes recoverying - waited for 320 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 320 seconds
	at __randomizedtesting.SeedInfo.seed([27819D609D1F050:8A2C260CA72D9DA8]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:920)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1477)
	at org.apache.solr.cloud.RestartWhileUpdatingTest.test(RestartWhileUpdatingTest.java:145)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1010)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:985)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.uninverting.TestDocTermOrdsUninvertLimit.testTriggerUnInvertLimit

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space
	at __randomizedtesting.SeedInfo.seed([27819D609D1F050:31CA311204662AE7]:0)
	at org.apache.lucene.store.RAMFile.newBuffer(RAMFile.java:84)
	at org.apache.lucene.store.RAMFile.addBuffer(RAMFile.java:57)
	at org.apache.lucene.store.RAMOutputStream.switchCurrentBuffer(RAMOutputStream.java:168)
	at org.apache.lucene.store.RAMOutputStream.writeBytes(RAMOutputStream.java:154)
	at org.apache.lucene.store.DataOutput.copyBytes(DataOutput.java:278)
	at org.apache.lucene.store.MockIndexOutputWrapper.copyBytes(MockIndexOutputWrapper.java:165)
	at org.apache.lucene.codecs.simpletext.SimpleTextCompoundFormat.write(SimpleTextCompoundFormat.java:186)
	at org.apache.lucene.index.IndexWriter.createCompoundFile(IndexWriter.java:4997)
	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4510)
	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4068)
	at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
	at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2170)
	at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5103)
	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1612)
	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1228)
	at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:189)
	at org.apache.solr.uninverting.TestDocTermOrdsUninvertLimit.testTriggerUnInvertLimit(TestDocTermOrdsUninvertLimit.java:66)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)


FAILED:  org.apache.solr.client.solrj.impl.CloudSolrClientTest.testVersionsAreReturned

Error Message:
Error from server at https://127.0.0.1:41935/solr/collection1_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/> <title>Error 404 Can not find: /solr/collection1_shard2_replica_n2/update</title> </head> <body><h2>HTTP ERROR 404</h2> <p>Problem accessing /solr/collection1_shard2_replica_n2/update. Reason: <pre>    Can not find: /solr/collection1_shard2_replica_n2/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.11.v20180605</a><hr/>  </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at https://127.0.0.1:41935/solr/collection1_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
<title>Error 404 Can not find: /solr/collection1_shard2_replica_n2/update</title>
</head>
<body><h2>HTTP ERROR 404</h2>
<p>Problem accessing /solr/collection1_shard2_replica_n2/update. Reason:
<pre>    Can not find: /solr/collection1_shard2_replica_n2/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.11.v20180605</a><hr/>

</body>
</html>

	at __randomizedtesting.SeedInfo.seed([9919E1BF1324B5F4:61DF188A803D2D3C]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:551)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1016)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
	at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:237)
	at org.apache.solr.client.solrj.impl.CloudSolrClientTest.testVersionsAreReturned(CloudSolrClientTest.java:725)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
	at org.junit.rules.ExpectedException$ExpectedExceptionStatement.evaluate(ExpectedException.java:110)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apa

[...truncated too long message...]


   [junit4]   1> </body>
   [junit4]   1> </html>
   [junit4]   1> , retry? 3
   [junit4]   1> 22:26:38.424 [TEST-CloudSolrClientTest.testVersionsAreReturned-seed#[9919E1BF1324B5F4]] ERROR org.apache.solr.client.solrj.impl.CloudSolrClient - Request to collection [collection1] failed due to (404) org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:41935/solr/collection1_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
   [junit4]   1> <head>
   [junit4]   1> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
   [junit4]   1> <title>Error 404 Can not find: /solr/collection1_shard2_replica_n2/update</title>
   [junit4]   1> </head>
   [junit4]   1> <body><h2>HTTP ERROR 404</h2>
   [junit4]   1> <p>Problem accessing /solr/collection1_shard2_replica_n2/update. Reason:
   [junit4]   1> <pre>    Can not find: /solr/collection1_shard2_replica_n2/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.11.v20180605</a><hr/>
   [junit4]   1> 
   [junit4]   1> </body>
   [junit4]   1> </html>
   [junit4]   1> , retry? 4
   [junit4]   1> 22:26:38.430 [TEST-CloudSolrClientTest.testVersionsAreReturned-seed#[9919E1BF1324B5F4]] ERROR org.apache.solr.client.solrj.impl.CloudSolrClient - Request to collection [collection1] failed due to (404) org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:41935/solr/collection1_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
   [junit4]   1> <head>
   [junit4]   1> <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
   [junit4]   1> <title>Error 404 Can not find: /solr/collection1_shard2_replica_n2/update</title>
   [junit4]   1> </head>
   [junit4]   1> <body><h2>HTTP ERROR 404</h2>
   [junit4]   1> <p>Problem accessing /solr/collection1_shard2_replica_n2/update. Reason:
   [junit4]   1> <pre>    Can not find: /solr/collection1_shard2_replica_n2/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.11.v20180605</a><hr/>
   [junit4]   1> 
   [junit4]   1> </body>
   [junit4]   1> </html>
   [junit4]   1> , retry? 5
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=CloudSolrClientTest -Dtests.method=testVersionsAreReturned -Dtests.seed=9919E1BF1324B5F4 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.badapples=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/test-data/enwiki.random.lines.txt -Dtests.locale=de-DE -Dtests.timezone=America/Indiana/Petersburg -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   4.66s J2 | CloudSolrClientTest.testVersionsAreReturned <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at https://127.0.0.1:41935/solr/collection1_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
   [junit4]    > <head>
   [junit4]    > <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
   [junit4]    > <title>Error 404 Can not find: /solr/collection1_shard2_replica_n2/update</title>
   [junit4]    > </head>
   [junit4]    > <body><h2>HTTP ERROR 404</h2>
   [junit4]    > <p>Problem accessing /solr/collection1_shard2_replica_n2/update. Reason:
   [junit4]    > <pre>    Can not find: /solr/collection1_shard2_replica_n2/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.11.v20180605</a><hr/>
   [junit4]    > </body>
   [junit4]    > </html>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([9919E1BF1324B5F4:61DF188A803D2D3C]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:551)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1016)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:949)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
   [junit4]    > 	at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:237)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClientTest.testVersionsAreReturned(CloudSolrClientTest.java:725)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]    > Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:41935/solr/collection1_shard2_replica_n2: Expected mime type application/octet-stream but got text/html. <html>
   [junit4]    > <head>
   [junit4]    > <meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>
   [junit4]    > <title>Error 404 Can not find: /solr/collection1_shard2_replica_n2/update</title>
   [junit4]    > </head>
   [junit4]    > <body><h2>HTTP ERROR 404</h2>
   [junit4]    > <p>Problem accessing /solr/collection1_shard2_replica_n2/update. Reason:
   [junit4]    > <pre>    Can not find: /solr/collection1_shard2_replica_n2/update</pre></p><hr><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.4.11.v20180605</a><hr/>
   [junit4]    > </body>
   [junit4]    > </html>
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:607)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:528)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    > 	... 1 more
   [junit4]   1> 22:26:51.151 [SUITE-CloudSolrClientTest-seed#[9919E1BF1324B5F4]-worker] ERROR org.apache.zookeeper.server.ZooKeeperServer - ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/solr/build/solr-solrj/test/J2/temp/solr.client.solrj.impl.CloudSolrClientTest_9919E1BF1324B5F4-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {multiDefault=Lucene50(blocksize=128), a_t=Lucene50(blocksize=128), title_s=Lucene50(blocksize=128), id=FST50, text=PostingsFormat(name=Memory)}, docValues:{_version_=DocValuesFormat(name=Memory), multiDefault=DocValuesFormat(name=Lucene70), a_t=DocValuesFormat(name=Lucene70), title_s=DocValuesFormat(name=Lucene70), intDefault=DocValuesFormat(name=Memory), id=DocValuesFormat(name=Lucene70), text=DocValuesFormat(name=Direct), timestamp=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=1316, maxMBSortInHeap=5.869300803513228, sim=RandomSimilarity(queryNorm=false): {}, locale=de-DE, timezone=America/Indiana/Petersburg
   [junit4]   2> NOTE: Linux 4.4.0-137-generic amd64/Oracle Corporation 1.8.0_172 (64-bit)/cpus=4,threads=1,free=179963528,total=438829056
   [junit4]   2> NOTE: All tests run in this JVM: [HttpSolrClientBadInputTest, SolrExampleStreamingTest, ConcurrentUpdateSolrClientMultiCollectionTest, QueryResponseTest, LessThanEvaluatorTest, NotEvaluatorTest, TestLang, JdbcDriverTest, TestSolrJErrorHandling, ClientUtilsTest, AscEvaluatorTest, EmpiricalDistributionEvaluatorTest, StreamingTest, TestDelegationTokenResponse, ArcSineEvaluatorTest, SolrExampleXMLTest, FieldValueEvaluatorTest, AndEvaluatorTest, TestPathTrie, SquareRootEvaluatorTest, LargeVolumeEmbeddedTest, CollectionAdminRequestRequiredParamsTest, StreamExpressionParserTest, MathExpressionTest, StreamExpressionTest, FacetFieldTest, CloudSolrClientTest]
   [junit4] Completed [140/162 (1!)] on J2 in 72.71s, 16 tests, 1 error <<< FAILURES!

[...truncated 7950 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/build.xml:661: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-BadApples-NightlyTests-7.x/checkout/build.xml:585: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid22393.hprof

Total time: 281 minutes 30 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any

Mime
View raw message