lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Policeman Jenkins Server <jenk...@thetaphi.de>
Subject [JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 373 - Still Unstable!
Date Thu, 04 Jan 2018 17:04:40 GMT
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/373/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC

3 tests failed.
FAILED:  org.apache.solr.cloud.autoscaling.sim.TestExecutePlanAction.testIntegration

Error Message:


Stack Trace:
java.util.ConcurrentModificationException
	at __randomizedtesting.SeedInfo.seed([C48B7B3B457DBD72:74EA751760421C57]:0)
	at java.util.ArrayList$Itr.checkForComodification(ArrayList.java:909)
	at java.util.ArrayList$Itr.next(ArrayList.java:859)
	at org.apache.solr.cloud.autoscaling.sim.SimSolrCloudTestCase.tearDown(SimSolrCloudTestCase.java:141)
	at sun.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTriggerRestoreState

Error Message:


Stack Trace:
java.util.ConcurrentModificationException
	at __randomizedtesting.SeedInfo.seed([C48B7B3B457DBD72:4CB6F2447FBD5CDF]:0)
	at java.util.ArrayList$Itr.checkForComodification(ArrayList.java:909)
	at java.util.ArrayList$Itr.next(ArrayList.java:859)
	at org.apache.solr.cloud.autoscaling.sim.SimSolrCloudTestCase.tearDown(SimSolrCloudTestCase.java:141)
	at sun.reflect.GeneratedMethodAccessor33.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:992)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.client.solrj.io.stream.StreamExpressionTest.testGammaDistribution

Error Message:


Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([1918B1C95D2400E1:24629A677E5CAAF6]:0)
	at org.junit.Assert.fail(Assert.java:92)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertTrue(Assert.java:54)
	at org.apache.solr.client.solrj.io.stream.StreamExpressionTest.testGammaDistribution(StreamExpressionTest.java:7808)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 11936 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestExecutePlanAction
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.sim.TestExecutePlanAction_C48B7B3B457DBD72-001/init-core-data-001
   [junit4]   2> 678206 INFO  (SUITE-TestExecutePlanAction-seed#[C48B7B3B457DBD72]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 678206 INFO  (SUITE-TestExecutePlanAction-seed#[C48B7B3B457DBD72]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 678206 INFO  (SUITE-TestExecutePlanAction-seed#[C48B7B3B457DBD72]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 678230 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 678230 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0
   [junit4]   2> 678230 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr]
   [junit4]   2> 678230 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
   [junit4]   2> 678230 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
   [junit4]   2> 678235 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 678235 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
   [junit4]   2> 678258 DEBUG (ScheduledTrigger-1711-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678260 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testExecute
   [junit4]   2> 678260 DEBUG (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 678260 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 678263 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 678263 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 678263 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 678264 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 678264 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 678269 DEBUG (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 678269 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 678269 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
   [junit4]   2> 678270 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 678270 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
   [junit4]   2> 678270 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 678270 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 678270 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 678270 DEBUG (ScheduledTrigger-1714-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678290 DEBUG (ScheduledTrigger-1714-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678314 DEBUG (ScheduledTrigger-1714-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678337 DEBUG (ScheduledTrigger-1714-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678358 DEBUG (ScheduledTrigger-1714-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678373 DEBUG (simCloudManagerPool-1713-thread-1) [    ] o.a.s.c.o.ClusterStateMutator building a new cName: testExecute
   [junit4]   2> 678378 DEBUG (ScheduledTrigger-1714-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678386 DEBUG (simCloudManagerPool-1713-thread-1) [    ] o.a.s.c.Assign Identify nodes using default
   [junit4]   2> 678398 DEBUG (ScheduledTrigger-1714-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678416 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.TestExecutePlanAction Collection ready after 1156ms
   [junit4]   2> 678418 DEBUG (ScheduledTrigger-1714-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678420 DEBUG (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.ExecutePlanAction -- processing event: {
   [junit4]   2>   "id":"1506a604cef519c0T5y74lvmt9hwilvifly8sh3hko",
   [junit4]   2>   "source":"mock_trigger_name",
   [junit4]   2>   "eventTime":1515080864239000000,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[1515080864239000000],
   [junit4]   2>     "nodeNames":["127.0.0.1:10000_solr"]}} with context properties: {operations=[org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@57fba7cb, org.apache.solr.cloud.autoscaling.sim.TestExecutePlanAction$1@74d02a4d]}
   [junit4]   2> 678420 DEBUG (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testExecute&targetNode=127.0.0.1:10001_solr&inPlaceMove=true&replica=core_node1
   [junit4]   2> 678438 DEBUG (ScheduledTrigger-1714-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678464 DEBUG (ScheduledTrigger-1714-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678484 DEBUG (ScheduledTrigger-1714-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678504 DEBUG (ScheduledTrigger-1714-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678521 DEBUG (simCloudManagerPool-1713-thread-4) [    ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node4":{
   [junit4]   2>     "core":"testExecute_shard1_replica_n3",
   [junit4]   2>     "shard":"shard1",
   [junit4]   2>     "collection":"testExecute",
   [junit4]   2>     "node_name":"127.0.0.1:10001_solr",
   [junit4]   2>     "type":"NRT"}}
   [junit4]   2> 678524 DEBUG (ScheduledTrigger-1714-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678550 DEBUG (ScheduledTrigger-1714-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678574 DEBUG (ScheduledTrigger-1714-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678596 DEBUG (ScheduledTrigger-1714-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678616 DEBUG (ScheduledTrigger-1714-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678636 DEBUG (ScheduledTrigger-1714-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678639 DEBUG (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.ExecutePlanAction Executing operation: action=OVERSEERSTATUS
   [junit4]   2> 678643 DEBUG (simCloudManagerPool-1713-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testExecute / shard1: {"core_node2":{
   [junit4]   2>     "node_name":"127.0.0.1:10001_solr",
   [junit4]   2>     "core":"testExecute_shard1_replica_n2",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "INDEX.sizeInBytes":123450000,
   [junit4]   2>     "shard":"shard1",
   [junit4]   2>     "collection":"testExecute"}}
   [junit4]   2> 678644 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.TestExecutePlanAction Collection ready after 7ms
   [junit4]   2> 678644 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.TestExecutePlanAction -------------_ FINAL STATE --------------
   [junit4]   2> 678644 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.TestExecutePlanAction * Node values: {
   [junit4]   2>   "127.0.0.1:10001_solr":{
   [junit4]   2>     "sysprop.java.vendor":"Oracle Corporation",
   [junit4]   2>     "cores":2,
   [junit4]   2>     "host":"127.0.0.1",
   [junit4]   2>     "freedisk":980,
   [junit4]   2>     "sysprop.java.version":"1.8.0_152",
   [junit4]   2>     "ip_1":"127",
   [junit4]   2>     "ip_3":"0",
   [junit4]   2>     "ip_2":"0",
   [junit4]   2>     "ip_4":"1",
   [junit4]   2>     "node":"127.0.0.1:10001_solr",
   [junit4]   2>     "metrics:solr.jvm:buffers.direct.Count":0,
   [junit4]   2>     "port":10001,
   [junit4]   2>     "heapUsage":123450000,
   [junit4]   2>     "sysLoadAvg":1.0,
   [junit4]   2>     "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0},
   [junit4]   2>   "127.0.0.1:10000_solr":{
   [junit4]   2>     "sysprop.java.vendor":"Oracle Corporation",
   [junit4]   2>     "cores":0,
   [junit4]   2>     "host":"127.0.0.1",
   [junit4]   2>     "freedisk":1000,
   [junit4]   2>     "sysprop.java.version":"1.8.0_152",
   [junit4]   2>     "ip_1":"127",
   [junit4]   2>     "ip_3":"0",
   [junit4]   2>     "ip_2":"0",
   [junit4]   2>     "ip_4":"1",
   [junit4]   2>     "node":"127.0.0.1:10000_solr",
   [junit4]   2>     "metrics:solr.jvm:buffers.direct.Count":0,
   [junit4]   2>     "port":10000,
   [junit4]   2>     "heapUsage":123450000,
   [junit4]   2>     "sysLoadAvg":1.0,
   [junit4]   2>     "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}}
   [junit4]   2> 678644 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.TestExecutePlanAction * Live nodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 678645 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.TestExecutePlanAction * Collection testExecute state: DocCollection(testExecute//clusterstate.json/3)={
   [junit4]   2>   "replicationFactor":"2",
   [junit4]   2>   "pullReplicas":"0",
   [junit4]   2>   "router":{"name":"compositeId"},
   [junit4]   2>   "maxShardsPerNode":"1",
   [junit4]   2>   "autoAddReplicas":"false",
   [junit4]   2>   "nrtReplicas":"2",
   [junit4]   2>   "tlogReplicas":"0",
   [junit4]   2>   "autoCreated":"true",
   [junit4]   2>   "shards":{"shard1":{
   [junit4]   2>       "replicas":{
   [junit4]   2>         "core_node2":{
   [junit4]   2>           "core":"testExecute_shard1_replica_n2",
   [junit4]   2>           "leader":"true",
   [junit4]   2>           "INDEX.sizeInBytes":123450000,
   [junit4]   2>           "node_name":"127.0.0.1:10001_solr",
   [junit4]   2>           "state":"active",
   [junit4]   2>           "type":"NRT"},
   [junit4]   2>         "core_node4":{
   [junit4]   2>           "node_name":"127.0.0.1:10001_solr",
   [junit4]   2>           "core":"testExecute_shard1_replica_n3",
   [junit4]   2>           "state":"active",
   [junit4]   2>           "INDEX.sizeInBytes":123450000,
   [junit4]   2>           "type":"NRT"}},
   [junit4]   2>       "range":"80000000-7fffffff",
   [junit4]   2>       "state":"active"}}}
   [junit4]   2> 678645 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testExecute
   [junit4]   2> 678645 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 678645 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 678645 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 678645 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 678645 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:		2
   [junit4]   2> 678646 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:	1
   [junit4]   2> 678646 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:		0
   [junit4]   2> 678647 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:	[testExecute]
   [junit4]   2> 678647 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:	2
   [junit4]   2> 678647 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:	0
   [junit4]   2> 678647 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:		2
   [junit4]   2> 678647 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## * testExecute		2
   [junit4]   2> 678648 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- active           2
   [junit4]   2> 678648 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 678648 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- CREATE             1
   [junit4]   2> 678648 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- DELETESTATUS       2
   [junit4]   2> 678648 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- MOVEREPLICA        1
   [junit4]   2> 678648 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- OVERSEERSTATUS     1
   [junit4]   2> 678648 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- REQUESTSTATUS      2
   [junit4]   2> 678649 INFO  (TEST-TestExecutePlanAction.testExecute-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
   [junit4]   2> 678658 DEBUG (ScheduledTrigger-1714-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678680 DEBUG (ScheduledTrigger-1714-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678688 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testIntegration
   [junit4]   2> 678688 DEBUG (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
   [junit4]   2> 678688 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 678688 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 678688 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 678688 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 678693 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1220)
   [junit4]   2> 	at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:154)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 678693 DEBUG (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 678694 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 678694 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
   [junit4]   2> 678694 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 678694 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion -1
   [junit4]   2> 678694 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
   [junit4]   2> 678694 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 678694 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
   [junit4]   2> 678694 DEBUG (ScheduledTrigger-1717-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678715 DEBUG (ScheduledTrigger-1717-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678735 DEBUG (ScheduledTrigger-1717-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678760 DEBUG (ScheduledTrigger-1717-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678780 DEBUG (ScheduledTrigger-1717-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678800 DEBUG (ScheduledTrigger-1717-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678813 DEBUG (simCloudManagerPool-1716-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 678813 DEBUG (simCloudManagerPool-1716-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5
   [junit4]   2> 678816 DEBUG (simCloudManagerPool-1716-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 678816 DEBUG (simCloudManagerPool-1716-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
   [junit4]   2> 678816 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
   [junit4]   2> 678817 DEBUG (simCloudManagerPool-1716-thread-2) [    ] o.a.s.c.o.ClusterStateMutator building a new cName: testIntegration
   [junit4]   2> 678817 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 678817 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5
   [junit4]   2> 678817 DEBUG (ScheduledTrigger-1717-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
   [junit4]   2> 678820 DEBUG (ScheduledTrigger-1717-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678827 DEBUG (simCloudManagerPool-1716-thread-2) [    ] o.a.s.c.Assign Identify nodes using default
   [junit4]   2> 678841 DEBUG (ScheduledTrigger-1717-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 678841 DEBUG (ScheduledTrigger-1717-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
   [junit4]   2> 678858 DEBUG (simCloudManagerPool-1716-thread-5) [    ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testIntegration / shard1: {"core_node1":{
   [junit4]   2>     "node_name":"127.0.0.1:10000_solr",
   [junit4]   2>     "core":"testIntegration_shard1_replica_n1",
   [junit4]   2>     "leader":"true",
   [junit4]   2>     "state":"active",
   [junit4]   2>     "type":"NRT",
   [junit4]   2>     "INDEX.sizeInBytes":123450000,
   [junit4]   2>     "shard":"shard1",
   [junit4]   2>     "collection":"testIntegration"}}
   [junit4]   2> 678862 DEBUG (ScheduledTrigger-1717-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 678862 DEBUG (ScheduledTrigger-1717-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10001_solr
   [junit4]   2> 678862 DEBUG (ScheduledTrigger-1717-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 1
   [junit4]   2> 678862 DEBUG (ScheduledTrigger-1717-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10001_solr
   [junit4]   2> 678883 DEBUG (ScheduledTrigger-1717-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 678883 DEBUG (ScheduledTrigger-1717-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 1
   [junit4]   2> 678883 DEBUG (ScheduledTrigger-1717-thread-4) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10001_solr]
   [junit4]   2> 678883 DEBUG (ScheduledTrigger-1717-thread-4) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"a091c092378ecT5y74lvmt9hwilvifly8sh3hkp",
   [junit4]   2>   "source":"node_lost_trigger",
   [junit4]   2>   "eventTime":2824765784160492,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2824765784160492],
   [junit4]   2>     "nodeNames":["127.0.0.1:10001_solr"]}}
   [junit4]   2> 678885 DEBUG (AutoscalingActionExecutor-1718-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"a091c092378ecT5y74lvmt9hwilvifly8sh3hkp",
   [junit4]   2>   "source":"node_lost_trigger",
   [junit4]   2>   "eventTime":2824765784160492,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2824765784160492],
   [junit4]   2>     "_enqueue_time_":2824766842661792,
   [junit4]   2>     "nodeNames":["127.0.0.1:10001_solr"]}}
   [junit4]   2> 678886 DEBUG (AutoscalingActionExecutor-1718-thread-1) [    ] o.a.s.c.a.ComputePlanAction -- processing event: {
   [junit4]   2>   "id":"a091c092378ecT5y74lvmt9hwilvifly8sh3hkp",
   [junit4]   2>   "source":"node_lost_trigger",
   [junit4]   2>   "eventTime":2824765784160492,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2824765784160492],
   [junit4]   2>     "_enqueue_time_":2824766842661792,
   [junit4]   2>     "nodeNames":["127.0.0.1:10001_solr"]}} with context properties: {BEFORE_ACTION=[compute_plan]}
   [junit4]   2> 678887 INFO  (AutoscalingActionExecutor-1718-thread-1) [    ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testIntegration&targetNode=127.0.0.1:10000_solr&inPlaceMove=true&replica=core_node2
   [junit4]   2> 678887 INFO  (AutoscalingActionExecutor-1718-thread-1) [    ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 2824767046 sessionWrapper.createTime 2824767035468342, this.sessionWrapper.createTime 2824767035468342 
   [junit4]   2> 678887 DEBUG (AutoscalingActionExecutor-1718-thread-1) [    ] o.a.s.c.a.ExecutePlanAction -- processing event: {
   [junit4]   2>   "id":"a091c092378ecT5y74lvmt9hwilvifly8sh3hkp",
   [junit4]   2>   "source":"node_lost_trigger",
   [junit4]   2>   "eventTime":2824765784160492,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2824765784160492],
   [junit4]   2>     "_enqueue_time_":2824766842661792,
   [junit4]   2>     "nodeNames":["127.0.0.1:10001_solr"]}} with context properties: {operations=[org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@760b1e58], AFTER_ACTION=[compute_plan], BEFORE_ACTION=[compute_plan, execute_plan]}
   [junit4]   2> 678887 DEBUG (AutoscalingActionExecutor-1718-thread-1) [    ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testIntegration&targetNode=127.0.0.1:10000_solr&inPlaceMove=true&replica=core_node2
   [junit4]   2> 678994 DEBUG (simCloudManagerPool-1716-thread-7) [    ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node4":{
   [junit4]   2>     "core":"testIntegration_shard1_replica_n3",
   [junit4]   2>     "shard":"shard1",
   [junit4]   2>     "collection":"testIntegration",
   [junit4]   2>     "node_name":"127.0.0.1:10000_solr",
   [junit4]   2>     "type":"NRT"}}
   [junit4]   2> 679111 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.TestExecutePlanAction -------------_ FINAL STATE --------------
   [junit4]   2> 679112 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.TestExecutePlanAction * Node values: {
   [junit4]   2>   "127.0.0.1:10001_solr":{
   [junit4]   2>     "sysprop.java.vendor":"Oracle Corporation",
   [junit4]   2>     "cores":1,
   [junit4]   2>     "host":"127.0.0.1",
   [junit4]   2>     "freedisk":970,
   [junit4]   2>     "sysprop.java.version":"1.8.0_152",
   [junit4]   2>     "ip_1":"127",
   [junit4]   2>     "ip_3":"0",
   [junit4]   2>     "ip_2":"0",
   [junit4]   2>     "ip_4":"1",
   [junit4]   2>     "node":"127.0.0.1:10001_solr",
   [junit4]   2>     "metrics:solr.jvm:buffers.direct.Count":0,
   [junit4]   2>     "port":10001,
   [junit4]   2>     "heapUsage":123450000,
   [junit4]   2>     "sysLoadAvg":1.0,
   [junit4]   2>     "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0},
   [junit4]   2>   "127.0.0.1:10000_solr":{
   [junit4]   2>     "sysprop.java.vendor":"Oracle Corporation",
   [junit4]   2>     "cores":2,
   [junit4]   2>     "host":"127.0.0.1",
   [junit4]   2>     "freedisk":980,
   [junit4]   2>     "sysprop.java.version":"1.8.0_152",
   [junit4]   2>     "ip_1":"127",
   [junit4]   2>     "ip_3":"0",
   [junit4]   2>     "ip_2":"0",
   [junit4]   2>     "ip_4":"1",
   [junit4]   2>     "node":"127.0.0.1:10000_solr",
   [junit4]   2>     "metrics:solr.jvm:buffers.direct.Count":0,
   [junit4]   2>     "port":10000,
   [junit4]   2>     "heapUsage":123450000,
   [junit4]   2>     "sysLoadAvg":1.0,
   [junit4]   2>     "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}}
   [junit4]   2> 679112 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.TestExecutePlanAction * Live nodes: [127.0.0.1:10000_solr]
   [junit4]   2> 679112 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.TestExecutePlanAction * Collection testIntegration state: DocCollection(testIntegration//clusterstate.json/10)={
   [junit4]   2>   "replicationFactor":"2",
   [junit4]   2>   "pullReplicas":"0",
   [junit4]   2>   "router":{"name":"compositeId"},
   [junit4]   2>   "maxShardsPerNode":"1",
   [junit4]   2>   "autoAddReplicas":"false",
   [junit4]   2>   "nrtReplicas":"2",
   [junit4]   2>   "tlogReplicas":"0",
   [junit4]   2>   "autoCreated":"true",
   [junit4]   2>   "shards":{"shard1":{
   [junit4]   2>       "replicas":{
   [junit4]   2>         "core_node1":{
   [junit4]   2>           "core":"testIntegration_shard1_replica_n1",
   [junit4]   2>           "leader":"true",
   [junit4]   2>           "INDEX.sizeInBytes":123450000,
   [junit4]   2>           "node_name":"127.0.0.1:10000_solr",
   [junit4]   2>           "state":"active",
   [junit4]   2>           "type":"NRT"},
   [junit4]   2>         "core_node4":{
   [junit4]   2>           "node_name":"127.0.0.1:10000_solr",
   [junit4]   2>           "core":"testIntegration_shard1_replica_n3",
   [junit4]   2>           "state":"active",
   [junit4]   2>           "INDEX.sizeInBytes":123450000,
   [junit4]   2>           "type":"NRT"}},
   [junit4]   2>       "range":"80000000-7fffffff",
   [junit4]   2>       "state":"active"}}}
   [junit4]   2> 679112 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testIntegration
   [junit4]   2> 679112 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 679112 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 679112 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 679112 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 679112 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:		1
   [junit4]   2> 679112 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:	0
   [junit4]   2> 679112 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:		1
   [junit4]   2> 679112 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		127.0.0.1:10001_solr
   [junit4]   2> 679112 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:	[testIntegration]
   [junit4]   2> 679112 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:	2
   [junit4]   2> 679112 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:	2
   [junit4]   2> 679112 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:		2
   [junit4]   2> 679112 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## * testIntegration		2
   [junit4]   2> 679112 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- active           2
   [junit4]   2> 679112 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 679112 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- CREATE             1
   [junit4]   2> 679113 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- DELETESTATUS       1
   [junit4]   2> 679113 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- MOVEREPLICA        1
   [junit4]   2> 679113 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- REQUESTSTATUS      1
   [junit4]   2> 679113 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- autoscaling        1
   [junit4]   2> 679113 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- update             1
   [junit4]   2> 679113 INFO  (TEST-TestExecutePlanAction.testIntegration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
   [junit4]   2> 679113 DEBUG (AutoscalingActionExecutor-1718-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 11382 ms for event id=a091c092378ecT5y74lvmt9hwilvifly8sh3hkp
   [junit4]   2> 679114 DEBUG (ScheduledTrigger-1717-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
   [junit4]   2> 679116 DEBUG (ScheduledTrigger-1717-thread-4) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 1
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestExecutePlanAction -Dtests.method=testIntegration -Dtests.seed=C48B7B3B457DBD72 -Dtests.slow=true -Dtests.locale=ar-QA -Dtests.timezone=Africa/Gaborone -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.47s J1 | TestExecutePlanAction.testIntegration <<<
   [junit4]    > Throwable #1: java.util.ConcurrentModificationException
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C48B7B3B457DBD72:74EA751760421C57]:0)
   [junit4]    > 	at java.util.ArrayList$Itr.checkForComodification(ArrayList.java:909)
   [junit4]    > 	at java.util.ArrayList$Itr.next(ArrayList.java:859)
   [junit4]    > 	at org.apache.solr.cloud.autoscaling.sim.SimSolrCloudTestCase.tearDown(SimSolrCloudTestCase.java:141)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 679116 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2048)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:166)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 679120 DEBUG (SUITE-TestExecutePlanAction-seed#[C48B7B3B457DBD72]-worker) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.sim.TestExecutePlanAction_C48B7B3B457DBD72-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1709, maxMBSortInHeap=5.080522274153202, sim=RandomSimilarity(queryNorm=true): {}, locale=ar-QA, timezone=Africa/Gaborone
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=3,threads=1,free=144327640,total=323878912
   [junit4]   2> NOTE: All tests run in this JVM: [TestLRUCache, AddSchemaFieldsUpdateProcessorFactoryTest, TestMacroExpander, CloneFieldUpdateProcessorFactoryTest, AnalyticsQueryTest, SolrCoreTest, BlockJoinFacetDistribTest, JavabinLoaderTest, JvmMetricsTest, TestCharFilters, TestSolrCoreParser, TriggerIntegrationTest, PropertiesRequestHandlerTest, SpellingQueryConverterTest, TestReload, ChaosMonkeySafeLeaderTest, LargeFieldTest, ReplaceNodeTest, DocumentAnalysisRequestHandlerTest, TestAnalyzeInfixSuggestions, AssignTest, TestCorePropertiesReload, DebugComponentTest, CdcrVersionReplicationTest, SolrSlf4jReporterTest, TestPerFieldSimilarityWithDefaultOverride, TestConfigSetsAPIExclusivity, TestFieldCacheWithThreads, SystemInfoHandlerTest, HdfsAutoAddReplicasIntegrationTest, TestSolrIndexConfig, TestIndexSearcher, QueryEqualityTest, TestLockTree, CollectionsAPIAsyncDistributedZkTest, DocValuesTest, TestFileDictionaryLookup, TestFastOutputStream, DistributedFacetPivotSmallAdvancedTest, HdfsBasicDistributedZkTest, AutoAddReplicasPlanActionTest, TestCryptoKeys, TestFieldCacheSortRandom, SpellPossibilityIteratorTest, TestExecutePlanAction]
   [junit4] Completed [99/765 (1!)] on J1 in 0.95s, 2 tests, 1 error <<< FAILURES!

[...truncated 735 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestTriggerIntegration_C48B7B3B457DBD72-001/init-core-data-001
   [junit4]   2> 1839217 WARN  (SUITE-TestTriggerIntegration-seed#[C48B7B3B457DBD72]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
   [junit4]   2> 1839217 INFO  (SUITE-TestTriggerIntegration-seed#[C48B7B3B457DBD72]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 1839218 INFO  (SUITE-TestTriggerIntegration-seed#[C48B7B3B457DBD72]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 1839218 INFO  (SUITE-TestTriggerIntegration-seed#[C48B7B3B457DBD72]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 1839219 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 1839219 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0
   [junit4]   2> 1839219 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10005_solr]
   [junit4]   2> 1839219 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
   [junit4]   2> 1839219 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
   [junit4]   2> 1839221 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTriggerRestoreState
   [junit4]   2> 1839221 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 1839221 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 1839221 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1839221 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1220)
   [junit4]   2> 	at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:154)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1839221 DEBUG (ScheduledTrigger-5649-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1839222 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 1839223 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 1839223 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
   [junit4]   2> 1839223 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10005_solr]
   [junit4]   2> 1839223 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
   [junit4]   2> 1839223 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 1839223 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1839223 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 1839223 DEBUG (ScheduledTrigger-5652-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1839243 DEBUG (ScheduledTrigger-5652-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1839266 DEBUG (ScheduledTrigger-5652-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1839286 DEBUG (ScheduledTrigger-5652-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1839306 DEBUG (ScheduledTrigger-5652-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1839325 DEBUG (simCloudManagerPool-5651-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1839325 DEBUG (simCloudManagerPool-5651-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
   [junit4]   2> 1839325 DEBUG (simCloudManagerPool-5651-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10005_solr]
   [junit4]   2> 1839325 DEBUG (simCloudManagerPool-5651-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger instantiated with properties: {event=nodeAdded, waitFor=5, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true}
   [junit4]   2> 1839326 DEBUG (simCloudManagerPool-5651-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10005_solr]
   [junit4]   2> 1839326 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 1839326 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 1839326 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 1839326 DEBUG (ScheduledTrigger-5652-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 1839326 DEBUG (ScheduledTrigger-5652-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1839326 DEBUG (ScheduledTrigger-5652-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10007_solr at time 2871230138595399
   [junit4]   2> 1839326 DEBUG (ScheduledTrigger-5652-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 1839347 DEBUG (ScheduledTrigger-5652-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 1839347 DEBUG (ScheduledTrigger-5652-thread-3) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1839347 DEBUG (ScheduledTrigger-5652-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 1839357 DEBUG (simCloudManagerPool-5651-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1839357 DEBUG (simCloudManagerPool-5651-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
   [junit4]   2> 1839357 DEBUG (simCloudManagerPool-5651-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10005_solr, 127.0.0.1:10007_solr]
   [junit4]   2> 1839357 DEBUG (simCloudManagerPool-5651-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true}
   [junit4]   2> 1839357 DEBUG (simCloudManagerPool-5651-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10005_solr, 127.0.0.1:10007_solr]
   [junit4]   2> 1839357 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
   [junit4]   2> 1839357 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 1839357 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeAddedTrigger Adding node from marker path: 127.0.0.1:10007_solr
   [junit4]   2> 1839357 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
   [junit4]   2> 1839358 DEBUG (ScheduledTrigger-5652-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 1839358 DEBUG (ScheduledTrigger-5652-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
   [junit4]   2> 1839358 DEBUG (ScheduledTrigger-5652-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger firing registered processor for nodes: [127.0.0.1:10007_solr] added at times [2871231699896449], now=2871231712000599
   [junit4]   2> 1839358 DEBUG (ScheduledTrigger-5652-thread-1) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"a335eb98a4881T5y74lvmt9hwilvifly8sh3oeu",
   [junit4]   2>   "source":"node_added_restore_trigger",
   [junit4]   2>   "eventTime":2871231699896449,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2871231699896449],
   [junit4]   2>     "nodeNames":["127.0.0.1:10007_solr"]}}
   [junit4]   2> 1839358 DEBUG (AutoscalingActionExecutor-5653-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"a335eb98a4881T5y74lvmt9hwilvifly8sh3oeu",
   [junit4]   2>   "source":"node_added_restore_trigger",
   [junit4]   2>   "eventTime":2871231699896449,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2871231699896449],
   [junit4]   2>     "_enqueue_time_":2871231718427699,
   [junit4]   2>     "nodeNames":["127.0.0.1:10007_solr"]}}
   [junit4]   2> 1839359 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTriggerRestoreState
   [junit4]   2> 1839359 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 1839359 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 1839359 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 1839359 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 1839359 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:		3
   [junit4]   2> 1839359 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:	3
   [junit4]   2> 1839359 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:		0
   [junit4]   2> 1839359 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:	[]
   [junit4]   2> 1839359 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:	0
   [junit4]   2> 1839359 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:	0
   [junit4]   2> 1839359 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:		0
   [junit4]   2> 1839359 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 1839359 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- autoscaling        2
   [junit4]   2> 1839359 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- update             1
   [junit4]   2> 1839359 INFO  (TEST-TestTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
   [junit4]   2> 1839359 DEBUG (AutoscalingActionExecutor-5653-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 43 ms for event id=a335eb98a4881T5y74lvmt9hwilvifly8sh3oeu
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestTriggerIntegration -Dtests.method=testNodeAddedTriggerRestoreState -Dtests.seed=C48B7B3B457DBD72 -Dtests.slow=true -Dtests.locale=id-ID -Dtests.timezone=America/Guayaquil -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.14s J0 | TestTriggerIntegration.testNodeAddedTriggerRestoreState <<<
   [junit4]    > Throwable #1: java.util.ConcurrentModificationException
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C48B7B3B457DBD72:4CB6F2447FBD5CDF]:0)
   [junit4]    > 	at java.util.ArrayList$Itr.checkForComodification(ArrayList.java:909)
   [junit4]    > 	at java.util.ArrayList$Itr.next(ArrayList.java:859)
   [junit4]    > 	at org.apache.solr.cloud.autoscaling.sim.SimSolrCloudTestCase.tearDown(SimSolrCloudTestCase.java:141)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1839367 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeLostTriggerRestoreState
   [junit4]   2> 1839367 DEBUG (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5
   [junit4]   2> 1839368 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 1839369 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 1839369 DEBUG (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 1839370 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 1839370 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6
   [junit4]   2> 1839370 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10005_solr]
   [junit4]   2> 1839370 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion -1
   [junit4]   2> 1839370 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
   [junit4]   2> 1839370 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1839370 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
   [junit4]   2> 1839370 DEBUG (ScheduledTrigger-5655-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1839390 DEBUG (ScheduledTrigger-5655-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1839411 DEBUG (ScheduledTrigger-5655-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1839433 DEBUG (ScheduledTrigger-5655-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1839453 DEBUG (ScheduledTrigger-5655-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1839473 DEBUG (simCloudManagerPool-5654-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1839473 DEBUG (simCloudManagerPool-5654-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 7
   [junit4]   2> 1839473 DEBUG (ScheduledTrigger-5655-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
   [junit4]   2> 1839474 DEBUG (simCloudManagerPool-5654-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10005_solr, 127.0.0.1:10008_solr]
   [junit4]   2> 1839474 DEBUG (simCloudManagerPool-5654-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10005_solr, 127.0.0.1:10008_solr]
   [junit4]   2> 1839474 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7
   [junit4]   2> 1839474 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 1839474 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1839474 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7
   [junit4]   2> 1839474 DEBUG (ScheduledTrigger-5655-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 2
   [junit4]   2> 1839474 DEBUG (ScheduledTrigger-5655-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10008_solr
   [junit4]   2> 1839493 DEBUG (ScheduledTrigger-5655-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1839494 DEBUG (ScheduledTrigger-5655-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10008_solr
   [junit4]   2> 1839502 DEBUG (ScheduledTrigger-5655-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 2
   [junit4]   2> 1839504 DEBUG (simCloudManagerPool-5654-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1839505 DEBUG (simCloudManagerPool-5654-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 8
   [junit4]   2> 1839505 DEBUG (simCloudManagerPool-5654-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10005_solr]
   [junit4]   2> 1839505 DEBUG (simCloudManagerPool-5654-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10005_solr]
   [junit4]   2> 1839505 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8
   [junit4]   2> 1839505 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
   [junit4]   2> 1839505 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1839505 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8
   [junit4]   2> 1839505 DEBUG (ScheduledTrigger-5655-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 2
   [junit4]   2> 1839505 DEBUG (ScheduledTrigger-5655-thread-3) [    ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10008_solr]
   [junit4]   2> 1839505 DEBUG (ScheduledTrigger-5655-thread-3) [    ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
   [junit4]   2>   "id":"a336015e5dbfdT5y74lvmt9hwilvifly8sh3oex",
   [junit4]   2>   "source":"node_lost_restore_trigger",
   [junit4]   2>   "eventTime":2871237544369149,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2871237544369149],
   [junit4]   2>     "nodeNames":["127.0.0.1:10008_solr"]}}
   [junit4]   2> 1839506 DEBUG (AutoscalingActionExecutor-5656-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"a336015e5dbfdT5y74lvmt9hwilvifly8sh3oex",
   [junit4]   2>   "source":"node_lost_restore_trigger",
   [junit4]   2>   "eventTime":2871237544369149,
   [junit4]   2>   "eventType":"NODELOST",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[2871237544369149],
   [junit4]   2>     "_enqueue_time_":2871239094614549,
   [junit4]   2>     "nodeNames":["127.0.0.1:10008_solr"]}}
   [junit4]   2> 1839506 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testNodeLostTriggerRestoreState
   [junit4]   2> 1839506 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase 
   [junit4]   2> 
   [junit4]   2> 1839506 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 1839506 DEBUG (AutoscalingActionExecutor-5656-thread-1) [    ] o.a.s.c.a.ScheduledTriggers -- processing took 22 ms for event id=a336015e5dbfdT5y74lvmt9hwilvifly8sh3oex
   [junit4]   2> 1839506 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
   [junit4]   2> 1839506 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
   [junit4]   2> 
   [junit4]   2> 1839506 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes:		2
   [junit4]   2> 1839506 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes:	2
   [junit4]   2> 1839506 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes:		1
   [junit4]   2> 1839506 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		127.0.0.1:10008_solr
   [junit4]   2> 1839507 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections:	[]
   [junit4]   2> 1839507 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node:	0
   [junit4]   2> 1839507 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node:	0
   [junit4]   2> 1839507 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas:		0
   [junit4]   2> 1839507 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
   [junit4]   2> 1839507 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- autoscaling        2
   [junit4]   2> 1839507 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- update             2
   [junit4]   2> 1839507 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
   [junit4]   2> 1839507 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_lost_restore_trigger
   [junit4]   2> 1839507 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- STARTED         1
   [junit4]   2> 1839507 INFO  (TEST-TestTriggerIntegration.testNodeLostTriggerRestoreState-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimSolrCloudTestCase ##		- SUCCEEDED       1
   [junit4]   2> 1839511 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeMarkersRegistration
   [junit4]   2> 1839511 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 9
   [junit4]   2> 1839511 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 9
   [junit4]   2> 1839511 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10008_solr
   [junit4]   2> 1839511 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 1839512 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 1839512 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1839512 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Interrupted
   [junit4]   2> java.lang.InterruptedException
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1220)
   [junit4]   2> 	at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:154)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 1839513 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 1839513 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 1839514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 10
   [junit4]   2> 1839514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10005_solr]
   [junit4]   2> 1839514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion -1
   [junit4]   2> 1839514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 10
   [junit4]   2> 1839514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1839514 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10
   [junit4]   2> 1839514 DEBUG (ScheduledTrigger-5658-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1839538 DEBUG (ScheduledTrigger-5658-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1839559 DEBUG (ScheduledTrigger-5658-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1839579 DEBUG (ScheduledTrigger-5658-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1839599 DEBUG (ScheduledTrigger-5658-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1839619 DEBUG (ScheduledTrigger-5658-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1839620 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.TestTriggerIntegration ====== KILL OVERSEER 1
   [junit4]   2> 1839620 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
   [junit4]   2> 1839620 WARN  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 1839621 DEBUG (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 1839621 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 10
   [junit4]   2> 1839622 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10009_solr]
   [junit4]   2> 1839622 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion -1
   [junit4]   2> 1839622 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 10
   [junit4]   2> 1839622 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 127.0.0.1:10005_solr
   [junit4]   2> 1839622 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 1839622 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10
   [junit4]   2> 1839622 DEBUG (ScheduledTrigger-5661-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1839622 DEBUG (ScheduledTrigger-5661-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10005_solr
   [junit4]   2> 1839642 DEBUG (ScheduledTrigger-5661-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1839662 DEBUG (ScheduledTrigger-5661-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1839693 DEBUG (ScheduledTrigger-5661-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1839722 DEBUG (ScheduledTrigger-5661-thread-3) [    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
   [junit4]   2> 1839722 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.TestTriggerIntegration ====== ADD TRIGGERS
   [junit4]   2> 1839724 DEBUG (simCloudManagerPool-5660-thread-1) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1839724 DEBUG (simCloudManagerPool-5660-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 11
   [junit4]   2> 1839725 DEBUG (simCloudManagerPool-5660-thread-1) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10009_solr]
   [junit4]   2> 1839725 DEBUG (simCloudManagerPool-5660-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10009_solr]
   [junit4]   2> 1839725 DEBUG (simCloudManagerPool-5660-thread-1) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventMarkerAction}], enabled=true}
   [junit4]   2> 1839725 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 11
   [junit4]   2> 1839725 INFO  (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.s.TestTriggerIntegration TestEventMarkerAction init
   [junit4]   2> 1839725 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion 11
   [junit4]   2> 1839725 DEBUG (ScheduledTrigger-5661-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 1839725 DEBUG (ScheduledTrigger-5661-thread-4) [    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
   [junit4]   2> 1839725 DEBUG (simCloudManagerPool-5660-thread-2) [    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 1839725 DEBUG (simCloudManagerPool-5660-thread-2) [    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 12
   [junit4]   2> 1839725 DEBUG (simCloudManagerPool-5660-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10009_solr]
   [junit4]   2> 1839725 DEBUG (simCloudManagerPool-5660-thread-2) [    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10009_solr]
   [junit4]   2> 1839725 DEBUG (simCloudManagerPool-5660-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10006_solr, 127.0.0.1:10009_solr]
   [junit4]   2> 1839726 DEBUG (simCloudManagerPool-5660-thread-2) [    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventMarkerAction}], enabled=true}
   [junit4]   2> 1839726 INFO  (TEST-TestTriggerIntegration.testNodeMarkersRegistration-seed#[C48B7B3B457DBD72]) [    ] o.a.s.c.a.s.TestTriggerIntegration ====== ADD NODE 1
   [junit4]   2> 1839726 DEBUG (Simulated OverseerAutoScalingTriggerThread) [    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 12
   [junit4]   2> 1839726 INFO  (Simu

[...truncated too long message...]

Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 143258 INFO  (jetty-closer-11-thread-4) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@757dfd3: rootName = solr_60604, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@4a7bb9f9
   [junit4]   2> 143260 INFO  (zkCallback-42-thread-2-processing-n:127.0.0.1:60604_solr) [n:127.0.0.1:60604_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:60604_solr
   [junit4]   2> 143261 INFO  (jetty-closer-11-thread-4) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:60604_solr
   [junit4]   2> 143263 INFO  (zkCallback-42-thread-2-processing-n:127.0.0.1:60604_solr) [n:127.0.0.1:60604_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
   [junit4]   2> 143263 INFO  (jetty-closer-11-thread-2) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7d7f82c{/solr,null,UNAVAILABLE}
   [junit4]   2> 143265 INFO  (zkCallback-38-thread-1-processing-n:127.0.0.1:34995_solr) [n:127.0.0.1:34995_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
   [junit4]   2> 143267 INFO  (jetty-closer-11-thread-4) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 143270 INFO  (zkCallback-44-thread-2-processing-n:127.0.0.1:50146_solr) [n:127.0.0.1:50146_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
   [junit4]   2> 143319 INFO  (jetty-closer-11-thread-3) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 143319 INFO  (jetty-closer-11-thread-3) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@588bfbc3: rootName = solr_34995, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@4a7bb9f9
   [junit4]   2> 143320 INFO  (zkCallback-38-thread-1-processing-n:127.0.0.1:34995_solr) [n:127.0.0.1:34995_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:34995_solr
   [junit4]   2> 143322 INFO  (jetty-closer-11-thread-3) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:34995_solr
   [junit4]   2> 143323 INFO  (jetty-closer-11-thread-3) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 143324 INFO  (zkCallback-38-thread-2-processing-n:127.0.0.1:34995_solr) [n:127.0.0.1:34995_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 143325 INFO  (zkCallback-44-thread-2-processing-n:127.0.0.1:50146_solr) [n:127.0.0.1:50146_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 143325 INFO  (coreCloseExecutor-274-thread-1) [n:127.0.0.1:34995_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n3] o.a.s.c.SolrCore [collection1_shard2_replica_n3]  CLOSING SolrCore org.apache.solr.core.SolrCore@1681d416
   [junit4]   2> 143325 INFO  (coreCloseExecutor-274-thread-1) [n:127.0.0.1:34995_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1.shard2.replica_n3, tag=377607190
   [junit4]   2> 143325 INFO  (coreCloseExecutor-274-thread-1) [n:127.0.0.1:34995_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n3] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3d610cca: rootName = solr_34995, domain = solr.core.collection1.shard2.replica_n3, service url = null, agent id = null] for registry solr.core.collection1.shard2.replica_n3 / com.codahale.metrics.MetricRegistry@4e9665be
   [junit4]   2> 143334 INFO  (coreCloseExecutor-274-thread-1) [n:127.0.0.1:34995_solr c:collection1 s:shard2 r:core_node4 x:collection1_shard2_replica_n3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard2.leader, tag=377607190
   [junit4]   2> 143335 INFO  (jetty-closer-11-thread-4) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@410a5ffa{/solr,null,UNAVAILABLE}
   [junit4]   2> 143339 INFO  (jetty-closer-11-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 143339 INFO  (jetty-closer-11-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@305ef: rootName = solr_50146, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@4e4434e9
   [junit4]   2> 143343 INFO  (jetty-closer-11-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 143343 INFO  (jetty-closer-11-thread-1) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@69f2d18b: rootName = solr_50146, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@4a7bb9f9
   [junit4]   2> 143345 INFO  (jetty-closer-11-thread-1) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:50146_solr
   [junit4]   2> 143346 INFO  (jetty-closer-11-thread-1) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 143346 INFO  (zkCallback-44-thread-2-processing-n:127.0.0.1:50146_solr) [n:127.0.0.1:50146_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 144860 WARN  (zkCallback-38-thread-2-processing-n:127.0.0.1:34995_solr) [n:127.0.0.1:34995_solr    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 144861 INFO  (jetty-closer-11-thread-3) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@614b4ee6{/solr,null,UNAVAILABLE}
   [junit4]   2> 144862 INFO  (jetty-closer-11-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@e84c30{/solr,null,UNAVAILABLE}
   [junit4]   2> 144865 ERROR (SUITE-StreamExpressionTest-seed#[1918B1C95D2400E1]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 144867 INFO  (SUITE-StreamExpressionTest-seed#[1918B1C95D2400E1]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:53399 53399
   [junit4]   2> 145021 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxnFactory Ignoring unexpected runtime exception
   [junit4]   2> java.nio.channels.CancelledKeyException
   [junit4]   2> 	at sun.nio.ch.SelectionKeyImpl.ensureValid(SelectionKeyImpl.java:73)
   [junit4]   2> 	at sun.nio.ch.SelectionKeyImpl.readyOps(SelectionKeyImpl.java:87)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:182)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 145021 INFO  (Thread-52) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:53399 53399
   [junit4]   2> 145064 WARN  (Thread-52) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	99	/solr/aliases.json
   [junit4]   2> 	17	/solr/configs/conf
   [junit4]   2> 	8	/solr/configs/ml
   [junit4]   2> 	4	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	99	/solr/clusterprops.json
   [junit4]   2> 	99	/solr/clusterstate.json
   [junit4]   2> 	38	/solr/collections/destinationCollection/state.json
   [junit4]   2> 	26	/solr/collections/parallelDestinationCollection1/state.json
   [junit4]   2> 	20	/solr/collections/parallelDestinationCollection/state.json
   [junit4]   2> 	10	/solr/collections/mainCorpus1/state.json
   [junit4]   2> 	10	/solr/collections/destination/state.json
   [junit4]   2> 	10	/solr/collections/destination1/state.json
   [junit4]   2> 	10	/solr/collections/collection1/state.json
   [junit4]   2> 	10	/solr/collections/workQueue1/state.json
   [junit4]   2> 	10	/solr/collections/workQueue/state.json
   [junit4]   2> 	10	/solr/collections/uknownCollection/state.json
   [junit4]   2> 	10	/solr/collections/modelCollection/state.json
   [junit4]   2> 	10	/solr/collections/checkpointCollection/state.json
   [junit4]   2> 	10	/solr/collections/mainCorpus/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	99	/solr/live_nodes
   [junit4]   2> 	99	/solr/collections
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-solrj/test/J1/temp/solr.client.solrj.io.stream.StreamExpressionTest_1918B1C95D2400E1-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=false): {}, locale=fr-CH, timezone=Etc/GMT+12
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=3,threads=1,free=97033240,total=227909632
   [junit4]   2> NOTE: All tests run in this JVM: [FacetFieldTest, HttpSolrClientConPoolTest, CorrelationEvaluatorTest, CubedRootEvaluatorTest, ArcCosineEvaluatorTest, SolrSchemalessExampleTest, ModuloEvaluatorTest, StreamExpressionParserTest, TestPathTrie, AndEvaluatorTest, LargeVolumeJettyTest, StreamExpressionToExplanationTest, SolrParamTest, StreamExpressionTest]
   [junit4] Completed [70/146 (1!)] on J1 in 129.45s, 148 tests, 1 failure <<< FAILURES!

[...truncated 47457 lines...]

Mime
View raw message