lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Policeman Jenkins Server <jenk...@thetaphi.de>
Subject [JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build # 1481 - Unstable!
Date Thu, 19 Oct 2017 05:54:53 GMT
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1481/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC

2 tests failed.
FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test

Error Message:
null

Stack Trace:
java.lang.NumberFormatException: null
	at __randomizedtesting.SeedInfo.seed([AE5B46876EAAA9F:82B18BB2D816C767]:0)
	at java.lang.Long.parseLong(Long.java:552)
	at java.lang.Long.parseLong(Long.java:631)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForReplicationFromReplicas(AbstractFullDistribZkTestBase.java:2125)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test(ChaosMonkeySafeLeaderWithPullReplicasTest.java:209)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeAddedTriggerRestoreState

Error Message:
The trigger did not fire at all

Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
	at __randomizedtesting.SeedInfo.seed([AE5B46876EAAA9F:82D83D174C2A4B32]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeAddedTriggerRestoreState(TriggerIntegrationTest.java:402)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 12016 lines...]
   [junit4] Suite: org.apache.solr.cloud.autoscaling.TriggerIntegrationTest
   [junit4]   2> 887272 INFO  (SUITE-TriggerIntegrationTest-seed#[AE5B46876EAAA9F]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.TriggerIntegrationTest_AE5B46876EAAA9F-001/init-core-data-001
   [junit4]   2> 887272 WARN  (SUITE-TriggerIntegrationTest-seed#[AE5B46876EAAA9F]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=6000 numCloses=6000
   [junit4]   2> 887272 INFO  (SUITE-TriggerIntegrationTest-seed#[AE5B46876EAAA9F]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 887273 INFO  (SUITE-TriggerIntegrationTest-seed#[AE5B46876EAAA9F]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 887274 INFO  (SUITE-TriggerIntegrationTest-seed#[AE5B46876EAAA9F]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.TriggerIntegrationTest_AE5B46876EAAA9F-001/tempDir-001
   [junit4]   2> 887274 INFO  (SUITE-TriggerIntegrationTest-seed#[AE5B46876EAAA9F]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 887274 INFO  (Thread-1702) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 887274 INFO  (Thread-1702) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 887278 ERROR (Thread-1702) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 887375 INFO  (SUITE-TriggerIntegrationTest-seed#[AE5B46876EAAA9F]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:59235
   [junit4]   2> 887384 INFO  (jetty-launcher-3054-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 887385 INFO  (jetty-launcher-3054-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@16bae0d5{/solr,null,AVAILABLE}
   [junit4]   2> 887386 INFO  (jetty-launcher-3054-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@48dbcff4{HTTP/1.1,[http/1.1]}{127.0.0.1:65452}
   [junit4]   2> 887386 INFO  (jetty-launcher-3054-thread-1) [    ] o.e.j.s.Server Started @890349ms
   [junit4]   2> 887386 INFO  (jetty-launcher-3054-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=65452}
   [junit4]   2> 887386 ERROR (jetty-launcher-3054-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 887386 INFO  (jetty-launcher-3054-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 887386 INFO  (jetty-launcher-3054-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 887386 INFO  (jetty-launcher-3054-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 887386 INFO  (jetty-launcher-3054-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-19T04:37:06.687Z
   [junit4]   2> 887393 INFO  (jetty-launcher-3054-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 887394 INFO  (jetty-launcher-3054-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@18f895a8{/solr,null,AVAILABLE}
   [junit4]   2> 887394 INFO  (jetty-launcher-3054-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@25705aae{HTTP/1.1,[http/1.1]}{127.0.0.1:37291}
   [junit4]   2> 887394 INFO  (jetty-launcher-3054-thread-2) [    ] o.e.j.s.Server Started @890357ms
   [junit4]   2> 887394 INFO  (jetty-launcher-3054-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=37291}
   [junit4]   2> 887394 ERROR (jetty-launcher-3054-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 887394 INFO  (jetty-launcher-3054-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 887394 INFO  (jetty-launcher-3054-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 887394 INFO  (jetty-launcher-3054-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 887394 INFO  (jetty-launcher-3054-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-19T04:37:06.695Z
   [junit4]   2> 887403 INFO  (jetty-launcher-3054-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 887416 INFO  (jetty-launcher-3054-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 887418 INFO  (jetty-launcher-3054-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59235/solr
   [junit4]   2> 887425 INFO  (jetty-launcher-3054-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59235/solr
   [junit4]   2> 887578 INFO  (jetty-launcher-3054-thread-2) [n:127.0.0.1:37291_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 887579 INFO  (jetty-launcher-3054-thread-2) [n:127.0.0.1:37291_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:37291_solr
   [junit4]   2> 887580 INFO  (jetty-launcher-3054-thread-2) [n:127.0.0.1:37291_solr    ] o.a.s.c.Overseer Overseer (id=98853704602550278-127.0.0.1:37291_solr-n_0000000000) starting
   [junit4]   2> 887593 INFO  (jetty-launcher-3054-thread-2) [n:127.0.0.1:37291_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37291_solr
   [junit4]   2> 887593 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 887597 INFO  (OverseerStateUpdate-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 887598 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
   [junit4]   2> 887599 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:37291_solr]
   [junit4]   2> 887599 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1
   [junit4]   2> 887599 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
   [junit4]   2> 887600 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 887601 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
   [junit4]   2> 887601 DEBUG (ScheduledTrigger-3017-thread-1) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:37291_solr]
   [junit4]   2> 887611 INFO  (jetty-launcher-3054-thread-1) [n:127.0.0.1:65452_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 887614 INFO  (jetty-launcher-3054-thread-1) [n:127.0.0.1:65452_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 887625 INFO  (jetty-launcher-3054-thread-1) [n:127.0.0.1:65452_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:65452_solr
   [junit4]   2> 887646 INFO  (zkCallback-3066-thread-1-processing-n:127.0.0.1:37291_solr) [n:127.0.0.1:37291_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 887653 INFO  (zkCallback-3062-thread-1-processing-n:127.0.0.1:65452_solr) [n:127.0.0.1:65452_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 887740 INFO  (jetty-launcher-3054-thread-2) [n:127.0.0.1:37291_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37291.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@15e8a90d
   [junit4]   2> 887746 INFO  (jetty-launcher-3054-thread-2) [n:127.0.0.1:37291_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37291.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@15e8a90d
   [junit4]   2> 887746 INFO  (jetty-launcher-3054-thread-2) [n:127.0.0.1:37291_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37291.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@15e8a90d
   [junit4]   2> 887749 INFO  (jetty-launcher-3054-thread-2) [n:127.0.0.1:37291_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.TriggerIntegrationTest_AE5B46876EAAA9F-001/tempDir-001/node2/.
   [junit4]   2> 887755 INFO  (jetty-launcher-3054-thread-1) [n:127.0.0.1:65452_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_65452.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@15e8a90d
   [junit4]   2> 887765 INFO  (jetty-launcher-3054-thread-1) [n:127.0.0.1:65452_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_65452.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@15e8a90d
   [junit4]   2> 887765 INFO  (jetty-launcher-3054-thread-1) [n:127.0.0.1:65452_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_65452.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@15e8a90d
   [junit4]   2> 887766 INFO  (jetty-launcher-3054-thread-1) [n:127.0.0.1:65452_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.TriggerIntegrationTest_AE5B46876EAAA9F-001/tempDir-001/node1/.
   [junit4]   2> 887802 INFO  (SUITE-TriggerIntegrationTest-seed#[AE5B46876EAAA9F]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 887803 INFO  (SUITE-TriggerIntegrationTest-seed#[AE5B46876EAAA9F]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:59235/solr ready
   [junit4]   2> 887860 INFO  (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[AE5B46876EAAA9F]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTriggerRestoreState
   [junit4]   2> 887870 INFO  (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[AE5B46876EAAA9F]) [    ] o.a.s.c.a.TriggerIntegrationTest /autoscaling.json reset, new znode version 2
   [junit4]   2> 887870 DEBUG (zkCallback-3066-thread-1-processing-n:127.0.0.1:37291_solr) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
   [junit4]   2> 887873 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
   [junit4]   2> 887875 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 887875 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 887882 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
   [junit4]   2> 887888 INFO  (qtp1994817313-10711) [n:127.0.0.1:37291_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 887889 INFO  (qtp1994817313-10711) [n:127.0.0.1:37291_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:59235/solr ready
   [junit4]   2> 887892 INFO  (qtp1994817313-10707) [n:127.0.0.1:37291_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 887896 INFO  (qtp1119640145-10702) [n:127.0.0.1:65452_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 887896 DEBUG (qtp1994817313-10711) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 887898 INFO  (qtp1994817313-10711) [n:127.0.0.1:37291_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=15
   [junit4]   2> 887898 INFO  (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[AE5B46876EAAA9F]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 887900 INFO  (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[AE5B46876EAAA9F]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6d7d5818{/solr,null,AVAILABLE}
   [junit4]   2> 887900 INFO  (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[AE5B46876EAAA9F]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@77fa711e{HTTP/1.1,[http/1.1]}{127.0.0.1:54906}
   [junit4]   2> 887900 INFO  (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[AE5B46876EAAA9F]) [    ] o.e.j.s.Server Started @890864ms
   [junit4]   2> 887900 INFO  (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[AE5B46876EAAA9F]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=54906}
   [junit4]   2> 887900 ERROR (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[AE5B46876EAAA9F]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 887901 INFO  (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[AE5B46876EAAA9F]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 887901 INFO  (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[AE5B46876EAAA9F]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 887901 INFO  (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[AE5B46876EAAA9F]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 887901 INFO  (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[AE5B46876EAAA9F]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-19T04:37:07.202Z
   [junit4]   2> 887903 DEBUG (zkCallback-3066-thread-1-processing-n:127.0.0.1:37291_solr) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
   [junit4]   2> 887905 DEBUG (zkCallback-3066-thread-1-processing-n:127.0.0.1:37291_solr) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:37291_solr, 127.0.0.1:65452_solr]
   [junit4]   2> 887905 DEBUG (zkCallback-3066-thread-1-processing-n:127.0.0.1:37291_solr) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger instantiated with properties: {event=nodeAdded, waitFor=5, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.TriggerIntegrationTest$TestTriggerAction}], enabled=true}
   [junit4]   2> 887905 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
   [junit4]   2> 887905 INFO  (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[AE5B46876EAAA9F]) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 887929 INFO  (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[AE5B46876EAAA9F]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59235/solr
   [junit4]   2> 887933 INFO  (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.TriggerIntegrationTest TestTriggerAction init
   [junit4]   2> 887933 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 887935 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
   [junit4]   2> 887936 DEBUG (ScheduledTrigger-3017-thread-1) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 887936 DEBUG (ScheduledTrigger-3017-thread-1) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: [127.0.0.1:37291_solr, 127.0.0.1:65452_solr]
   [junit4]   2> 887943 INFO  (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[AE5B46876EAAA9F]) [n:127.0.0.1:54906_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 887945 INFO  (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[AE5B46876EAAA9F]) [n:127.0.0.1:54906_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 887948 INFO  (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[AE5B46876EAAA9F]) [n:127.0.0.1:54906_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54906_solr
   [junit4]   2> 888016 INFO  (zkCallback-3073-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 888017 INFO  (zkCallback-3081-thread-1-processing-n:127.0.0.1:54906_solr) [n:127.0.0.1:54906_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 888018 INFO  (zkCallback-3062-thread-1-processing-n:127.0.0.1:65452_solr) [n:127.0.0.1:65452_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 888018 INFO  (zkCallback-3066-thread-2-processing-n:127.0.0.1:37291_solr) [n:127.0.0.1:37291_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 888047 INFO  (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[AE5B46876EAAA9F]) [n:127.0.0.1:54906_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54906.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@15e8a90d
   [junit4]   2> 888057 INFO  (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[AE5B46876EAAA9F]) [n:127.0.0.1:54906_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54906.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@15e8a90d
   [junit4]   2> 888057 INFO  (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[AE5B46876EAAA9F]) [n:127.0.0.1:54906_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54906.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@15e8a90d
   [junit4]   2> 888058 INFO  (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[AE5B46876EAAA9F]) [n:127.0.0.1:54906_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.TriggerIntegrationTest_AE5B46876EAAA9F-001/tempDir-001/node3/.
   [junit4]   2> 888939 DEBUG (ScheduledTrigger-3017-thread-2) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 888939 DEBUG (ScheduledTrigger-3017-thread-2) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: [127.0.0.1:37291_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 888939 DEBUG (ScheduledTrigger-3017-thread-2) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:54906_solr at time 1508387828240000000
   [junit4]   2> 889603 INFO  (qtp1994817313-10710) [n:127.0.0.1:37291_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 889611 INFO  (qtp1994817313-10710) [n:127.0.0.1:37291_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:59235/solr ready
   [junit4]   2> 889614 INFO  (qtp1994817313-10705) [n:127.0.0.1:37291_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 889618 INFO  (qtp1074832861-10759) [n:127.0.0.1:54906_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 889619 INFO  (qtp1119640145-10704) [n:127.0.0.1:65452_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 889619 DEBUG (qtp1994817313-10710) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 889620 INFO  (qtp1994817313-10710) [n:127.0.0.1:37291_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=24
   [junit4]   2> 889621 DEBUG (zkCallback-3066-thread-2-processing-n:127.0.0.1:37291_solr) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
   [junit4]   2> 889621 DEBUG (zkCallback-3066-thread-2-processing-n:127.0.0.1:37291_solr) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:37291_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 889621 DEBUG (zkCallback-3066-thread-2-processing-n:127.0.0.1:37291_solr) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.TriggerIntegrationTest$TestTriggerAction}], enabled=true}
   [junit4]   2> 889622 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
   [junit4]   2> 889622 INFO  (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.TriggerIntegrationTest TestTriggerAction init
   [junit4]   2> 889623 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Adding node from marker path: 127.0.0.1:54906_solr
   [junit4]   2> 889623 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 889623 DEBUG (ScheduledTrigger-3017-thread-1) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 889623 DEBUG (ScheduledTrigger-3017-thread-1) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: [127.0.0.1:37291_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 889623 DEBUG (ScheduledTrigger-3017-thread-1) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger firing registered processor for nodes: [127.0.0.1:54906_solr] added at times [1508387828924000000]
   [junit4]   2> 889623 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
   [junit4]   2> 889623 INFO  (ScheduledTrigger-3017-thread-1) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.SystemLogListener Collection .system does not exist, disabling logging.
   [junit4]   2> 889623 DEBUG (AutoscalingActionExecutor-3018-thread-1-processing-n:127.0.0.1:37291_solr) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"14eedebd1689a700T3pg81goq8ctr8ue829rocy5k2",
   [junit4]   2>   "source":"node_added_restore_trigger",
   [junit4]   2>   "eventTime":1508387828924000000,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[1508387828924000000],
   [junit4]   2>     "_enqueue_time_":1508387828924000000,
   [junit4]   2>     "nodeNames":["127.0.0.1:54906_solr"]}}
   [junit4]   2> 889627 DEBUG (AutoscalingActionExecutor-3018-thread-1-processing-n:127.0.0.1:37291_solr) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.TriggerIntegrationTest --throwable
   [junit4]   2> java.lang.AssertionError: NodeAddedListener was fired before the configured waitFor period
   [junit4]   2> 	at org.junit.Assert.fail(Assert.java:93)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.TriggerIntegrationTest$TestTriggerAction.process(TriggerIntegrationTest.java:586)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$2(ScheduledTriggers.java:229)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 890629 DEBUG (ScheduledTrigger-3017-thread-1) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 890629 DEBUG (ScheduledTrigger-3017-thread-1) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: [127.0.0.1:37291_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 891630 DEBUG (ScheduledTrigger-3017-thread-1) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 891630 DEBUG (ScheduledTrigger-3017-thread-1) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: [127.0.0.1:37291_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 892631 DEBUG (ScheduledTrigger-3017-thread-1) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 892631 DEBUG (ScheduledTrigger-3017-thread-1) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: [127.0.0.1:37291_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 893632 DEBUG (ScheduledTrigger-3017-thread-1) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 893632 DEBUG (ScheduledTrigger-3017-thread-1) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: [127.0.0.1:37291_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 894623 INFO  (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[AE5B46876EAAA9F]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTriggerRestoreState
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TriggerIntegrationTest -Dtests.method=testNodeAddedTriggerRestoreState -Dtests.seed=AE5B46876EAAA9F -Dtests.slow=true -Dtests.locale=is -Dtests.timezone=Asia/Kuala_Lumpur -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE 6.81s J0 | TriggerIntegrationTest.testNodeAddedTriggerRestoreState <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: The trigger did not fire at all
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AE5B46876EAAA9F:82D83D174C2A4B32]:0)
   [junit4]    > 	at org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeAddedTriggerRestoreState(TriggerIntegrationTest.java:402)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 894633 DEBUG (ScheduledTrigger-3017-thread-1) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
   [junit4]   2> 894634 DEBUG (ScheduledTrigger-3017-thread-1) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: [127.0.0.1:37291_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 894648 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testEventQueue
   [junit4]   2> 894648 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.a.s.c.a.TriggerIntegrationTest /autoscaling.json reset, new znode version 5
   [junit4]   2> 894649 DEBUG (zkCallback-3066-thread-2-processing-n:127.0.0.1:37291_solr) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5
   [junit4]   2> 894649 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
   [junit4]   2> 894651 WARN  (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.ScheduledTriggers Failed to remove events for removed trigger /autoscaling/events/node_added_restore_trigger
   [junit4]   2> java.util.NoSuchElementException: /autoscaling/events/node_added_restore_trigger
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.ZkDistribStateManager.listData(ZkDistribStateManager.java:68)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers.removeTriggerZKData(ScheduledTriggers.java:352)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers.remove(ScheduledTriggers.java:337)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:197)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 894651 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 894652 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 894652 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5
   [junit4]   2> 894655 INFO  (qtp1994817313-10708) [n:127.0.0.1:37291_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :overseerstatus with params action=OVERSEERSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 894669 INFO  (qtp1994817313-10708) [n:127.0.0.1:37291_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=OVERSEERSTATUS&wt=javabin&version=2} status=0 QTime=14
   [junit4]   2> 894675 INFO  (qtp1119640145-10694) [n:127.0.0.1:65452_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 894676 INFO  (qtp1119640145-10694) [n:127.0.0.1:65452_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:59235/solr ready
   [junit4]   2> 894677 INFO  (qtp1994817313-10706) [n:127.0.0.1:37291_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 894679 INFO  (qtp1074832861-10761) [n:127.0.0.1:54906_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 894680 INFO  (qtp1119640145-10695) [n:127.0.0.1:65452_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 894680 DEBUG (qtp1119640145-10694) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 894682 INFO  (qtp1119640145-10694) [n:127.0.0.1:65452_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=12
   [junit4]   2> 894683 DEBUG (zkCallback-3066-thread-2-processing-n:127.0.0.1:37291_solr) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6
   [junit4]   2> 894683 INFO  (zkCallback-3066-thread-2-processing-n:127.0.0.1:37291_solr) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.TriggerIntegrationTest TestEventQueueAction instantiated
   [junit4]   2> 894684 DEBUG (zkCallback-3066-thread-2-processing-n:127.0.0.1:37291_solr) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:37291_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 894684 DEBUG (zkCallback-3066-thread-2-processing-n:127.0.0.1:37291_solr) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.TriggerIntegrationTest$TestEventQueueAction}], enabled=true}
   [junit4]   2> 894684 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
   [junit4]   2> 894688 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.TriggerIntegrationTest TestTriggerAction init
   [junit4]   2> 894689 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 894689 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 894690 DEBUG (ScheduledTrigger-3017-thread-2) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 894690 DEBUG (ScheduledTrigger-3017-thread-2) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: [127.0.0.1:37291_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 894690 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@67fd2074{/solr,null,AVAILABLE}
   [junit4]   2> 894691 DEBUG (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
   [junit4]   2> 894691 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@1256e7da{HTTP/1.1,[http/1.1]}{127.0.0.1:51480}
   [junit4]   2> 894691 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.e.j.s.Server Started @897656ms
   [junit4]   2> 894691 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=51480}
   [junit4]   2> 894691 ERROR (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 894691 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 894691 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 894691 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 894691 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-19T04:37:13.992Z
   [junit4]   2> 894695 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 894704 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59235/solr
   [junit4]   2> 894727 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [n:127.0.0.1:51480_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 894730 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [n:127.0.0.1:51480_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 894733 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [n:127.0.0.1:51480_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51480_solr
   [junit4]   2> 894736 INFO  (zkCallback-3066-thread-2-processing-n:127.0.0.1:37291_solr) [n:127.0.0.1:37291_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 894736 INFO  (zkCallback-3081-thread-1-processing-n:127.0.0.1:54906_solr) [n:127.0.0.1:54906_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 894736 INFO  (zkCallback-3062-thread-1-processing-n:127.0.0.1:65452_solr) [n:127.0.0.1:65452_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 894736 INFO  (zkCallback-3073-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 894737 INFO  (zkCallback-3093-thread-1-processing-n:127.0.0.1:51480_solr) [n:127.0.0.1:51480_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 894827 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [n:127.0.0.1:51480_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51480.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@15e8a90d
   [junit4]   2> 894836 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [n:127.0.0.1:51480_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51480.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@15e8a90d
   [junit4]   2> 894836 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [n:127.0.0.1:51480_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51480.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@15e8a90d
   [junit4]   2> 894837 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [n:127.0.0.1:51480_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.TriggerIntegrationTest_AE5B46876EAAA9F-001/tempDir-001/node4/.
   [junit4]   2> 895693 DEBUG (ScheduledTrigger-3017-thread-2) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 895693 DEBUG (ScheduledTrigger-3017-thread-2) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: [127.0.0.1:37291_solr, 127.0.0.1:51480_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 895693 DEBUG (ScheduledTrigger-3017-thread-2) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:51480_solr at time 1508387834994000000
   [junit4]   2> 896663 INFO  (OverseerCollectionConfigSetProcessor-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 896695 DEBUG (ScheduledTrigger-3017-thread-2) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 896695 DEBUG (ScheduledTrigger-3017-thread-2) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: [127.0.0.1:37291_solr, 127.0.0.1:51480_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 896695 DEBUG (ScheduledTrigger-3017-thread-2) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing registered processor for nodes: [127.0.0.1:51480_solr] added at times [1508387834994000000]
   [junit4]   2> 896696 INFO  (ScheduledTrigger-3017-thread-2) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.SystemLogListener Collection .system does not exist, disabling logging.
   [junit4]   2> 896696 DEBUG (AutoscalingActionExecutor-3018-thread-1-processing-n:127.0.0.1:37291_solr) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"14eedebe80568080T3pg81goq8ctr8ue829rocy5k4",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":1508387834994000000,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[1508387834994000000],
   [junit4]   2>     "_enqueue_time_":1508387835996000000,
   [junit4]   2>     "nodeNames":["127.0.0.1:51480_solr"]}}
   [junit4]   2> 896696 INFO  (AutoscalingActionExecutor-3018-thread-1-processing-n:127.0.0.1:37291_solr) [n:127.0.0.1:37291_solr    ] o.a.s.c.ActionThrottle The last action attempt started 7074ms ago.
   [junit4]   2> 896696 INFO  (AutoscalingActionExecutor-3018-thread-1-processing-n:127.0.0.1:37291_solr) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.TriggerIntegrationTest -- event: {
   [junit4]   2>   "id":"14eedebe80568080T3pg81goq8ctr8ue829rocy5k4",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":1508387834994000000,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[1508387834994000000],
   [junit4]   2>     "_enqueue_time_":1508387835996000000,
   [junit4]   2>     "nodeNames":["127.0.0.1:51480_solr"]}}
   [junit4]   2> 896697 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@25705aae{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 896698 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1497231559
   [junit4]   2> 896698 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 896698 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1d00d03a: rootName = solr_37291, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@3bfd9c57
   [junit4]   2> 896704 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 896704 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3a4eff01: rootName = solr_37291, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@3792a008
   [junit4]   2> 896707 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 896707 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@74a82fee: rootName = solr_37291, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@33ad2222
   [junit4]   2> 896709 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:37291_solr
   [junit4]   2> 896710 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 896710 INFO  (zkCallback-3081-thread-1-processing-n:127.0.0.1:54906_solr) [n:127.0.0.1:54906_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 896710 INFO  (zkCallback-3066-thread-2-processing-n:127.0.0.1:37291_solr) [n:127.0.0.1:37291_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 896710 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 896711 INFO  (zkCallback-3093-thread-1-processing-n:127.0.0.1:51480_solr) [n:127.0.0.1:51480_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 896711 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.a.s.c.Overseer Overseer (id=98853704602550278-127.0.0.1:37291_solr-n_0000000000) closing
   [junit4]   2> 896711 INFO  (OverseerStateUpdate-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:37291_solr
   [junit4]   2> 896711 INFO  (zkCallback-3062-thread-1-processing-n:127.0.0.1:65452_solr) [n:127.0.0.1:65452_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 896711 INFO  (zkCallback-3073-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 896712 WARN  (OverseerAutoScalingTriggerThread-98853704602550278-127.0.0.1:37291_solr-n_0000000000) [n:127.0.0.1:37291_solr    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 896712 DEBUG (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 896715 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@18f895a8{/solr,null,UNAVAILABLE}
   [junit4]   2> 896715 INFO  (zkCallback-3062-thread-1-processing-n:127.0.0.1:65452_solr) [n:127.0.0.1:65452_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:65452_solr
   [junit4]   2> 896718 INFO  (zkCallback-3062-thread-1-processing-n:127.0.0.1:65452_solr) [n:127.0.0.1:65452_solr    ] o.a.s.c.Overseer Overseer (id=98853704602550276-127.0.0.1:65452_solr-n_0000000001) starting
   [junit4]   2> 896727 DEBUG (OverseerAutoScalingTriggerThread-98853704602550276-127.0.0.1:65452_solr-n_0000000001) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 896728 DEBUG (OverseerAutoScalingTriggerThread-98853704602550276-127.0.0.1:65452_solr-n_0000000001) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 7
   [junit4]   2> 896728 INFO  (OverseerAutoScalingTriggerThread-98853704602550276-127.0.0.1:65452_solr-n_0000000001) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.TriggerIntegrationTest TestEventQueueAction instantiated
   [junit4]   2> 896729 DEBUG (OverseerAutoScalingTriggerThread-98853704602550276-127.0.0.1:65452_solr-n_0000000001) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:51480_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 896729 DEBUG (OverseerAutoScalingTriggerThread-98853704602550276-127.0.0.1:65452_solr-n_0000000001) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.TriggerIntegrationTest$TestEventQueueAction}], enabled=true}
   [junit4]   2> 896729 DEBUG (OverseerAutoScalingTriggerThread-98853704602550276-127.0.0.1:65452_solr-n_0000000001) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:51480_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 896729 DEBUG (OverseerAutoScalingTriggerThread-98853704602550276-127.0.0.1:65452_solr-n_0000000001) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion -1
   [junit4]   2> 896729 DEBUG (OverseerAutoScalingTriggerThread-98853704602550276-127.0.0.1:65452_solr-n_0000000001) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7
   [junit4]   2> 896732 DEBUG (OverseerAutoScalingTriggerThread-98853704602550276-127.0.0.1:65452_solr-n_0000000001) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.TriggerIntegrationTest TestTriggerAction init
   [junit4]   2> 896733 DEBUG (ScheduledTrigger-3038-thread-1) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:51480_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 896733 DEBUG (OverseerAutoScalingTriggerThread-98853704602550276-127.0.0.1:65452_solr-n_0000000001) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7
   [junit4]   2> 896734 INFO  (ScheduledTrigger-3038-thread-2) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.SystemLogListener Collection .system does not exist, disabling logging.
   [junit4]   2> 896734 DEBUG (AutoscalingActionExecutor-3039-thread-1-processing-n:127.0.0.1:65452_solr) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
   [junit4]   2>   "id":"14eedebe80568080T3pg81goq8ctr8ue829rocy5k4",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":1508387834994000000,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[1508387834994000000],
   [junit4]   2>     "_dequeue_time_":1508387836035000000,
   [junit4]   2>     "_enqueue_time_":1508387835996000000,
   [junit4]   2>     "nodeNames":["127.0.0.1:51480_solr"],
   [junit4]   2>     "replaying":true}}
   [junit4]   2> 896734 INFO  (AutoscalingActionExecutor-3039-thread-1-processing-n:127.0.0.1:65452_solr) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.TriggerIntegrationTest -- event: {
   [junit4]   2>   "id":"14eedebe80568080T3pg81goq8ctr8ue829rocy5k4",
   [junit4]   2>   "source":"node_added_trigger1",
   [junit4]   2>   "eventTime":1508387834994000000,
   [junit4]   2>   "eventType":"NODEADDED",
   [junit4]   2>   "properties":{
   [junit4]   2>     "eventTimes":[1508387834994000000],
   [junit4]   2>     "_dequeue_time_":1508387836035000000,
   [junit4]   2>     "_enqueue_time_":1508387835996000000,
   [junit4]   2>     "nodeNames":["127.0.0.1:51480_solr"],
   [junit4]   2>     "replaying":true}}
   [junit4]   2> 896736 DEBUG (ScheduledTrigger-3038-thread-2) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 896736 DEBUG (ScheduledTrigger-3038-thread-2) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: [127.0.0.1:51480_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 897734 DEBUG (ScheduledTrigger-3038-thread-1) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:51480_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 897737 DEBUG (ScheduledTrigger-3038-thread-3) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 897737 DEBUG (ScheduledTrigger-3038-thread-3) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: [127.0.0.1:51480_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 898739 DEBUG (ScheduledTrigger-3038-thread-4) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 898740 DEBUG (ScheduledTrigger-3038-thread-4) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: [127.0.0.1:51480_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 898740 DEBUG (ScheduledTrigger-3038-thread-2) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:51480_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 899740 DEBUG (ScheduledTrigger-3038-thread-4) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 899740 DEBUG (ScheduledTrigger-3038-thread-4) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: [127.0.0.1:51480_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 899740 DEBUG (ScheduledTrigger-3038-thread-3) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:51480_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 900742 DEBUG (ScheduledTrigger-3038-thread-4) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 900742 DEBUG (ScheduledTrigger-3038-thread-4) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: [127.0.0.1:51480_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 900742 DEBUG (ScheduledTrigger-3038-thread-1) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:51480_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 901717 INFO  (TEST-TriggerIntegrationTest.testEventQueue-seed#[AE5B46876EAAA9F]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testEventQueue
   [junit4]   2> 901743 DEBUG (ScheduledTrigger-3038-thread-4) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
   [junit4]   2> 901743 DEBUG (ScheduledTrigger-3038-thread-2) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:51480_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 901743 DEBUG (ScheduledTrigger-3038-thread-4) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: [127.0.0.1:51480_solr, 127.0.0.1:65452_solr, 127.0.0.1:54906_solr]
   [junit4]   2> 901749 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testNodeMarkersRegistration
   [junit4]   2> 901750 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.a.s.c.a.TriggerIntegrationTest /autoscaling.json reset, new znode version 8
   [junit4]   2> 901751 DEBUG (zkCallback-3062-thread-1-processing-n:127.0.0.1:65452_solr) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 8
   [junit4]   2> 901751 DEBUG (OverseerAutoScalingTriggerThread-98853704602550276-127.0.0.1:65452_solr-n_0000000001) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8
   [junit4]   2> 901757 WARN  (OverseerAutoScalingTriggerThread-98853704602550276-127.0.0.1:65452_solr-n_0000000001) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.ScheduledTriggers Failed to remove events for removed trigger /autoscaling/events/.auto_add_replicas
   [junit4]   2> java.util.NoSuchElementException: [org.apache.zookeeper.Op$Delete@3e00904c]
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.ZkDistribStateManager.multi(ZkDistribStateManager.java:138)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers.removeTriggerZKData(ScheduledTriggers.java:358)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers.remove(ScheduledTriggers.java:337)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:197)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 901759 DEBUG (OverseerAutoScalingTriggerThread-98853704602550276-127.0.0.1:65452_solr-n_0000000001) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
   [junit4]   2> 901760 DEBUG (OverseerAutoScalingTriggerThread-98853704602550276-127.0.0.1:65452_solr-n_0000000001) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 901761 DEBUG (OverseerAutoScalingTriggerThread-98853704602550276-127.0.0.1:65452_solr-n_0000000001) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8
   [junit4]   2> 901767 INFO  (qtp1558037731-10796) [n:127.0.0.1:51480_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :overseerstatus with params action=OVERSEERSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 901778 INFO  (qtp1558037731-10796) [n:127.0.0.1:51480_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=OVERSEERSTATUS&wt=javabin&version=2} status=0 QTime=11
   [junit4]   2> 901779 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 901781 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5ba63959{/solr,null,AVAILABLE}
   [junit4]   2> 901781 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@1c8b5011{HTTP/1.1,[http/1.1]}{127.0.0.1:46044}
   [junit4]   2> 901781 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.e.j.s.Server Started @904748ms
   [junit4]   2> 901781 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=46044}
   [junit4]   2> 901782 ERROR (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 901782 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 8.0.0
   [junit4]   2> 901782 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 901782 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null, Default config dir: null
   [junit4]   2> 901782 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-19T04:37:21.083Z
   [junit4]   2> 901786 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 901795 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59235/solr
   [junit4]   2> 901818 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [n:127.0.0.1:46044_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 901820 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [n:127.0.0.1:46044_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 901823 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [n:127.0.0.1:46044_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46044_solr
   [junit4]   2> 901835 INFO  (zkCallback-3093-thread-1-processing-n:127.0.0.1:51480_solr) [n:127.0.0.1:51480_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 901836 INFO  (zkCallback-3081-thread-1-processing-n:127.0.0.1:54906_solr) [n:127.0.0.1:54906_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 901836 INFO  (zkCallback-3073-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 901836 INFO  (zkCallback-3062-thread-2-processing-n:127.0.0.1:65452_solr) [n:127.0.0.1:65452_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 901837 INFO  (zkCallback-3100-thread-1-processing-n:127.0.0.1:46044_solr) [n:127.0.0.1:46044_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 901966 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [n:127.0.0.1:46044_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46044.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@15e8a90d
   [junit4]   2> 901987 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [n:127.0.0.1:46044_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46044.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@15e8a90d
   [junit4]   2> 901988 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [n:127.0.0.1:46044_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46044.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@15e8a90d
   [junit4]   2> 901988 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [n:127.0.0.1:46044_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.TriggerIntegrationTest_AE5B46876EAAA9F-001/tempDir-001/node5/.
   [junit4]   2> 902024 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.a.s.c.a.TriggerIntegrationTest ====== KILL OVERSEER 1
   [junit4]   2> 902026 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@48dbcff4{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 902026 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1508212059
   [junit4]   2> 902027 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 902027 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@362755c9: rootName = solr_65452, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@610552da
   [junit4]   2> 902032 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 902032 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1770cf5a: rootName = solr_65452, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@3792a008
   [junit4]   2> 902035 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 902035 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@56a3e2bc: rootName = solr_65452, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@33ad2222
   [junit4]   2> 902038 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:65452_solr
   [junit4]   2> 902039 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 902039 INFO  (zkCallback-3093-thread-1-processing-n:127.0.0.1:51480_solr) [n:127.0.0.1:51480_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 902039 INFO  (zkCallback-3081-thread-1-processing-n:127.0.0.1:54906_solr) [n:127.0.0.1:54906_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 902039 INFO  (zkCallback-3073-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 902039 INFO  (zkCallback-3062-thread-2-processing-n:127.0.0.1:65452_solr) [n:127.0.0.1:65452_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 902039 INFO  (zkCallback-3100-thread-1-processing-n:127.0.0.1:46044_solr) [n:127.0.0.1:46044_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
   [junit4]   2> 902040 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 902040 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.a.s.c.Overseer Overseer (id=98853704602550276-127.0.0.1:65452_solr-n_0000000001) closing
   [junit4]   2> 902040 INFO  (OverseerStateUpdate-98853704602550276-127.0.0.1:65452_solr-n_0000000001) [n:127.0.0.1:65452_solr    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:65452_solr
   [junit4]   2> 902041 WARN  (OverseerAutoScalingTriggerThread-98853704602550276-127.0.0.1:65452_solr-n_0000000001) [n:127.0.0.1:65452_solr    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 902042 DEBUG (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 902044 INFO  (zkCallback-3081-thread-1-processing-n:127.0.0.1:54906_solr) [n:127.0.0.1:54906_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:54906_solr
   [junit4]   2> 902045 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@16bae0d5{/solr,null,UNAVAILABLE}
   [junit4]   2> 902047 INFO  (zkCallback-3081-thread-1-processing-n:127.0.0.1:54906_solr) [n:127.0.0.1:54906_solr    ] o.a.s.c.Overseer Overseer (id=98853704602550284-127.0.0.1:54906_solr-n_0000000002) starting
   [junit4]   2> 902057 DEBUG (OverseerAutoScalingTriggerThread-98853704602550284-127.0.0.1:54906_solr-n_0000000002) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
   [junit4]   2> 902058 DEBUG (OverseerAutoScalingTriggerThread-98853704602550284-127.0.0.1:54906_solr-n_0000000002) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 9
   [junit4]   2> 902058 DEBUG (OverseerAutoScalingTriggerThread-98853704602550284-127.0.0.1:54906_solr-n_0000000002) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:51480_solr, 127.0.0.1:54906_solr, 127.0.0.1:46044_solr]
   [junit4]   2> 902058 DEBUG (OverseerAutoScalingTriggerThread-98853704602550284-127.0.0.1:54906_solr-n_0000000002) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 9, lastZnodeVersion -1
   [junit4]   2> 902058 DEBUG (OverseerAutoScalingTriggerThread-98853704602550284-127.0.0.1:54906_solr-n_0000000002) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 9
   [junit4]   2> 902060 DEBUG (OverseerAutoScalingTriggerThread-98853704602550284-127.0.0.1:54906_solr-n_0000000002) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
   [junit4]   2> 902061 DEBUG (OverseerAutoScalingTriggerThread-98853704602550284-127.0.0.1:54906_solr-n_0000000002) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 9, lastZnodeVersion 9
   [junit4]   2> 902061 DEBUG (ScheduledTrigger-3049-thread-1) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:51480_solr, 127.0.0.1:54906_solr, 127.0.0.1:46044_solr]
   [junit4]   2> 903062 DEBUG (ScheduledTrigger-3049-thread-1) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:51480_solr, 127.0.0.1:54906_solr, 127.0.0.1:46044_solr]
   [junit4]   2> 904064 DEBUG (ScheduledTrigger-3049-thread-2) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:51480_solr, 127.0.0.1:54906_solr, 127.0.0.1:46044_solr]
   [junit4]   2> 905065 DEBUG (ScheduledTrigger-3049-thread-1) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:51480_solr, 127.0.0.1:54906_solr, 127.0.0.1:46044_solr]
   [junit4]   2> 906066 DEBUG (ScheduledTrigger-3049-thread-1) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:51480_solr, 127.0.0.1:54906_solr, 127.0.0.1:46044_solr]
   [junit4]   2> 907049 INFO  (TEST-TriggerIntegrationTest.testNodeMarkersRegistration-seed#[AE5B46876EAAA9F]) [    ] o.a.s.c.a.TriggerIntegrationTest ====== ADD TRIGGERS
   [junit4]   2> 907062 INFO  (qtp1651972274-10826) [n:127.0.0.1:46044_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 907064 INFO  (qtp1651972274-10826) [n:127.0.0.1:46044_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:59235/solr ready
   [junit4]   2> 907066 INFO  (qtp1074832861-10760) [n:127.0.0.1:54906_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 907067 DEBUG (ScheduledTrigger-3049-thread-1) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: [127.0.0.1:51480_solr, 127.0.0.1:54906_solr, 127.0.0.1:46044_solr]
   [junit4]   2> 907067 INFO  (qtp1651972274-10829) [n:127.0.0.1:46044_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 907068 INFO  (qtp1558037731-10791) [n:127.0.0.1:51480_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 907068 DEBUG (qtp1651972274-10826) [n:127.0.0.1:46044_solr    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 907070 INFO  (qtp1651972274-10826) [n:127.0.0.1:46044_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=15
   [junit4]   2> 907071 DEBUG (zkCallback-3081-thread-1-processing-n:127.0.0.1:54906_solr) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 10
   [junit4]   2> 907072 DEBUG (zkCallback-3081-thread-1-processing-n:127.0.0.1:54906_solr) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:51480_solr, 127.0.0.1:54906_solr, 127.0.0.1:46044_solr]
   [junit4]   2> 907073 DEBUG (zkCallback-3081-thread-1-processing-n:127.0.0.1:54906_solr) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:51480_solr, 127.0.0.1:54906_solr, 127.0.0.1:46044_solr]
   [junit4]   2> 907073 DEBUG (zkCallback-3081-thread-1-processing-n:127.0.0.1:54906_solr) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.TriggerIntegrationTest$TestEventMarkerAction}], enabled=true}
   [junit4]   2> 907073 DEBUG (OverseerAutoScalingTriggerThread-98853704602550284-127.0.0.1:54906_solr-n_0000000002) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 10
   [junit4]   2> 907076 INFO  (OverseerAutoScalingTriggerThread-98853704602550284-127.0.0.1:54906_solr-n_0000000002) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.TriggerIntegrationTest TestEventMarkerAction init
   [junit4]   2> 907077 INFO  (qtp1558037731-10791) [n:127.0.0.1:51480_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 907077 DEBUG (OverseerAutoScalingTriggerThread-98853704602550284-127.0.0.1:54906_solr-n_0000000002) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10
   [junit4]   2> 907078 DEBUG (ScheduledTrigger-3049-thread-3) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
   [junit4]   2> 907078 DEBUG (ScheduledTrigger-3049-thread-3) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.NodeAddedTrigger Found livenodes: [127.0.0.1:51480_solr, 127.0.0.1:54906_solr, 127.0.0.1:46044_solr]
   [junit4]   2> 907079 INFO  (qtp1558037731-10791) [n:127.0.0.1:51480_solr    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:59235/solr ready
   [junit4]   2> 907080 INFO  (qtp1074832861-10761) [n:127.0.0.1:54906_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 907082 INFO  (qtp1651972274-10831) [n:127.0.0.1:46044_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 907084 INFO  (qtp1558037731-10797) [n:127.0.0.1:51480_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
   [junit4]   2> 907084 DEBUG (qtp1558037731-10791) [n:127.0.0.1:51480_solr    ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
   [junit4]   2> 907086 INFO  (qtp1558037731-10791) [n:127.0.0.1:51480_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=13
   [junit4]   2> 907087 DEBUG (zkCallback-3081-thread-1-processing-n:127.0.0.1:54906_solr) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 11
   [junit4]   2> 907087 INFO  (qtp1651972274-10832) [n:127.0.0.1:46044_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :overseerstatus with params action=OVERSEERSTATUS&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 907088 DEBUG (zkCallback-3081-thread-1-processing-n:127.0.0.1:54906_solr) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:51480_solr, 127.0.0.1:54906_solr, 127.0.0.1:46044_solr]
   [junit4]   2> 907088 DEBUG (zkCallback-3081-thread-1-processing-n:127.0.0.1:54906_solr) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:51480_solr, 127.0.0.1:54906_solr, 127.0.0.1:46044_solr]
   [junit4]   2> 907089 DEBUG (zkCallback-3081-thread-1-processing-n:127.0.0.1:54906_solr) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:51480_solr, 127.0.0.1:54906_solr, 127.0.0.1:46044_solr]
   [junit4]   2> 907089 DEBUG (zkCallback-3081-thread-1-processing-n:127.0.0.1:54906_solr) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.TriggerIntegrationTest$TestEventMarkerAction}], enabled=true}
   [junit4]   2> 907089 DEBUG (OverseerAutoScalingTriggerThread-98853704602550284-127.0.0.1:54906_solr-n_0000000002) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 11
   [junit4]   2> 907092 INFO  (OverseerAutoScalingTriggerThread-98853704602550284-127.0.0.1:54906_solr-n_0000000002) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.TriggerIntegrationTest TestEventMarkerAction init
   [junit4]   2> 907093 DEBUG (OverseerAutoScalingTriggerThread-98853704602550284-127.0.0.1:54906_solr-n_0000000002) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion 11
   [junit4]   2> 907094 DEBUG (ScheduledTrigger-3049-thread-1) [n:127.0.0.1:54906_solr    ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_

[...truncated too long message...]

-thread-1) [n:127.0.0.1:44506_ c:collection1 s:shard1 r:core_node52 x:collection1_shard1_replica_p51] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard1.leader, tag=962103764
   [junit4]   2> 2853284 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[AE5B46876EAAA9F]) [    ] o.a.s.c.Overseer Overseer (id=98853831641464856-127.0.0.1:44506_-n_0000000006) closing
   [junit4]   2> 2853284 INFO  (OverseerStateUpdate-98853831641464856-127.0.0.1:44506_-n_0000000006) [n:127.0.0.1:44506_    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:44506_
   [junit4]   2> 2853289 WARN  (OverseerAutoScalingTriggerThread-98853831641464856-127.0.0.1:44506_-n_0000000006) [n:127.0.0.1:44506_    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 2853295 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[AE5B46876EAAA9F]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@2c65914c{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2853295 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[AE5B46876EAAA9F]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@61116ae{/,null,UNAVAILABLE}
   [junit4]   2> 2853298 ERROR (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[AE5B46876EAAA9F]) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2853302 INFO  (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[AE5B46876EAAA9F]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60330 60330
   [junit4]   2> 2863360 INFO  (Thread-5269) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60330 60330
   [junit4]   2> 2863365 WARN  (Thread-5269) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	9	/solr/aliases.json
   [junit4]   2> 	7	/solr/security.json
   [junit4]   2> 	7	/solr/configs/conf1
   [junit4]   2> 	6	/solr/collections/collection1/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	9	/solr/clusterstate.json
   [junit4]   2> 	9	/solr/clusterprops.json
   [junit4]   2> 	6	/solr/autoscaling.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	9	/solr/live_nodes
   [junit4]   2> 	9	/solr/collections
   [junit4]   2> 	6	/solr/overseer/queue
   [junit4]   2> 	6	/solr/autoscaling/events/.auto_add_replicas
   [junit4]   2> 	6	/solr/overseer/collection-queue-work
   [junit4]   2> 	6	/solr/overseer/queue-work
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ChaosMonkeySafeLeaderWithPullReplicasTest -Dtests.method=test -Dtests.seed=AE5B46876EAAA9F -Dtests.slow=true -Dtests.locale=zh-SG -Dtests.timezone=Australia/Perth -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   37.9s J1 | ChaosMonkeySafeLeaderWithPullReplicasTest.test <<<
   [junit4]    > Throwable #1: java.lang.NumberFormatException: null
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AE5B46876EAAA9F:82B18BB2D816C767]:0)
   [junit4]    > 	at java.lang.Long.parseLong(Long.java:552)
   [junit4]    > 	at java.lang.Long.parseLong(Long.java:631)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForReplicationFromReplicas(AbstractFullDistribZkTestBase.java:2125)
   [junit4]    > 	at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test(ChaosMonkeySafeLeaderWithPullReplicasTest.java:209)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_AE5B46876EAAA9F-001
   [junit4]   2> NOTE: test params are: codec=DummyCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=DUMMY, chunkSize=2583, maxDocsPerChunk=6, blockSize=704), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=DUMMY, chunkSize=2583, blockSize=704)), sim=RandomSimilarity(queryNorm=false): {}, locale=zh-SG, timezone=Australia/Perth
   [junit4]   2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_141 (64-bit)/cpus=3,threads=1,free=49389624,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [VersionInfoTest, MoreLikeThisHandlerTest, TestSkipOverseerOperations, TestSort, SynonymTokenizerTest, LoggingHandlerTest, TestExportWriter, SolrGraphiteReporterTest, LeaderFailureAfterFreshStartTest, SolrIndexConfigTest, TestBadConfig, TestSearcherReuse, V2ApiIntegrationTest, TlogReplayBufferedWhileIndexingTest, TestMacroExpander, TestSolrConfigHandlerConcurrent, TestComponentsName, HttpTriggerListenerTest, DocValuesMultiTest, SolrCloudExampleTest, TestDocBasedVersionConstraints, TestAnalyzedSuggestions, ExecutePlanActionTest, CoreAdminRequestStatusTest, TestMissingGroups, TestLegacyFieldCache, TestRebalanceLeaders, TestFoldingMultitermQuery, MetricsConfigTest, TestLockTree, CopyFieldTest, SolrCoreTest, DocValuesNotIndexedTest, TestTrie, TestFastOutputStream, TestDocumentBuilder, TestCrossCoreJoin, TestScoreJoinQPScore, BlockJoinFacetRandomTest, TestValueSourceCache, TestSolrJ, TestClassicSimilarityFactory, CloudExitableDirectoryReaderTest, PreAnalyzedFieldTest, StatelessScriptUpdateProcessorFactoryTest, SearchHandlerTest, TestLeaderInitiatedRecoveryThread, ResponseHeaderTest, TestMultiValuedNumericRangeQuery, TestLRUCache, PeerSyncTest, SubstringBytesRefFilterTest, TestPayloadCheckQParserPlugin, TestLegacyNumericUtils, OpenExchangeRatesOrgProviderTest, TestLuceneMatchVersion, FileBasedSpellCheckerTest, BufferStoreTest, TestCloudInspectUtil, ImplicitSnitchTest, TestNumericTerms64, TestConfig, TestReload, MoveReplicaHDFSTest, TestStressReorder, TestTolerantUpdateProcessorCloud, HdfsChaosMonkeyNothingIsSafeTest, ManagedSchemaRoundRobinCloudTest, TestCSVLoader, DeleteLastCustomShardedReplicaTest, TestExclusionRuleCollectionAccess, TestQueryTypes, TestConfigReload, RecoveryAfterSoftCommitTest, HLLSerializationTest, SolrJmxReporterTest, TestManagedResourceStorage, CustomCollectionTest, SimplePostToolTest, FullSolrCloudDistribCmdsTest, BasicAuthIntegrationTest, StressHdfsTest, SimpleCollectionCreateDeleteTest, TestCryptoKeys, LeaderInitiatedRecoveryOnShardRestartTest, ShardRoutingCustomTest, TestFieldCacheSort, TestQueryUtils, PeerSyncWithIndexFingerprintCachingTest, TestSystemIdResolver, AnalysisAfterCoreReloadTest, ClassificationUpdateProcessorTest, TestApiFramework, DocValuesTest, SpellPossibilityIteratorTest, TestNoOpRegenerator, SpellingQueryConverterTest, OutOfBoxZkACLAndCredentialsProvidersTest, AliasIntegrationTest, CdcrRequestHandlerTest, TestCustomStream, ZkSolrClientTest, TestWordDelimiterFilterFactory, TestCharFilters, TestSolrCoreParser, DistributedQueueTest, TestStandardQParsers, PluginInfoTest, BasicZkTest, DirectoryFactoryTest, ConfigureRecoveryStrategyTest, TestConfigsApi, TestSolrDeletionPolicy1, TestSQLHandlerNonCloud, SpellCheckCollatorWithCollapseTest, TestFastLRUCache, TestAddFieldRealTimeGet, TestSuggestSpellingConverter, TestNestedDocsSort, SampleTest, ShardSplitTest, CollectionStateFormat2Test, TestGroupingSearch, HdfsLockFactoryTest, TestTrieFacet, DocumentAnalysisRequestHandlerTest, OverseerTaskQueueTest, TestCustomSort, OverseerCollectionConfigSetProcessorTest, TestPolicyCloud, InfoHandlerTest, JsonLoaderTest, SolrCoreCheckLockOnStartupTest, DocValuesMissingTest, TestHdfsUpdateLog, RulesTest, HdfsBasicDistributedZk2Test, TestSlowCompositeReaderWrapper, TestComplexPhraseLeadingWildcard, TestHashPartitioner, FieldMutatingUpdateProcessorTest, TestHttpShardHandlerFactory, SimpleFacetsTest, BasicDistributedZk2Test, CoreAdminHandlerTest, TestDocSet, TestBulkSchemaConcurrent, DistribDocExpirationUpdateProcessorTest, UpdateRequestProcessorFactoryTest, TestTrackingShardHandlerFactory, DistributedDebugComponentTest, BlockCacheTest, TestTestInjection, BadComponentTest, HdfsRestartWhileUpdatingTest, TestRTGBase, RestartWhileUpdatingTest, TermVectorComponentDistributedTest, TestFieldCacheVsDocValues, TestBackupRepositoryFactory, TestCloudPivotFacet, LargeFieldTest, OpenCloseCoreStressTest, TestPerFieldSimilarity, HealthCheckHandlerTest, TestSerializedLuceneMatchVersion, TestLuceneIndexBackCompat, TestBM25SimilarityFactory, UUIDUpdateProcessorFallbackTest, DistributedFacetPivotSmallTest, LeaderElectionContextKeyTest, TestStressLucene, RankQueryTest, TestSchemaNameResource, DistributedFacetPivotLongTailTest, AutoscalingHistoryHandlerTest, DeleteStatusTest, LukeRequestHandlerTest, TestPartialUpdateDeduplication, DirectSolrSpellCheckerTest, BasicDistributedZkTest, OverseerTest, TestRandomFaceting, TestZkChroot, TestRandomDVFaceting, TestFaceting, ZkControllerTest, TestCoreContainer, SpellCheckComponentTest, QueryElevationComponentTest, BadIndexSchemaTest, BasicFunctionalityTest, TestIndexSearcher, HighlighterTest, SoftAutoCommitTest, ExternalFileFieldSortTest, TestPhraseSuggestions, TestSweetSpotSimilarityFactory, TestFastWriter, DOMUtilTest, TestUtils, SystemInfoHandlerTest, FileUtilsTest, DistributedMLTComponentTest, DistributedIntervalFacetingTest, TestEmbeddedSolrServerConstructors, TestEmbeddedSolrServerSchemaAPI, TestJettySolrRunner, ConnectionReuseTest, AssignTest, ChaosMonkeySafeLeaderWithPullReplicasTest]
   [junit4] Completed [493/744 (2!)] on J1 in 37.89s, 1 test, 1 error <<< FAILURES!

[...truncated 48072 lines...]

Mime
View raw message