lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Policeman Jenkins Server <jenk...@thetaphi.de>
Subject [JENKINS] Lucene-Solr-7.x-Linux (64bit/jdk1.8.0_144) - Build # 709 - Still Unstable!
Date Tue, 31 Oct 2017 08:35:46 GMT
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Linux/709/
Java: 64bit/jdk1.8.0_144 -XX:+UseCompressedOops -XX:+UseParallelGC

3 tests failed.
FAILED:  org.apache.solr.cloud.MoveReplicaHDFSFailoverTest.testDataDirAndUlogAreMaintained

Error Message:
Error from server at http://127.0.0.1:37977/solr: Timed out waiting to see all replicas: [movereplicatest_coll2_shard1_replica_n21] in cluster state.

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:37977/solr: Timed out waiting to see all replicas: [movereplicatest_coll2_shard1_replica_n21] in cluster state.
	at __randomizedtesting.SeedInfo.seed([9CC2B3855A3EE02F:B3F12656E64E520D]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:658)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1096)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:875)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:808)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
	at org.apache.solr.cloud.MoveReplicaHDFSFailoverTest.testDataDirAndUlogAreMaintained(MoveReplicaHDFSFailoverTest.java:84)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.MoveReplicaHDFSFailoverTest.testOldReplicaIsDeletedInRaceCondition

Error Message:
Error from server at http://127.0.0.1:37977/solr: Could not fully create collection: movereplicatest_coll4

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:37977/solr: Could not fully create collection: movereplicatest_coll4
	at __randomizedtesting.SeedInfo.seed([9CC2B3855A3EE02F:96923CF39A3E818E]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:658)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1096)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:875)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:808)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
	at org.apache.solr.cloud.MoveReplicaHDFSFailoverTest.testOldReplicaIsDeletedInRaceCondition(MoveReplicaHDFSFailoverTest.java:172)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.cloud.autoscaling.ExecutePlanActionTest.testIntegration

Error Message:
Timed out waiting for replicas of collection to be 2 again null Live Nodes: [127.0.0.1:32879_solr] Last available state: DocCollection(testIntegration//collections/testIntegration/state.json/9)={   "pullReplicas":"0",   "replicationFactor":"2",   "shards":{"shard1":{       "range":"80000000-7fffffff",       "state":"active",       "replicas":{         "core_node3":{           "core":"testIntegration_shard1_replica_n1",           "base_url":"http://127.0.0.1:38341/solr",           "node_name":"127.0.0.1:38341_solr",           "state":"down",           "type":"NRT"},         "core_node4":{           "core":"testIntegration_shard1_replica_n2",           "base_url":"http://127.0.0.1:32879/solr",           "node_name":"127.0.0.1:32879_solr",           "state":"active",           "type":"NRT",           "leader":"true"}}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"false",   "nrtReplicas":"2",   "tlogReplicas":"0"}

Stack Trace:
java.lang.AssertionError: Timed out waiting for replicas of collection to be 2 again
null
Live Nodes: [127.0.0.1:32879_solr]
Last available state: DocCollection(testIntegration//collections/testIntegration/state.json/9)={
  "pullReplicas":"0",
  "replicationFactor":"2",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node3":{
          "core":"testIntegration_shard1_replica_n1",
          "base_url":"http://127.0.0.1:38341/solr",
          "node_name":"127.0.0.1:38341_solr",
          "state":"down",
          "type":"NRT"},
        "core_node4":{
          "core":"testIntegration_shard1_replica_n2",
          "base_url":"http://127.0.0.1:32879/solr",
          "node_name":"127.0.0.1:32879_solr",
          "state":"active",
          "type":"NRT",
          "leader":"true"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false",
  "nrtReplicas":"2",
  "tlogReplicas":"0"}
	at __randomizedtesting.SeedInfo.seed([9CC2B3855A3EE02F:2CA3BDA97F01410A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
	at org.apache.solr.cloud.autoscaling.ExecutePlanActionTest.testIntegration(ExecutePlanActionTest.java:209)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)




Build Log:
[...truncated 13106 lines...]
   [junit4] Suite: org.apache.solr.cloud.MoveReplicaHDFSFailoverTest
   [junit4]   2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.MoveReplicaHDFSFailoverTest_9CC2B3855A3EE02F-001/init-core-data-001
   [junit4]   2> 2043149 WARN  (SUITE-MoveReplicaHDFSFailoverTest-seed#[9CC2B3855A3EE02F]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=12 numCloses=12
   [junit4]   2> 2043149 INFO  (SUITE-MoveReplicaHDFSFailoverTest-seed#[9CC2B3855A3EE02F]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 2043150 INFO  (SUITE-MoveReplicaHDFSFailoverTest-seed#[9CC2B3855A3EE02F]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 2043150 INFO  (SUITE-MoveReplicaHDFSFailoverTest-seed#[9CC2B3855A3EE02F]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> 2043150 INFO  (SUITE-MoveReplicaHDFSFailoverTest-seed#[9CC2B3855A3EE02F]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.MoveReplicaHDFSFailoverTest_9CC2B3855A3EE02F-001/tempDir-001
   [junit4]   2> 2043150 INFO  (SUITE-MoveReplicaHDFSFailoverTest-seed#[9CC2B3855A3EE02F]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 2043151 INFO  (Thread-4229) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2043151 INFO  (Thread-4229) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 2043152 ERROR (Thread-4229) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 2043251 INFO  (SUITE-MoveReplicaHDFSFailoverTest-seed#[9CC2B3855A3EE02F]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:46209
   [junit4]   2> 2043255 INFO  (jetty-launcher-2921-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2043255 INFO  (jetty-launcher-2921-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 2043255 INFO  (jetty-launcher-2921-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2e4b9e43{/solr,null,AVAILABLE}
   [junit4]   2> 2043255 INFO  (jetty-launcher-2921-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@50e35c9b{/solr,null,AVAILABLE}
   [junit4]   2> 2043256 INFO  (jetty-launcher-2921-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@ebb8050{HTTP/1.1,[http/1.1]}{127.0.0.1:37977}
   [junit4]   2> 2043256 INFO  (jetty-launcher-2921-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@7c08c455{HTTP/1.1,[http/1.1]}{127.0.0.1:38163}
   [junit4]   2> 2043256 INFO  (jetty-launcher-2921-thread-1) [    ] o.e.j.s.Server Started @2045375ms
   [junit4]   2> 2043256 INFO  (jetty-launcher-2921-thread-2) [    ] o.e.j.s.Server Started @2045375ms
   [junit4]   2> 2043256 INFO  (jetty-launcher-2921-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=37977}
   [junit4]   2> 2043256 INFO  (jetty-launcher-2921-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=38163}
   [junit4]   2> 2043256 ERROR (jetty-launcher-2921-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2043256 ERROR (jetty-launcher-2921-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 2043256 INFO  (jetty-launcher-2921-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.2.0
   [junit4]   2> 2043256 INFO  (jetty-launcher-2921-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.2.0
   [junit4]   2> 2043256 INFO  (jetty-launcher-2921-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2043256 INFO  (jetty-launcher-2921-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 2043256 INFO  (jetty-launcher-2921-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2043257 INFO  (jetty-launcher-2921-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 2043257 INFO  (jetty-launcher-2921-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-31T07:55:21.861Z
   [junit4]   2> 2043257 INFO  (jetty-launcher-2921-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-10-31T07:55:21.861Z
   [junit4]   2> 2043258 INFO  (jetty-launcher-2921-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2043258 INFO  (jetty-launcher-2921-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 2043291 INFO  (jetty-launcher-2921-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46209/solr
   [junit4]   2> 2043291 INFO  (jetty-launcher-2921-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:46209/solr
   [junit4]   2> 2043331 INFO  (jetty-launcher-2921-thread-2) [n:127.0.0.1:38163_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2043334 INFO  (jetty-launcher-2921-thread-2) [n:127.0.0.1:38163_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:38163_solr
   [junit4]   2> 2043334 INFO  (jetty-launcher-2921-thread-2) [n:127.0.0.1:38163_solr    ] o.a.s.c.Overseer Overseer (id=98922431889735685-127.0.0.1:38163_solr-n_0000000000) starting
   [junit4]   2> 2043334 INFO  (jetty-launcher-2921-thread-1) [n:127.0.0.1:37977_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 2043335 INFO  (jetty-launcher-2921-thread-1) [n:127.0.0.1:37977_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37977_solr
   [junit4]   2> 2043338 INFO  (zkCallback-2933-thread-1-processing-n:127.0.0.1:37977_solr) [n:127.0.0.1:37977_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2043338 INFO  (zkCallback-2932-thread-1-processing-n:127.0.0.1:38163_solr) [n:127.0.0.1:38163_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 2043348 INFO  (jetty-launcher-2921-thread-2) [n:127.0.0.1:38163_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38163_solr
   [junit4]   2> 2043348 INFO  (zkCallback-2933-thread-1-processing-n:127.0.0.1:37977_solr) [n:127.0.0.1:37977_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2043349 INFO  (zkCallback-2932-thread-1-processing-n:127.0.0.1:38163_solr) [n:127.0.0.1:38163_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 2043420 INFO  (jetty-launcher-2921-thread-1) [n:127.0.0.1:37977_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37977.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3f587dec
   [junit4]   2> 2043425 INFO  (jetty-launcher-2921-thread-1) [n:127.0.0.1:37977_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37977.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3f587dec
   [junit4]   2> 2043425 INFO  (jetty-launcher-2921-thread-1) [n:127.0.0.1:37977_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37977.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3f587dec
   [junit4]   2> 2043426 INFO  (jetty-launcher-2921-thread-1) [n:127.0.0.1:37977_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.MoveReplicaHDFSFailoverTest_9CC2B3855A3EE02F-001/tempDir-001/node1/.
   [junit4]   2> 2043434 INFO  (jetty-launcher-2921-thread-2) [n:127.0.0.1:38163_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38163.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3f587dec
   [junit4]   2> 2043442 INFO  (jetty-launcher-2921-thread-2) [n:127.0.0.1:38163_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38163.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3f587dec
   [junit4]   2> 2043442 INFO  (jetty-launcher-2921-thread-2) [n:127.0.0.1:38163_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38163.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3f587dec
   [junit4]   2> 2043443 INFO  (jetty-launcher-2921-thread-2) [n:127.0.0.1:38163_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.MoveReplicaHDFSFailoverTest_9CC2B3855A3EE02F-001/tempDir-001/node2/.
   [junit4]   2> 2043461 INFO  (SUITE-MoveReplicaHDFSFailoverTest-seed#[9CC2B3855A3EE02F]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 2043461 INFO  (SUITE-MoveReplicaHDFSFailoverTest-seed#[9CC2B3855A3EE02F]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:46209/solr ready
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 2043519 WARN  (SUITE-MoveReplicaHDFSFailoverTest-seed#[9CC2B3855A3EE02F]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 2043527 WARN  (SUITE-MoveReplicaHDFSFailoverTest-seed#[9CC2B3855A3EE02F]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 2043528 INFO  (SUITE-MoveReplicaHDFSFailoverTest-seed#[9CC2B3855A3EE02F]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 2043542 INFO  (SUITE-MoveReplicaHDFSFailoverTest-seed#[9CC2B3855A3EE02F]-worker) [    ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_localdomain_44891_hdfs____.t3lpp2/webapp
   [junit4]   2> 2044009 INFO  (SUITE-MoveReplicaHDFSFailoverTest-seed#[9CC2B3855A3EE02F]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost.localdomain:44891
   [junit4]   2> 2044095 WARN  (SUITE-MoveReplicaHDFSFailoverTest-seed#[9CC2B3855A3EE02F]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 2044096 INFO  (SUITE-MoveReplicaHDFSFailoverTest-seed#[9CC2B3855A3EE02F]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 2044106 INFO  (SUITE-MoveReplicaHDFSFailoverTest-seed#[9CC2B3855A3EE02F]-worker) [    ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_39665_datanode____.a2dwzu/webapp
   [junit4]   2> 2044525 INFO  (SUITE-MoveReplicaHDFSFailoverTest-seed#[9CC2B3855A3EE02F]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:39665
   [junit4]   2> 2044576 WARN  (SUITE-MoveReplicaHDFSFailoverTest-seed#[9CC2B3855A3EE02F]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 2044577 INFO  (SUITE-MoveReplicaHDFSFailoverTest-seed#[9CC2B3855A3EE02F]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 2044589 INFO  (SUITE-MoveReplicaHDFSFailoverTest-seed#[9CC2B3855A3EE02F]-worker) [    ] o.m.log Extract jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.4-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_37695_datanode____.i9gi7f/webapp
   [junit4]   2> 2044675 ERROR (DataNode: [[[DISK]file:/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.MoveReplicaHDFSFailoverTest_9CC2B3855A3EE02F-001/tempDir-002/hdfsBaseDir/data/data1/, [DISK]file:/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.MoveReplicaHDFSFailoverTest_9CC2B3855A3EE02F-001/tempDir-002/hdfsBaseDir/data/data2/]]  heartbeating to localhost.localdomain/127.0.0.1:43961) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 2044697 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x9338a369806b: from storage DS-44f3eb55-044e-4c00-9468-1e1d474439a5 node DatanodeRegistration(127.0.0.1:34339, datanodeUuid=ccd1c1c4-c6c2-4193-8c03-a2361fea8fca, infoPort=45391, infoSecurePort=0, ipcPort=39583, storageInfo=lv=-56;cid=testClusterID;nsid=1040198061;c=0), blocks: 0, hasStaleStorage: true, processing time: 1 msecs
   [junit4]   2> 2044697 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x9338a369806b: from storage DS-c90bd919-9d3a-4d4c-bd2e-ec3594ad6810 node DatanodeRegistration(127.0.0.1:34339, datanodeUuid=ccd1c1c4-c6c2-4193-8c03-a2361fea8fca, infoPort=45391, infoSecurePort=0, ipcPort=39583, storageInfo=lv=-56;cid=testClusterID;nsid=1040198061;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 2045101 INFO  (SUITE-MoveReplicaHDFSFailoverTest-seed#[9CC2B3855A3EE02F]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:37695
   [junit4]   2> 2045237 ERROR (DataNode: [[[DISK]file:/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.MoveReplicaHDFSFailoverTest_9CC2B3855A3EE02F-001/tempDir-002/hdfsBaseDir/data/data3/, [DISK]file:/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.MoveReplicaHDFSFailoverTest_9CC2B3855A3EE02F-001/tempDir-002/hdfsBaseDir/data/data4/]]  heartbeating to localhost.localdomain/127.0.0.1:43961) [    ] o.a.h.h.s.d.DirectoryScanner dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value below 1 ms/sec. Assuming default value of 1000
   [junit4]   2> 2045243 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x9338c40bdef9: from storage DS-f0fc8002-61af-4d14-8f31-97b125b47c8e node DatanodeRegistration(127.0.0.1:46383, datanodeUuid=202a6829-4b5a-47d9-9ecf-a4c3a50605ab, infoPort=34141, infoSecurePort=0, ipcPort=37173, storageInfo=lv=-56;cid=testClusterID;nsid=1040198061;c=0), blocks: 0, hasStaleStorage: true, processing time: 1 msecs
   [junit4]   2> 2045243 INFO  (Block report processor) [    ] BlockStateChange BLOCK* processReport 0x9338c40bdef9: from storage DS-026d6351-484e-4068-ad9c-71ac8f3f2d83 node DatanodeRegistration(127.0.0.1:46383, datanodeUuid=202a6829-4b5a-47d9-9ecf-a4c3a50605ab, infoPort=34141, infoSecurePort=0, ipcPort=37173, storageInfo=lv=-56;cid=testClusterID;nsid=1040198061;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 2045421 INFO  (TEST-MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted-seed#[9CC2B3855A3EE02F]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testOldReplicaIsDeleted
   [junit4]   2> 2045422 INFO  (qtp1754572279-16141) [n:127.0.0.1:37977_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=movereplicatest_coll3&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:37977_solr&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2045423 INFO  (OverseerThreadFactory-6264-thread-1-processing-n:127.0.0.1:38163_solr) [n:127.0.0.1:38163_solr    ] o.a.s.c.CreateCollectionCmd Create collection movereplicatest_coll3
   [junit4]   2> 2045525 INFO  (OverseerStateUpdate-98922431889735685-127.0.0.1:38163_solr-n_0000000000) [n:127.0.0.1:38163_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"movereplicatest_coll3",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"movereplicatest_coll3_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"http://127.0.0.1:37977/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 2045726 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf1&newCollection=true&name=movereplicatest_coll3_shard1_replica_n1&action=CREATE&numShards=1&collection=movereplicatest_coll3&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 2045727 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2045828 INFO  (zkCallback-2933-thread-1-processing-n:127.0.0.1:37977_solr) [n:127.0.0.1:37977_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/movereplicatest_coll3/state.json] for collection [movereplicatest_coll3] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2045829 INFO  (zkCallback-2933-thread-2-processing-n:127.0.0.1:37977_solr) [n:127.0.0.1:37977_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/movereplicatest_coll3/state.json] for collection [movereplicatest_coll3] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2046732 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 2046750 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.s.IndexSchema [movereplicatest_coll3_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 2046752 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2046752 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'movereplicatest_coll3_shard1_replica_n1' using configuration from collection movereplicatest_coll3, trusted=true
   [junit4]   2> 2046752 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_37977.solr.core.movereplicatest_coll3.shard1.replica_n1' (registry 'solr.core.movereplicatest_coll3.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3f587dec
   [junit4]   2> 2046752 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost.localdomain:43961/data
   [junit4]   2> 2046752 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 2046752 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2046752 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.SolrCore [[movereplicatest_coll3_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.MoveReplicaHDFSFailoverTest_9CC2B3855A3EE02F-001/tempDir-001/node1/movereplicatest_coll3_shard1_replica_n1], dataDir=[hdfs://localhost.localdomain:43961/data/movereplicatest_coll3/core_node2/data/]
   [junit4]   2> 2046753 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost.localdomain:43961/data/movereplicatest_coll3/core_node2/data/snapshot_metadata
   [junit4]   2> 2046763 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost.localdomain:43961/data/movereplicatest_coll3/core_node2/data
   [junit4]   2> 2046778 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost.localdomain:43961/data/movereplicatest_coll3/core_node2/data/index
   [junit4]   2> 2046811 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34339 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-026d6351-484e-4068-ad9c-71ac8f3f2d83:NORMAL:127.0.0.1:46383|RBW], ReplicaUC[[DISK]DS-44f3eb55-044e-4c00-9468-1e1d474439a5:NORMAL:127.0.0.1:34339|RBW]]} size 0
   [junit4]   2> 2046812 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:46383 is added to blk_1073741825_1001{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-026d6351-484e-4068-ad9c-71ac8f3f2d83:NORMAL:127.0.0.1:46383|RBW], ReplicaUC[[DISK]DS-44f3eb55-044e-4c00-9468-1e1d474439a5:NORMAL:127.0.0.1:34339|RBW]]} size 0
   [junit4]   2> 2046855 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 2046855 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2046855 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
   [junit4]   2> 2046862 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2046862 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2046875 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2b5c2936[movereplicatest_coll3_shard1_replica_n1] main]
   [junit4]   2> 2046877 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2046877 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2046878 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2046879 INFO  (searcherExecutor-6269-thread-1-processing-n:127.0.0.1:37977_solr x:movereplicatest_coll3_shard1_replica_n1 s:shard1 c:movereplicatest_coll3 r:core_node2) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.SolrCore [movereplicatest_coll3_shard1_replica_n1] Registered new searcher Searcher@2b5c2936[movereplicatest_coll3_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2046879 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1582758914144862208
   [junit4]   2> 2046882 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2046882 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2046882 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:37977/solr/movereplicatest_coll3_shard1_replica_n1/
   [junit4]   2> 2046882 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2046882 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:37977/solr/movereplicatest_coll3_shard1_replica_n1/ has no replicas
   [junit4]   2> 2046882 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 2046883 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:37977/solr/movereplicatest_coll3_shard1_replica_n1/ shard1
   [junit4]   2> 2046986 INFO  (zkCallback-2933-thread-2-processing-n:127.0.0.1:37977_solr) [n:127.0.0.1:37977_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/movereplicatest_coll3/state.json] for collection [movereplicatest_coll3] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2046986 INFO  (zkCallback-2933-thread-1-processing-n:127.0.0.1:37977_solr) [n:127.0.0.1:37977_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/movereplicatest_coll3/state.json] for collection [movereplicatest_coll3] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2047034 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2047035 INFO  (qtp1754572279-16145) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf1&newCollection=true&name=movereplicatest_coll3_shard1_replica_n1&action=CREATE&numShards=1&collection=movereplicatest_coll3&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1308
   [junit4]   2> 2047037 INFO  (qtp1754572279-16141) [n:127.0.0.1:37977_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 2047135 INFO  (zkCallback-2933-thread-2-processing-n:127.0.0.1:37977_solr) [n:127.0.0.1:37977_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/movereplicatest_coll3/state.json] for collection [movereplicatest_coll3] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2047135 INFO  (zkCallback-2933-thread-1-processing-n:127.0.0.1:37977_solr) [n:127.0.0.1:37977_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/movereplicatest_coll3/state.json] for collection [movereplicatest_coll3] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 2047424 INFO  (OverseerCollectionConfigSetProcessor-98922431889735685-127.0.0.1:38163_solr-n_0000000000) [n:127.0.0.1:38163_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2048037 INFO  (qtp1754572279-16141) [n:127.0.0.1:37977_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=movereplicatest_coll3&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:37977_solr&wt=javabin&version=2} status=0 QTime=2615
   [junit4]   2> 2048062 INFO  (qtp1754572279-16143) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [movereplicatest_coll3_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[1 (1582758915361210368)]} 0 22
   [junit4]   2> 2048064 INFO  (qtp1754572279-16133) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [movereplicatest_coll3_shard1_replica_n1]  webapp=/solr path=/update params={wt=javabin&version=2}{add=[2 (1582758915385327616)]} 0 1
   [junit4]   2> 2048074 INFO  (TEST-MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted-seed#[9CC2B3855A3EE02F]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@ebb8050{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2048074 INFO  (TEST-MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted-seed#[9CC2B3855A3EE02F]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1484254271
   [junit4]   2> 2048074 INFO  (TEST-MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted-seed#[9CC2B3855A3EE02F]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 2048074 INFO  (TEST-MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted-seed#[9CC2B3855A3EE02F]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4f50103c: rootName = solr_37977, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@281f22e9
   [junit4]   2> 2048079 INFO  (TEST-MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted-seed#[9CC2B3855A3EE02F]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 2048079 INFO  (TEST-MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted-seed#[9CC2B3855A3EE02F]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2adfb12f: rootName = solr_37977, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@56070cd8
   [junit4]   2> 2048082 INFO  (TEST-MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted-seed#[9CC2B3855A3EE02F]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 2048082 INFO  (TEST-MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted-seed#[9CC2B3855A3EE02F]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3a04415b: rootName = solr_37977, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@635df43b
   [junit4]   2> 2048082 INFO  (TEST-MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted-seed#[9CC2B3855A3EE02F]) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:37977_solr
   [junit4]   2> 2048083 INFO  (zkCallback-2940-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 2048083 INFO  (zkCallback-2932-thread-1-processing-n:127.0.0.1:38163_solr) [n:127.0.0.1:38163_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 2048083 INFO  (zkCallback-2933-thread-1-processing-n:127.0.0.1:37977_solr) [n:127.0.0.1:37977_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
   [junit4]   2> 2048083 INFO  (TEST-MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted-seed#[9CC2B3855A3EE02F]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 2048083 INFO  (coreCloseExecutor-6274-thread-1) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.SolrCore [movereplicatest_coll3_shard1_replica_n1]  CLOSING SolrCore org.apache.solr.core.SolrCore@5144413a
   [junit4]   2> 2048083 INFO  (coreCloseExecutor-6274-thread-1) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.movereplicatest_coll3.shard1.replica_n1, tag=1363427642
   [junit4]   2> 2048084 INFO  (coreCloseExecutor-6274-thread-1) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@113d08c0: rootName = solr_37977, domain = solr.core.movereplicatest_coll3.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.movereplicatest_coll3.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@68405e46
   [junit4]   2> 2048091 INFO  (coreCloseExecutor-6274-thread-1) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.movereplicatest_coll3.shard1.leader, tag=1363427642
   [junit4]   2> 2048091 INFO  (coreCloseExecutor-6274-thread-1) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 Committing on IndexWriter close.
   [junit4]   2> 2048091 INFO  (coreCloseExecutor-6274-thread-1) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@62b83536 commitCommandVersion:0
   [junit4]   2> 2048115 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:46383 is added to blk_1073741827_1003{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-44f3eb55-044e-4c00-9468-1e1d474439a5:NORMAL:127.0.0.1:34339|RBW], ReplicaUC[[DISK]DS-026d6351-484e-4068-ad9c-71ac8f3f2d83:NORMAL:127.0.0.1:46383|FINALIZED]]} size 0
   [junit4]   2> 2048116 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34339 is added to blk_1073741827_1003 size 182
   [junit4]   2> 2048127 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:46383 is added to blk_1073741828_1004{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-c90bd919-9d3a-4d4c-bd2e-ec3594ad6810:NORMAL:127.0.0.1:34339|RBW], ReplicaUC[[DISK]DS-f0fc8002-61af-4d14-8f31-97b125b47c8e:NORMAL:127.0.0.1:46383|FINALIZED]]} size 0
   [junit4]   2> 2048128 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34339 is added to blk_1073741828_1004{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-c90bd919-9d3a-4d4c-bd2e-ec3594ad6810:NORMAL:127.0.0.1:34339|RBW], ReplicaUC[[DISK]DS-f0fc8002-61af-4d14-8f31-97b125b47c8e:NORMAL:127.0.0.1:46383|FINALIZED]]} size 0
   [junit4]   2> 2048144 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34339 is added to blk_1073741829_1005{UCState=COMMITTED, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-f0fc8002-61af-4d14-8f31-97b125b47c8e:NORMAL:127.0.0.1:46383|RBW], ReplicaUC[[DISK]DS-44f3eb55-044e-4c00-9468-1e1d474439a5:NORMAL:127.0.0.1:34339|RBW]]} size 59
   [junit4]   2> 2048144 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:46383 is added to blk_1073741829_1005 size 59
   [junit4]   2> 2048183 INFO  (zkCallback-2933-thread-2-processing-n:127.0.0.1:37977_solr) [n:127.0.0.1:37977_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/movereplicatest_coll3/state.json] for collection [movereplicatest_coll3] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2048183 INFO  (zkCallback-2933-thread-1-processing-n:127.0.0.1:37977_solr) [n:127.0.0.1:37977_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/movereplicatest_coll3/state.json] for collection [movereplicatest_coll3] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2048543 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34339 is added to blk_1073741830_1006{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-f0fc8002-61af-4d14-8f31-97b125b47c8e:NORMAL:127.0.0.1:46383|RBW], ReplicaUC[[DISK]DS-c90bd919-9d3a-4d4c-bd2e-ec3594ad6810:NORMAL:127.0.0.1:34339|FINALIZED]]} size 0
   [junit4]   2> 2048544 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:46383 is added to blk_1073741830_1006{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-f0fc8002-61af-4d14-8f31-97b125b47c8e:NORMAL:127.0.0.1:46383|RBW], ReplicaUC[[DISK]DS-c90bd919-9d3a-4d4c-bd2e-ec3594ad6810:NORMAL:127.0.0.1:34339|FINALIZED]]} size 0
   [junit4]   2> 2048550 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34339 is added to blk_1073741831_1007{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-f0fc8002-61af-4d14-8f31-97b125b47c8e:NORMAL:127.0.0.1:46383|RBW], ReplicaUC[[DISK]DS-44f3eb55-044e-4c00-9468-1e1d474439a5:NORMAL:127.0.0.1:34339|FINALIZED]]} size 0
   [junit4]   2> 2048555 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:46383 is added to blk_1073741831_1007{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-44f3eb55-044e-4c00-9468-1e1d474439a5:NORMAL:127.0.0.1:34339|FINALIZED], ReplicaUC[[DISK]DS-026d6351-484e-4068-ad9c-71ac8f3f2d83:NORMAL:127.0.0.1:46383|FINALIZED]]} size 0
   [junit4]   2> 2048562 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:46383 is added to blk_1073741832_1008{UCState=COMMITTED, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-44f3eb55-044e-4c00-9468-1e1d474439a5:NORMAL:127.0.0.1:34339|RBW], ReplicaUC[[DISK]DS-f0fc8002-61af-4d14-8f31-97b125b47c8e:NORMAL:127.0.0.1:46383|RBW]]} size 69
   [junit4]   2> 2048562 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34339 is added to blk_1073741832_1008 size 69
   [junit4]   2> 2048967 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34339 is added to blk_1073741833_1009{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-f0fc8002-61af-4d14-8f31-97b125b47c8e:NORMAL:127.0.0.1:46383|RBW], ReplicaUC[[DISK]DS-44f3eb55-044e-4c00-9468-1e1d474439a5:NORMAL:127.0.0.1:34339|FINALIZED]]} size 0
   [junit4]   2> 2048967 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:46383 is added to blk_1073741833_1009 size 78
   [junit4]   2> 2048971 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34339 is added to blk_1073741834_1010{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-026d6351-484e-4068-ad9c-71ac8f3f2d83:NORMAL:127.0.0.1:46383|RBW], ReplicaUC[[DISK]DS-c90bd919-9d3a-4d4c-bd2e-ec3594ad6810:NORMAL:127.0.0.1:34339|FINALIZED]]} size 0
   [junit4]   2> 2048972 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:46383 is added to blk_1073741834_1010{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-c90bd919-9d3a-4d4c-bd2e-ec3594ad6810:NORMAL:127.0.0.1:34339|FINALIZED], ReplicaUC[[DISK]DS-f0fc8002-61af-4d14-8f31-97b125b47c8e:NORMAL:127.0.0.1:46383|FINALIZED]]} size 0
   [junit4]   2> 2048976 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34339 is added to blk_1073741835_1011{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-026d6351-484e-4068-ad9c-71ac8f3f2d83:NORMAL:127.0.0.1:46383|RBW], ReplicaUC[[DISK]DS-44f3eb55-044e-4c00-9468-1e1d474439a5:NORMAL:127.0.0.1:34339|RBW]]} size 0
   [junit4]   2> 2048976 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:46383 is added to blk_1073741835_1011{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-026d6351-484e-4068-ad9c-71ac8f3f2d83:NORMAL:127.0.0.1:46383|RBW], ReplicaUC[[DISK]DS-44f3eb55-044e-4c00-9468-1e1d474439a5:NORMAL:127.0.0.1:34339|RBW]]} size 0
   [junit4]   2> 2048980 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:46383 is added to blk_1073741836_1012{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-44f3eb55-044e-4c00-9468-1e1d474439a5:NORMAL:127.0.0.1:34339|RBW], ReplicaUC[[DISK]DS-f0fc8002-61af-4d14-8f31-97b125b47c8e:NORMAL:127.0.0.1:46383|RBW]]} size 0
   [junit4]   2> 2048981 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34339 is added to blk_1073741836_1012 size 177
   [junit4]   2> 2048985 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:46383 is added to blk_1073741837_1013{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-c90bd919-9d3a-4d4c-bd2e-ec3594ad6810:NORMAL:127.0.0.1:34339|RBW], ReplicaUC[[DISK]DS-026d6351-484e-4068-ad9c-71ac8f3f2d83:NORMAL:127.0.0.1:46383|FINALIZED]]} size 0
   [junit4]   2> 2048985 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34339 is added to blk_1073741837_1013{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-026d6351-484e-4068-ad9c-71ac8f3f2d83:NORMAL:127.0.0.1:46383|FINALIZED], ReplicaUC[[DISK]DS-44f3eb55-044e-4c00-9468-1e1d474439a5:NORMAL:127.0.0.1:34339|FINALIZED]]} size 0
   [junit4]   2> 2048990 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:46383 is added to blk_1073741838_1014{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-44f3eb55-044e-4c00-9468-1e1d474439a5:NORMAL:127.0.0.1:34339|RBW], ReplicaUC[[DISK]DS-f0fc8002-61af-4d14-8f31-97b125b47c8e:NORMAL:127.0.0.1:46383|RBW]]} size 0
   [junit4]   2> 2048990 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34339 is added to blk_1073741838_1014 size 148
   [junit4]   2> 2048995 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:46383 is added to blk_1073741839_1015{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-44f3eb55-044e-4c00-9468-1e1d474439a5:NORMAL:127.0.0.1:34339|RBW], ReplicaUC[[DISK]DS-026d6351-484e-4068-ad9c-71ac8f3f2d83:NORMAL:127.0.0.1:46383|RBW]]} size 0
   [junit4]   2> 2048995 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34339 is added to blk_1073741839_1015{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-44f3eb55-044e-4c00-9468-1e1d474439a5:NORMAL:127.0.0.1:34339|RBW], ReplicaUC[[DISK]DS-026d6351-484e-4068-ad9c-71ac8f3f2d83:NORMAL:127.0.0.1:46383|RBW]]} size 0
   [junit4]   2> 2049000 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34339 is added to blk_1073741840_1016{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-026d6351-484e-4068-ad9c-71ac8f3f2d83:NORMAL:127.0.0.1:46383|RBW], ReplicaUC[[DISK]DS-c90bd919-9d3a-4d4c-bd2e-ec3594ad6810:NORMAL:127.0.0.1:34339|RBW]]} size 0
   [junit4]   2> 2049000 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:46383 is added to blk_1073741840_1016{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-c90bd919-9d3a-4d4c-bd2e-ec3594ad6810:NORMAL:127.0.0.1:34339|RBW], ReplicaUC[[DISK]DS-f0fc8002-61af-4d14-8f31-97b125b47c8e:NORMAL:127.0.0.1:46383|FINALIZED]]} size 0
   [junit4]   2> 2049004 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34339 is added to blk_1073741841_1017{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-f0fc8002-61af-4d14-8f31-97b125b47c8e:NORMAL:127.0.0.1:46383|RBW], ReplicaUC[[DISK]DS-44f3eb55-044e-4c00-9468-1e1d474439a5:NORMAL:127.0.0.1:34339|RBW]]} size 0
   [junit4]   2> 2049006 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:46383 is added to blk_1073741841_1017 size 249
   [junit4]   2> 2049014 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:46383 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-c90bd919-9d3a-4d4c-bd2e-ec3594ad6810:NORMAL:127.0.0.1:34339|RBW], ReplicaUC[[DISK]DS-f0fc8002-61af-4d14-8f31-97b125b47c8e:NORMAL:127.0.0.1:46383|RBW]]} size 74
   [junit4]   2> 2049015 INFO  (Block report processor) [    ] BlockStateChange BLOCK* addStoredBlock: blockMap updated: 127.0.0.1:34339 is added to blk_1073741826_1002{UCState=UNDER_CONSTRUCTION, truncateBlock=null, primaryNodeIndex=-1, replicas=[ReplicaUC[[DISK]DS-c90bd919-9d3a-4d4c-bd2e-ec3594ad6810:NORMAL:127.0.0.1:34339|RBW], ReplicaUC[[DISK]DS-f0fc8002-61af-4d14-8f31-97b125b47c8e:NORMAL:127.0.0.1:46383|RBW]]} size 74
   [junit4]   2> 2049022 INFO  (coreCloseExecutor-6274-thread-1) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.s.h.HdfsDirectory Closing hdfs directory hdfs://localhost.localdomain:43961/data/movereplicatest_coll3/core_node2/data/index
   [junit4]   2> 2049022 INFO  (coreCloseExecutor-6274-thread-1) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.s.h.HdfsDirectory Closing hdfs directory hdfs://localhost.localdomain:43961/data/movereplicatest_coll3/core_node2/data
   [junit4]   2> 2049022 INFO  (coreCloseExecutor-6274-thread-1) [n:127.0.0.1:37977_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.s.h.HdfsDirectory Closing hdfs directory hdfs://localhost.localdomain:43961/data/movereplicatest_coll3/core_node2/data/snapshot_metadata
   [junit4]   2> 2049023 INFO  (TEST-MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted-seed#[9CC2B3855A3EE02F]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@50e35c9b{/solr,null,UNAVAILABLE}
   [junit4]   2> 2049024 INFO  (qtp620801915-16140) [n:127.0.0.1:38163_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :movereplica with params replica=core_node2&action=MOVEREPLICA&collection=movereplicatest_coll3&targetNode=127.0.0.1:38163_solr&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 2049026 INFO  (OverseerThreadFactory-6264-thread-2-processing-n:127.0.0.1:38163_solr) [n:127.0.0.1:38163_solr    ] o.a.s.c.MoveReplicaCmd Replica will be moved core_node2:{"dataDir":"hdfs://localhost.localdomain:43961/data/movereplicatest_coll3/core_node2/data/","base_url":"http://127.0.0.1:37977/solr","node_name":"127.0.0.1:37977_solr","type":"NRT","ulogDir":"hdfs://localhost.localdomain:43961/data/movereplicatest_coll3/core_node2/data/tlog","core":"movereplicatest_coll3_shard1_replica_n1","state":"down","leader":"true"}
   [junit4]   2> 2049026 INFO  (OverseerThreadFactory-6264-thread-2-processing-n:127.0.0.1:38163_solr) [n:127.0.0.1:38163_solr    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:38163_solr for creating new replica
   [junit4]   2> 2049027 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&dataDir=hdfs://localhost.localdomain:43961/data/movereplicatest_coll3/core_node2/data/&collection.configName=conf1&name=movereplicatest_coll3_shard1_replica_n1&action=CREATE&collection=movereplicatest_coll3&shard=shard1&wt=javabin&version=2&ulogDir=hdfs://localhost.localdomain:43961/data/movereplicatest_coll3/core_node2/data/&replicaType=NRT
   [junit4]   2> 2049027 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 2049129 INFO  (zkCallback-2932-thread-1-processing-n:127.0.0.1:38163_solr) [n:127.0.0.1:38163_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/movereplicatest_coll3/state.json] for collection [movereplicatest_coll3] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2049129 INFO  (zkCallback-2932-thread-2-processing-n:127.0.0.1:38163_solr) [n:127.0.0.1:38163_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/movereplicatest_coll3/state.json] for collection [movereplicatest_coll3] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2050032 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 2050035 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.s.IndexSchema [movereplicatest_coll3_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 2050037 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 2050037 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'movereplicatest_coll3_shard1_replica_n1' using configuration from collection movereplicatest_coll3, trusted=true
   [junit4]   2> 2050037 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38163.solr.core.movereplicatest_coll3.shard1.replica_n1' (registry 'solr.core.movereplicatest_coll3.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3f587dec
   [junit4]   2> 2050037 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://localhost.localdomain:43961/data
   [junit4]   2> 2050037 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
   [junit4]   2> 2050037 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 2050037 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.SolrCore [[movereplicatest_coll3_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.MoveReplicaHDFSFailoverTest_9CC2B3855A3EE02F-001/tempDir-001/node2/movereplicatest_coll3_shard1_replica_n1], dataDir=[hdfs://localhost.localdomain:43961/data/movereplicatest_coll3/core_node2/data/]
   [junit4]   2> 2050038 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost.localdomain:43961/data/movereplicatest_coll3/core_node2/data/snapshot_metadata
   [junit4]   2> 2050044 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost.localdomain:43961/data/movereplicatest_coll3/core_node2/data
   [junit4]   2> 2050093 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.HdfsUpdateLog
   [junit4]   2> 2050093 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 2050093 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.u.HdfsUpdateLog Initializing HdfsUpdateLog: tlogDfsReplication=3
   [junit4]   2> 2050101 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.u.FSHDFSUtils Recovering lease on dfs file hdfs://localhost.localdomain:43961/data/movereplicatest_coll3/core_node2/data/tlog/tlog.0000000000000000000
   [junit4]   2> 2050190 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2050190 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2050192 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://localhost.localdomain:43961/data/movereplicatest_coll3/core_node2/data/index
   [junit4]   2> 2050213 INFO  (IPC Server handler 2 on 43961) [    ] BlockStateChange BLOCK* addToInvalidates: blk_1073741825_1001 127.0.0.1:46383 127.0.0.1:34339 
   [junit4]   2> 2050254 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@62bf75c7[movereplicatest_coll3_shard1_replica_n1] main]
   [junit4]   2> 2050255 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2050255 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2050256 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 2050257 INFO  (searcherExecutor-6276-thread-1-processing-n:127.0.0.1:38163_solr x:movereplicatest_coll3_shard1_replica_n1 s:shard1 c:movereplicatest_coll3 r:core_node2) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.SolrCore [movereplicatest_coll3_shard1_replica_n1] Registered new searcher Searcher@62bf75c7[movereplicatest_coll3_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.2.0):C2)))}
   [junit4]   2> 2050267 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 2050267 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 2050267 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:38163/solr/movereplicatest_coll3_shard1_replica_n1/
   [junit4]   2> 2050267 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 2050267 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:38163/solr/movereplicatest_coll3_shard1_replica_n1/ has no replicas
   [junit4]   2> 2050267 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 2050268 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:38163/solr/movereplicatest_coll3_shard1_replica_n1/ shard1
   [junit4]   2> 2050285 WARN  (Thread-4463) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.h.h.DFSClient DataStreamer Exception
   [junit4]   2> java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[127.0.0.1:34339,DS-c90bd919-9d3a-4d4c-bd2e-ec3594ad6810,DISK], DatanodeInfoWithStorage[127.0.0.1:46383,DS-f0fc8002-61af-4d14-8f31-97b125b47c8e,DISK]], original=[DatanodeInfoWithStorage[127.0.0.1:34339,DS-c90bd919-9d3a-4d4c-bd2e-ec3594ad6810,DISK], DatanodeInfoWithStorage[127.0.0.1:46383,DS-f0fc8002-61af-4d14-8f31-97b125b47c8e,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration.
   [junit4]   2> 	at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:1044)
   [junit4]   2> 	at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1107)
   [junit4]   2> 	at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1276)
   [junit4]   2> 	at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:560)
   [junit4]   2> 2050285 ERROR (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.u.HdfsTransactionLog Could not close tlog output
   [junit4]   2> java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[127.0.0.1:34339,DS-c90bd919-9d3a-4d4c-bd2e-ec3594ad6810,DISK], DatanodeInfoWithStorage[127.0.0.1:46383,DS-f0fc8002-61af-4d14-8f31-97b125b47c8e,DISK]], original=[DatanodeInfoWithStorage[127.0.0.1:34339,DS-c90bd919-9d3a-4d4c-bd2e-ec3594ad6810,DISK], DatanodeInfoWithStorage[127.0.0.1:46383,DS-f0fc8002-61af-4d14-8f31-97b125b47c8e,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration.
   [junit4]   2> 	at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:1044)
   [junit4]   2> 	at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1107)
   [junit4]   2> 	at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1276)
   [junit4]   2> 	at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:560)
   [junit4]   2> 2050285 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 2050286 INFO  (qtp620801915-16132) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&dataDir=hdfs://localhost.localdomain:43961/data/movereplicatest_coll3/core_node2/data/&collection.configName=conf1&name=movereplicatest_coll3_shard1_replica_n1&action=CREATE&collection=movereplicatest_coll3&shard=shard1&wt=javabin&version=2&ulogDir=hdfs://localhost.localdomain:43961/data/movereplicatest_coll3/core_node2/data/&replicaType=NRT} status=0 QTime=1259
   [junit4]   2> 2050288 INFO  (qtp620801915-16140) [n:127.0.0.1:38163_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replica=core_node2&action=MOVEREPLICA&collection=movereplicatest_coll3&targetNode=127.0.0.1:38163_solr&wt=javabin&version=2} status=0 QTime=1263
   [junit4]   2> 2050387 INFO  (zkCallback-2932-thread-2-processing-n:127.0.0.1:38163_solr) [n:127.0.0.1:38163_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/movereplicatest_coll3/state.json] for collection [movereplicatest_coll3] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2050387 INFO  (zkCallback-2932-thread-1-processing-n:127.0.0.1:38163_solr) [n:127.0.0.1:38163_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/movereplicatest_coll3/state.json] for collection [movereplicatest_coll3] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 2051026 INFO  (OverseerCollectionConfigSetProcessor-98922431889735685-127.0.0.1:38163_solr-n_0000000000) [n:127.0.0.1:38163_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 2051289 INFO  (TEST-MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted-seed#[9CC2B3855A3EE02F]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@7c08c455{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2051290 INFO  (TEST-MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted-seed#[9CC2B3855A3EE02F]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1515892540
   [junit4]   2> 2051290 INFO  (TEST-MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted-seed#[9CC2B3855A3EE02F]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 2051290 INFO  (TEST-MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted-seed#[9CC2B3855A3EE02F]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@694e2f42: rootName = solr_38163, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@3b160c73
   [junit4]   2> 2051294 INFO  (TEST-MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted-seed#[9CC2B3855A3EE02F]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 2051294 INFO  (TEST-MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted-seed#[9CC2B3855A3EE02F]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6310e890: rootName = solr_38163, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@56070cd8
   [junit4]   2> 2051297 INFO  (TEST-MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted-seed#[9CC2B3855A3EE02F]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 2051297 INFO  (TEST-MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted-seed#[9CC2B3855A3EE02F]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@61a3624e: rootName = solr_38163, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@635df43b
   [junit4]   2> 2051298 INFO  (TEST-MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted-seed#[9CC2B3855A3EE02F]) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:38163_solr
   [junit4]   2> 2051298 INFO  (TEST-MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted-seed#[9CC2B3855A3EE02F]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 2051298 INFO  (zkCallback-2940-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 2051298 INFO  (zkCallback-2932-thread-1-processing-n:127.0.0.1:38163_solr) [n:127.0.0.1:38163_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 2051298 INFO  (coreCloseExecutor-6281-thread-1) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.c.SolrCore [movereplicatest_coll3_shard1_replica_n1]  CLOSING SolrCore org.apache.solr.core.SolrCore@6091bfea
   [junit4]   2> 2051298 INFO  (coreCloseExecutor-6281-thread-1) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.movereplicatest_coll3.shard1.replica_n1, tag=1620164586
   [junit4]   2> 2051298 INFO  (coreCloseExecutor-6281-thread-1) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4949fb78: rootName = solr_38163, domain = solr.core.movereplicatest_coll3.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.movereplicatest_coll3.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@26b5b683
   [junit4]   2> 2051303 INFO  (coreCloseExecutor-6281-thread-1) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.movereplicatest_coll3.shard1.leader, tag=1620164586
   [junit4]   2> 2051305 ERROR (coreCloseExecutor-6281-thread-1) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.h.h.DFSClient Failed to close inode 16402
   [junit4]   2> java.io.IOException: Failed to replace a bad datanode on the existing pipeline due to no more good datanodes being available to try. (Nodes: current=[DatanodeInfoWithStorage[127.0.0.1:34339,DS-c90bd919-9d3a-4d4c-bd2e-ec3594ad6810,DISK], DatanodeInfoWithStorage[127.0.0.1:46383,DS-f0fc8002-61af-4d14-8f31-97b125b47c8e,DISK]], original=[DatanodeInfoWithStorage[127.0.0.1:34339,DS-c90bd919-9d3a-4d4c-bd2e-ec3594ad6810,DISK], DatanodeInfoWithStorage[127.0.0.1:46383,DS-f0fc8002-61af-4d14-8f31-97b125b47c8e,DISK]]). The current failed datanode replacement policy is DEFAULT, and a client may configure this via 'dfs.client.block.write.replace-datanode-on-failure.policy' in its configuration.
   [junit4]   2> 	at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.findNewDatanode(DFSOutputStream.java:1044)
   [junit4]   2> 	at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.addDatanode2ExistingPipeline(DFSOutputStream.java:1107)
   [junit4]   2> 	at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.setupPipelineForAppendOrRecovery(DFSOutputStream.java:1276)
   [junit4]   2> 	at org.apache.hadoop.hdfs.DFSOutputStream$DataStreamer.run(DFSOutputStream.java:560)
   [junit4]   2> 2051309 INFO  (coreCloseExecutor-6281-thread-1) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.s.h.HdfsDirectory Closing hdfs directory hdfs://localhost.localdomain:43961/data/movereplicatest_coll3/core_node2/data/index
   [junit4]   2> 2051309 INFO  (coreCloseExecutor-6281-thread-1) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.s.h.HdfsDirectory Closing hdfs directory hdfs://localhost.localdomain:43961/data/movereplicatest_coll3/core_node2/data
   [junit4]   2> 2051309 INFO  (coreCloseExecutor-6281-thread-1) [n:127.0.0.1:38163_solr c:movereplicatest_coll3 s:shard1 r:core_node2 x:movereplicatest_coll3_shard1_replica_n1] o.a.s.s.h.HdfsDirectory Closing hdfs directory hdfs://localhost.localdomain:43961/data/movereplicatest_coll3/core_node2/data/snapshot_metadata
   [junit4]   2> 2051310 INFO  (TEST-MoveReplicaHDFSFailoverTest.testOldReplicaIsDeleted-seed#[9CC2B3855A3EE02F]) [    ] o.a.s.c.Overseer Overseer (id=98922431889735685-127.0.0.1:38163_solr-n_0000000000) closing
   [junit4]   2> 2051310 INFO  (OverseerStateUpdate-98922431889735685-127.0.0.1:3816

[...truncated too long message...]

unit4]   2> 	at org.apache.solr.cloud.autoscaling.ExecutePlanAction.waitForTaskToFinish(ExecutePlanAction.java:130)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers.waitForPendingTasks(ScheduledTriggers.java:289)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$2(ScheduledTriggers.java:221)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Caused by: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:460)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1096)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:875)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:808)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:178)
   [junit4]   2> 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:195)
   [junit4]   2> 	at org.apache.solr.client.solrj.impl.SolrClientCloudManager.request(SolrClientCloudManager.java:99)
   [junit4]   2> 	... 9 more
   [junit4]   2> 3455428 WARN  (OverseerThreadFactory-6454-thread-1-processing-n:127.0.0.1:32879_solr) [n:127.0.0.1:32879_solr    ] o.a.s.c.MoveReplicaCmd Timed out waiting for leader's replica to recover, collection=testExecute shard=shard1 on node=127.0.0.1:32879_solr
   [junit4]   2> 3455430 INFO  (OverseerThreadFactory-6454-thread-1-processing-n:127.0.0.1:32879_solr) [n:127.0.0.1:32879_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 3455431 WARN  (OverseerAutoScalingTriggerThread-98922483197411334-127.0.0.1:32879_solr-n_0000000001) [n:127.0.0.1:32879_solr    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 3455432 DEBUG (jetty-closer-2325-thread-1) [    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
   [junit4]   2> 3455432 ERROR (AutoscalingActionExecutor-6453-thread-1-processing-n:127.0.0.1:32879_solr) [n:127.0.0.1:32879_solr    ] o.a.s.c.a.ScheduledTriggers Unexpected exception while waiting for pending task with requestid: node_lost_trigger/14f2993f32cfb540T4mywjgydhjysvs86kpl77f2s1/0 to finish
   [junit4]   2> java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 	at java.lang.Thread.sleep(Native Method)
   [junit4]   2> 	at java.lang.Thread.sleep(Thread.java:340)
   [junit4]   2> 	at java.util.concurrent.TimeUnit.sleep(TimeUnit.java:386)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ExecutePlanAction.waitForTaskToFinish(ExecutePlanAction.java:155)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers.waitForPendingTasks(ScheduledTriggers.java:289)
   [junit4]   2> 	at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$2(ScheduledTriggers.java:221)
   [junit4]   2> 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
   [junit4]   2> 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]   2> 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 3455432 INFO  (jetty-closer-2325-thread-1) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@785bd1e7{/solr,null,UNAVAILABLE}
   [junit4]   2> 3455433 ERROR (SUITE-ExecutePlanActionTest-seed#[9CC2B3855A3EE02F]-worker) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 3455433 INFO  (SUITE-ExecutePlanActionTest-seed#[9CC2B3855A3EE02F]-worker) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:46629 46629
   [junit4]   2> 3460469 INFO  (Thread-3661) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:46629 46629
   [junit4]   2> 3460469 WARN  (Thread-3661) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	4	/solr/aliases.json
   [junit4]   2> 	4	/solr/clusterprops.json
   [junit4]   2> 	3	/solr/configs/conf
   [junit4]   2> 	2	/solr/security.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	19	/solr/collections/testIntegration/state.json
   [junit4]   2> 	18	/solr/collections/testExecute/state.json
   [junit4]   2> 	4	/solr/clusterstate.json
   [junit4]   2> 	2	/solr/autoscaling.json
   [junit4]   2> 	2	/solr/collections/testExecute/leader_elect/shard1/election/98922483197411332-core_node4-n_0000000000
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	4	/solr/live_nodes
   [junit4]   2> 	4	/solr/collections
   [junit4]   2> 	2	/solr/overseer/queue
   [junit4]   2> 	2	/solr/autoscaling/events/.auto_add_replicas
   [junit4]   2> 	2	/solr/autoscaling/events/node_lost_trigger
   [junit4]   2> 	2	/solr/overseer/collection-queue-work
   [junit4]   2> 	2	/solr/overseer/queue-work
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-7.x-Linux/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.ExecutePlanActionTest_9CC2B3855A3EE02F-001
   [junit4]   2> NOTE: test params are: codec=FastCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST, chunkSize=5, maxDocsPerChunk=553, blockSize=645), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST, chunkSize=5, blockSize=645)), sim=RandomSimilarity(queryNorm=false): {}, locale=zh-SG, timezone=Asia/Aden
   [junit4]   2> NOTE: Linux 4.10.0-37-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=8,threads=1,free=262885912,total=522715136
   [junit4]   2> NOTE: All tests run in this JVM: [TestWordDelimiterFilterFactory, SimpleFacetsTest, TestIBSimilarityFactory, BlockJoinFacetDistribTest, SearchHandlerTest, CurrencyFieldTypeTest, TestRandomFaceting, TestReplicationHandler, TestConfigSetsAPIExclusivity, TestSmileRequest, TestAnalyzeInfixSuggestions, CollectionsAPISolrJTest, TestReversedWildcardFilterFactory, SolrCoreMetricManagerTest, UpdateParamsTest, SolrPluginUtilsTest, TestPostingsSolrHighlighter, DocumentBuilderTest, TestExactSharedStatsCache, BasicZkTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestSolrConfigHandlerCloud, TestConfigSetImmutable, TestNumericTerms32, TestAuthenticationFramework, SuggestComponentTest, ActionThrottleTest, StressHdfsTest, TestNoOpRegenerator, TestSSLRandomization, TestExportWriter, TestCSVLoader, TestReplicaProperties, ComputePlanActionTest, DistribCursorPagingTest, DistributedMLTComponentTest, FacetPivotSmallTest, ExternalFileFieldSortTest, SortSpecParsingTest, SyncSliceTest, TestSubQueryTransformerDistrib, SolrCoreTest, JsonLoaderTest, TestSlowCompositeReaderWrapper, TestSimpleTextCodec, TestCloudRecovery, TestDistributedSearch, TestSolrCloudWithKerberosAlt, TestSolrFieldCacheBean, TriggerIntegrationTest, AtomicUpdatesTest, TestQueryUtils, SliceStateTest, TestOrdValues, CleanupOldIndexTest, TestPrepRecovery, QueryResultKeyTest, CacheHeaderTest, AssignBackwardCompatibilityTest, TestFreeTextSuggestions, TestRestoreCore, HdfsRestartWhileUpdatingTest, PeerSyncReplicationTest, BasicDistributedZkTest, RollingRestartTest, DeleteReplicaTest, TestApiFramework, TestCloudManagedSchema, TestPhraseSuggestions, TestMergePolicyConfig, LargeFieldTest, MultiTermTest, CloudMLTQParserTest, DirectoryFactoryTest, TestSchemaVersionResource, IndexSchemaRuntimeFieldTest, DeleteInactiveReplicaTest, LeaderFailoverAfterPartitionTest, TestImplicitCoreProperties, TermsComponentTest, DistributedFacetExistsSmallTest, PeerSyncTest, FastVectorHighlighterTest, SpellCheckCollatorTest, CursorPagingTest, HdfsNNFailoverTest, PrimitiveFieldTypeTest, SearchHandlerTest, TestSuggestSpellingConverter, AnalysisAfterCoreReloadTest, TestRTimerTree, TestFieldCacheReopen, TestSolrDeletionPolicy1, LeaderElectionIntegrationTest, TestLegacyNumericUtils, TestIndexingPerformance, TestSizeLimitedDistributedMap, BasicAuthIntegrationTest, TestFieldSortValues, SmileWriterTest, TestCodecSupport, ZkStateWriterTest, TestAuthorizationFramework, ResourceLoaderTest, HdfsLockFactoryTest, StatelessScriptUpdateProcessorFactoryTest, TestReloadDeadlock, CdcrReplicationHandlerTest, TestFastOutputStream, DirectUpdateHandlerOptimizeTest, TestJoin, BaseCdcrDistributedZkTest, SecurityConfHandlerTest, TolerantUpdateProcessorTest, TestHashQParserPlugin, TestEmbeddedSolrServerSchemaAPI, TestDynamicFieldCollectionResource, TestLocalFSCloudBackupRestore, TestRandomFlRTGCloud, TestSegmentSorting, TestSolrCloudWithDelegationTokens, TestStressInPlaceUpdates, AutoAddReplicasPlanActionTest, ExecutePlanActionTest]
   [junit4] Completed [744/745 (2!)] on J0 in 634.47s, 2 tests, 1 failure <<< FAILURES!

[...truncated 47249 lines...]

Mime
View raw message