lucene-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Apache Jenkins Server <jenk...@builds.apache.org>
Subject [JENKINS] Lucene-Solr-Tests-4.x-Java6 - Build # 2323 - Failure
Date Sat, 01 Mar 2014 07:37:12 GMT
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java6/2323/

1 tests failed.
REGRESSION:  org.apache.solr.cloud.OverseerTest.testOverseerFailure

Error Message:
KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1

Stack Trace:
org.apache.zookeeper.KeeperException$NodeExistsException: KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1
	at __randomizedtesting.SeedInfo.seed([A7DF8B62F1F8EA9C:A3D70491E35D05BD]:0)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:119)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
	at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
	at org.apache.solr.common.cloud.SolrZkClient$10.execute(SolrZkClient.java:429)
	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:426)
	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:383)
	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:370)
	at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:112)
	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:164)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:108)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:156)
	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:289)
	at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:154)
	at org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:590)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:616)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:679)




Build Log:
[...truncated 10130 lines...]
   [junit4] Suite: org.apache.solr.cloud.OverseerTest
   [junit4]   2> 1071059 T5300 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (true)
   [junit4]   2> 1071060 T5300 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build/solr-core/test/J0/./solrtest-OverseerTest-1393656765135
   [junit4]   2> 1071061 T5300 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 1071065 T5300 oas.SolrTestCaseJ4.setUp ###Starting testPlaceholders
   [junit4]   2> 1071065 T5300 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 1071066 T5301 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1071167 T5300 oasc.ZkTestServer.run start zk server on port:11807
   [junit4]   2> 1071168 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1071172 T5307 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c356b2 name:ZooKeeperConnection Watcher:127.0.0.1:11807/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1071172 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1071173 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1071175 T5309 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7592283 name:ZooKeeperConnection Watcher:127.0.0.1:11807 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1071176 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1071193 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1071197 T5311 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@55f4411a name:ZooKeeperConnection Watcher:127.0.0.1:11807 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1071197 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1071198 T5300 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1071251 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1071280 T5300 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1071283 T5300 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1071312 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1071315 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1071317 T5313 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6803bf6 name:ZooKeeperConnection Watcher:127.0.0.1:11807/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1071317 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1071319 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1071320 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
   [junit4]   2> 1071377 T5307 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 1071377 T5313 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 1071378 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1071407 T5315 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@68a8018a name:ZooKeeperConnection Watcher:127.0.0.1:11807/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1071408 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1071409 T5300 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1071414 T5300 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1071495 T5300 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:11807_solr
   [junit4]   2> 1071496 T5300 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1071540 T5300 oasc.Overseer.start Overseer (id=91334689760739332-127.0.0.1:11807_solr-n_0000000000) starting
   [junit4]   2> 1071594 T5300 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1071619 T5300 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1071623 T5300 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1071662 T5317 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1071663 T5317 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 1071663 T5316 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1071665 T5316 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1071666 T5316 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=12 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"recovering",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"12",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 1071666 T5316 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2, shard3, shard4, shard5, shard6, shard7, shard8, shard9, shard10, shard11, shard12]
   [junit4]   2> 1071666 T5316 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1071668 T5315 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1071669 T5307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1071669 T5313 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1072165 T5300 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1072194 T5300 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1072207 T5315 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1072207 T5300 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1072208 T5316 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1072212 T5307 oascc.ZkStateReader$3.process Updating live nodes... (0)
   [junit4]   2> 1072212 T5313 oascc.ZkStateReader$3.process Updating live nodes... (0)
   [junit4]   2> 1072224 T5300 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:11807 11807
   [junit4]   2> 1072354 T5300 oas.SolrTestCaseJ4.tearDown ###Ending testPlaceholders
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1072360 T5300 oas.SolrTestCaseJ4.setUp ###Starting testShardAssignmentBigger
   [junit4]   2> 1072361 T5300 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 1072362 T5318 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1072462 T5300 oasc.ZkTestServer.run start zk server on port:11813
   [junit4]   2> 1072463 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072466 T5324 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@459f5b73 name:ZooKeeperConnection Watcher:127.0.0.1:11813 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072466 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072499 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072501 T5326 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b8a7ac3 name:ZooKeeperConnection Watcher:127.0.0.1:11813 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072501 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072501 T5300 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1072505 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072506 T5328 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7d6e8b68 name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072506 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072507 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1072510 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072512 T5330 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@104b0dbd name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072512 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072513 T5300 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1072521 T5300 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1072525 T5300 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:11813_solr
   [junit4]   2> 1072525 T5300 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1072535 T5300 oasc.Overseer.start Overseer (id=91334689845542915-127.0.0.1:11813_solr-n_0000000000) starting
   [junit4]   2> 1072538 T5300 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1072549 T5300 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1072557 T5300 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1072560 T5332 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1072560 T5300 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1072560 T5331 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1072560 T5332 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 1072562 T5300 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1072572 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072574 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072576 T5334 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@24bed2d9 name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072576 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072578 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072579 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node0
   [junit4]   2> 1072582 T5328 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 1072582 T5334 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 1072582 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072584 T5336 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@643f0932 name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072584 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072585 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072587 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
   [junit4]   2> 1072589 T5328 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1072589 T5336 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1072589 T5334 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1072590 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072592 T5338 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3782dd61 name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072592 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072593 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072595 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node2
   [junit4]   2> 1072597 T5328 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1072597 T5338 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1072597 T5334 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1072597 T5336 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1072598 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072600 T5340 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3b6ae902 name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072600 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072601 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072603 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node3
   [junit4]   2> 1072606 T5328 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1072606 T5336 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1072606 T5338 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1072606 T5340 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1072606 T5334 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1072607 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072608 T5342 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1f1a447a name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072609 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072610 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072612 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node4
   [junit4]   2> 1072614 T5328 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1072614 T5338 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1072614 T5342 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1072614 T5336 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1072614 T5334 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1072614 T5340 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1072615 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072617 T5344 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@74bc7213 name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072617 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072618 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072620 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node5
   [junit4]   2> 1072623 T5328 oascc.ZkStateReader$3.process Updating live nodes... (6)
   [junit4]   2> 1072623 T5336 oascc.ZkStateReader$3.process Updating live nodes... (6)
   [junit4]   2> 1072623 T5334 oascc.ZkStateReader$3.process Updating live nodes... (6)
   [junit4]   2> 1072623 T5344 oascc.ZkStateReader$3.process Updating live nodes... (6)
   [junit4]   2> 1072623 T5338 oascc.ZkStateReader$3.process Updating live nodes... (6)
   [junit4]   2> 1072623 T5342 oascc.ZkStateReader$3.process Updating live nodes... (6)
   [junit4]   2> 1072623 T5340 oascc.ZkStateReader$3.process Updating live nodes... (6)
   [junit4]   2> 1072624 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072625 T5346 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@50c5b7a0 name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072625 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072627 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072628 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node6
   [junit4]   2> 1072631 T5336 oascc.ZkStateReader$3.process Updating live nodes... (7)
   [junit4]   2> 1072632 T5328 oascc.ZkStateReader$3.process Updating live nodes... (7)
   [junit4]   2> 1072632 T5334 oascc.ZkStateReader$3.process Updating live nodes... (7)
   [junit4]   2> 1072632 T5346 oascc.ZkStateReader$3.process Updating live nodes... (7)
   [junit4]   2> 1072632 T5340 oascc.ZkStateReader$3.process Updating live nodes... (7)
   [junit4]   2> 1072632 T5342 oascc.ZkStateReader$3.process Updating live nodes... (7)
   [junit4]   2> 1072632 T5338 oascc.ZkStateReader$3.process Updating live nodes... (7)
   [junit4]   2> 1072631 T5344 oascc.ZkStateReader$3.process Updating live nodes... (7)
   [junit4]   2> 1072633 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072634 T5348 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@64322b4d name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072634 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072636 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072637 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node7
   [junit4]   2> 1072640 T5340 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 1072640 T5344 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 1072641 T5338 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 1072641 T5334 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 1072641 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072641 T5346 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 1072641 T5328 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 1072641 T5348 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 1072640 T5342 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 1072640 T5336 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 1072643 T5350 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5341d06c name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072643 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072644 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072646 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node8
   [junit4]   2> 1072649 T5344 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 1072650 T5348 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 1072650 T5350 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 1072650 T5340 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 1072650 T5338 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 1072650 T5328 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 1072650 T5346 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 1072649 T5336 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 1072649 T5342 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 1072649 T5334 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 1072651 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072652 T5352 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@54ca886 name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072653 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072654 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072655 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node9
   [junit4]   2> 1072658 T5348 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 1072659 T5340 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 1072659 T5350 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 1072659 T5346 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 1072659 T5344 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 1072659 T5328 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 1072659 T5334 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 1072659 T5336 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 1072659 T5342 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 1072659 T5338 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 1072660 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072659 T5352 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 1072662 T5354 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@75ca5d55 name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072662 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072663 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072665 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node10
   [junit4]   2> 1072669 T5344 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 1072669 T5348 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 1072669 T5328 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 1072670 T5342 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 1072669 T5346 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 1072669 T5340 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 1072669 T5334 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 1072669 T5352 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 1072669 T5354 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 1072669 T5338 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 1072669 T5350 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 1072669 T5336 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 1072670 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072672 T5356 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@66cf4a66 name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072672 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072674 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072675 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node11
   [junit4]   2> 1072678 T5346 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 1072679 T5336 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 1072679 T5356 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 1072679 T5354 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 1072679 T5334 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 1072679 T5340 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 1072679 T5344 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 1072679 T5352 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 1072679 T5348 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 1072679 T5342 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 1072679 T5338 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 1072679 T5350 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 1072679 T5328 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 1072682 T5358 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@53aebcfa name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072680 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072683 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072684 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072686 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node12
   [junit4]   2> 1072688 T5348 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 1072689 T5334 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 1072689 T5346 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 1072689 T5340 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 1072690 T5342 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 1072690 T5352 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 1072689 T5344 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 1072690 T5328 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 1072690 T5354 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 1072690 T5358 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 1072690 T5356 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 1072690 T5336 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 1072690 T5350 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 1072690 T5338 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 1072691 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072693 T5360 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@74ef0025 name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072693 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072694 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072696 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node13
   [junit4]   2> 1072699 T5336 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 1072699 T5338 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 1072699 T5348 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 1072700 T5342 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 1072700 T5352 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 1072700 T5360 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 1072700 T5334 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 1072700 T5340 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 1072700 T5344 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 1072700 T5328 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 1072700 T5346 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 1072700 T5356 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 1072700 T5358 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 1072700 T5350 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 1072700 T5354 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 1072703 T5362 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1ac9378c name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072701 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072704 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072705 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072707 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node14
   [junit4]   2> 1072710 T5344 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 1072711 T5328 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 1072711 T5362 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 1072711 T5350 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 1072711 T5342 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 1072711 T5358 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 1072711 T5348 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 1072711 T5356 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 1072711 T5338 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 1072711 T5360 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 1072711 T5346 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 1072711 T5354 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 1072713 T5364 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3d2934d0 name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072710 T5352 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 1072710 T5334 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 1072710 T5336 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 1072710 T5340 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 1072712 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072715 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072717 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072719 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node15
   [junit4]   2> 1072722 T5336 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 1072722 T5342 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 1072723 T5364 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 1072723 T5354 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 1072723 T5338 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 1072723 T5334 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 1072722 T5350 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 1072722 T5348 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 1072722 T5358 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 1072722 T5362 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 1072724 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072723 T5346 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 1072723 T5356 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 1072723 T5360 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 1072726 T5366 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6de30150 name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072723 T5340 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 1072723 T5352 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 1072723 T5344 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 1072723 T5328 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 1072726 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072728 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072730 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node16
   [junit4]   2> 1072734 T5362 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 1072734 T5338 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 1072735 T5354 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 1072734 T5352 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 1072734 T5356 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 1072734 T5358 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 1072734 T5336 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 1072734 T5344 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 1072734 T5328 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 1072736 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072735 T5342 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 1072735 T5340 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 1072735 T5364 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 1072737 T5368 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5c4d2ad0 name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072735 T5366 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 1072735 T5350 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 1072735 T5348 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 1072735 T5334 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 1072734 T5346 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 1072734 T5360 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 1072738 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072740 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072742 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node17
   [junit4]   2> 1072746 T5344 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 1072746 T5348 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 1072747 T5368 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 1072747 T5356 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 1072747 T5364 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 1072748 T5354 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 1072747 T5328 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 1072747 T5336 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 1072747 T5334 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 1072747 T5350 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 1072746 T5366 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 1072746 T5360 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 1072746 T5352 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 1072748 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072747 T5340 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 1072747 T5346 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 1072747 T5358 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 1072747 T5342 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 1072747 T5362 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 1072747 T5338 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 1072750 T5370 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5334a5d5 name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072751 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072752 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072754 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node18
   [junit4]   2> 1072758 T5328 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 1072758 T5334 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 1072759 T5360 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 1072759 T5348 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 1072759 T5342 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 1072759 T5338 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 1072759 T5358 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 1072759 T5350 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 1072760 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072759 T5368 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 1072759 T5366 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 1072758 T5352 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 1072758 T5340 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 1072758 T5364 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 1072760 T5356 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 1072760 T5344 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 1072760 T5370 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 1072760 T5346 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 1072759 T5354 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 1072759 T5362 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 1072759 T5336 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 1072762 T5372 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1a218e6f name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072764 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072765 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072767 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node19
   [junit4]   2> 1072770 T5344 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 1072771 T5356 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 1072771 T5370 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 1072772 T5360 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 1072772 T5354 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 1072771 T5342 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 1072771 T5338 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 1072771 T5352 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 1072771 T5346 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 1072771 T5350 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 1072771 T5368 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 1072771 T5334 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 1072771 T5336 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 1072771 T5348 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 1072781 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072772 T5328 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 1072772 T5362 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 1072772 T5366 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 1072772 T5364 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 1072772 T5372 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 1072772 T5340 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 1072772 T5358 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 1072783 T5374 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1ff18885 name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072785 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072786 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072788 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node20
   [junit4]   2> 1072792 T5372 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 1072792 T5362 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 1072793 T5358 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 1072794 T5356 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 1072794 T5328 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 1072794 T5350 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 1072792 T5344 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 1072795 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072794 T5368 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 1072794 T5374 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 1072794 T5370 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 1072794 T5340 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 1072794 T5346 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 1072794 T5334 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 1072793 T5352 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 1072793 T5364 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 1072793 T5360 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 1072793 T5338 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 1072793 T5354 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 1072793 T5342 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 1072793 T5366 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 1072793 T5348 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 1072793 T5336 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 1072797 T5376 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2f9977ee name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072799 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072800 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072801 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node21
   [junit4]   2> 1072805 T5328 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 1072806 T5344 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 1072806 T5340 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 1072807 T5374 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 1072807 T5354 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 1072806 T5370 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 1072805 T5364 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 1072807 T5334 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 1072807 T5362 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 1072807 T5356 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 1072807 T5346 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 1072807 T5376 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 1072807 T5352 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 1072807 T5366 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 1072807 T5358 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 1072806 T5342 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 1072810 T5378 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1527da7 name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072806 T5348 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 1072806 T5350 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 1072806 T5360 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 1072806 T5372 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 1072806 T5338 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 1072806 T5336 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 1072806 T5368 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 1072808 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072812 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072813 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072815 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node22
   [junit4]   2> 1072818 T5352 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 1072841 T5358 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 1072841 T5328 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 1072842 T5368 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 1072842 T5350 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 1072842 T5340 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 1072842 T5362 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 1072842 T5376 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 1072842 T5354 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 1072844 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072842 T5342 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 1072842 T5360 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 1072842 T5336 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 1072842 T5344 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 1072841 T5366 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 1072841 T5346 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 1072841 T5338 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 1072841 T5334 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 1072845 T5380 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@38c85a2a name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072843 T5364 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 1072843 T5378 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 1072843 T5356 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 1072843 T5348 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 1072842 T5374 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 1072842 T5370 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 1072842 T5372 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 1072846 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072849 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072851 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node23
   [junit4]   2> 1072854 T5376 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072855 T5346 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072856 T5356 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072856 T5334 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072857 T5368 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072857 T5328 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072857 T5342 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072856 T5372 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072856 T5360 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072856 T5378 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072856 T5358 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072856 T5340 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072856 T5336 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072856 T5352 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072856 T5362 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072856 T5366 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072856 T5370 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072855 T5354 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072860 T5382 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@65fa8e18 name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072858 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072857 T5348 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072857 T5374 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072857 T5344 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072857 T5350 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072857 T5338 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072857 T5364 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072857 T5380 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 1072861 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072863 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072865 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node24
   [junit4]   2> 1072869 T5346 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072870 T5368 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072871 T5334 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072871 T5336 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072871 T5374 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072871 T5340 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072871 T5358 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072871 T5376 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072871 T5360 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072870 T5350 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072870 T5366 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072870 T5372 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072870 T5382 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072870 T5362 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072870 T5380 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072870 T5352 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072870 T5370 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072870 T5354 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072870 T5356 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072869 T5348 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072874 T5384 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@31d3d886 name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072872 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072871 T5344 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072871 T5338 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072871 T5364 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072871 T5328 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072871 T5342 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072871 T5378 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 1072876 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072878 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072880 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node25
   [junit4]   2> 1072884 T5364 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072884 T5372 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072884 T5370 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072884 T5368 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072885 T5340 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072885 T5350 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072885 T5342 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072885 T5382 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072885 T5378 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072885 T5380 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072885 T5374 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072887 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072887 T5358 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072886 T5352 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072886 T5336 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072886 T5344 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072886 T5328 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072886 T5348 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072886 T5366 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072886 T5334 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072886 T5356 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072886 T5362 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072886 T5360 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072886 T5338 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072886 T5346 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072886 T5384 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072886 T5354 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072886 T5376 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 1072889 T5386 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@fe9e310 name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072892 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072893 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072895 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node26
   [junit4]   2> 1072899 T5328 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072900 T5368 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072901 T5372 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072901 T5348 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072902 T5340 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072901 T5334 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072900 T5338 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072900 T5344 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072900 T5342 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072900 T5364 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072903 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072902 T5358 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072902 T5356 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072902 T5350 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072904 T5388 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5fb7b66c name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072902 T5380 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072902 T5352 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072902 T5362 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072902 T5382 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072902 T5370 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072902 T5378 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072902 T5386 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072902 T5336 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072902 T5376 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072901 T5384 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072901 T5360 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072901 T5354 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072901 T5366 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072901 T5346 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072901 T5374 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 1072905 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072909 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072911 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node27
   [junit4]   2> 1072914 T5362 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072915 T5366 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072916 T5386 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072916 T5384 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072916 T5372 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072917 T5328 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072916 T5382 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072917 T5370 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072916 T5352 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072916 T5348 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072916 T5378 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072916 T5334 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072915 T5346 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072915 T5338 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072915 T5368 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072915 T5358 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072915 T5380 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072918 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072917 T5388 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072917 T5356 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072917 T5360 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072917 T5344 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072917 T5350 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072917 T5336 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072917 T5342 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072917 T5354 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072916 T5374 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072916 T5376 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072916 T5340 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072916 T5364 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 1072920 T5390 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6761a9a4 name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072923 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072924 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072925 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node28
   [junit4]   2> 1072929 T5352 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072929 T5340 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072931 T5348 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072931 T5328 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072930 T5356 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072932 T5358 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072932 T5344 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072930 T5388 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072930 T5346 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072930 T5364 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072930 T5372 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072933 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072932 T5334 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072932 T5354 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072932 T5374 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072932 T5370 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072932 T5378 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072932 T5368 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072932 T5390 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072932 T5336 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072932 T5342 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072932 T5338 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072931 T5350 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072931 T5366 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072931 T5376 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072931 T5380 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072931 T5362 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072931 T5360 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072931 T5386 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072931 T5382 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072931 T5384 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 1072935 T5392 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7e73bc58 name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072938 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072939 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072941 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node29
   [junit4]   2> 1072945 T5382 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072945 T5374 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072946 T5350 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072947 T5364 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072947 T5362 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072948 T5370 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072948 T5356 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072947 T5344 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072946 T5338 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072946 T5390 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072946 T5380 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072946 T5388 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072946 T5328 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072946 T5392 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072946 T5360 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072945 T5348 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072949 T5394 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@49ab4fe name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072948 T5354 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072948 T5386 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072948 T5372 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072948 T5384 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072948 T5366 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072948 T5352 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072948 T5342 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072948 T5334 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072948 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072947 T5368 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072947 T5340 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072947 T5376 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072947 T5358 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072947 T5336 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072947 T5378 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072947 T5346 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 1072953 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072955 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072957 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node30
   [junit4]   2> 1072960 T5328 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072961 T5382 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072962 T5386 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072962 T5376 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072963 T5368 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072963 T5378 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072963 T5392 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072963 T5348 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072963 T5334 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072963 T5380 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072963 T5338 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072963 T5370 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072963 T5350 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072963 T5360 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072962 T5384 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072962 T5362 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072962 T5352 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072962 T5358 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072962 T5388 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072961 T5354 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072966 T5396 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@44ea502d name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072965 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072964 T5372 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072964 T5336 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072964 T5394 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072964 T5390 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072964 T5356 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072964 T5366 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072964 T5340 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072964 T5346 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072964 T5342 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072963 T5344 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072963 T5364 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072963 T5374 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 1072968 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072972 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1072973 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node31
   [junit4]   2> 1072977 T5366 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072978 T5384 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072980 T5358 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072978 T5348 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072977 T5362 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072981 T5364 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072980 T5396 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072980 T5372 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072980 T5386 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072980 T5338 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072980 T5340 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072980 T5370 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072980 T5390 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072980 T5376 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072980 T5346 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072980 T5336 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072980 T5374 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072980 T5354 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072980 T5342 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072980 T5392 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072980 T5382 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072980 T5380 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072980 T5388 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072980 T5328 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072979 T5344 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072979 T5356 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072979 T5360 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072979 T5350 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072979 T5378 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072979 T5352 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072979 T5334 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072978 T5394 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072978 T5368 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 1072983 T5398 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@26062fd7 name:ZooKeeperConnection Watcher:127.0.0.1:11813/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1072981 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1072988 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1072989 T5300 oascc.ZkStateReader.createClusterStateW

[...truncated too long message...]

erElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 1094557 T5662 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 1094755 T5300 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1094780 T5300 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1094785 T5707 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1094786 T5300 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1094786 T5708 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1094788 T5699 oascc.ZkStateReader$3.process Updating live nodes... (0)
   [junit4]   2> 1094788 T5705 oascc.ZkStateReader$3.process Updating live nodes... (0)
   [junit4]   2> 1094789 T5300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1094790 T5707 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1094791 T5711 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@12e67187 name:ZooKeeperConnection Watcher:127.0.0.1:11946/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1094791 T5300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1094792 T5300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1094794 T5300 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
   [junit4]   2> 1094796 T5711 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 1094796 T5699 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 1094799 T5707 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1094799 T5300 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1094799 T5708 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"recovering",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 1094800 T5708 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 1094813 T5707 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1094816 T5300 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1094827 T5707 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1094836 T5707 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1094939 T5711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1094939 T5699 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1094939 T5300 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1094950 T5699 oascc.ZkStateReader$3.process Updating live nodes... (0)
   [junit4]   2> 1094951 T5711 oascc.ZkStateReader$3.process Updating live nodes... (0)
   [junit4]   2> 1094954 T5300 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:11946 11946
   [junit4]   2> 1095081 T5300 oas.SolrTestCaseJ4.tearDown ###Ending testDoubleAssignment
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1095083 T5300 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> 1095083 T5300 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 1098085 T5300 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: test params are: codec=Lucene3x, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=en_CA, timezone=Etc/GMT+11
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Sun Microsystems Inc. 1.6.0_32 (64-bit)/cpus=16,threads=1,free=199314280,total=415629312
   [junit4]   2> NOTE: All tests run in this JVM: [SuggesterTest, SchemaVersionSpecificBehaviorTest, SolrIndexSplitterTest, ZkControllerTest, CopyFieldTest, TestSchemaVersionResource, TestSuggestSpellingConverter, SOLR749Test, SolrCoreTest, TestPostingsSolrHighlighter, RecoveryZkTest, HdfsCollectionsAPIDistributedZkTest, TestBlendedInfixSuggestions, SliceStateUpdateTest, CustomCollectionTest, TestTrie, BadComponentTest, BJQParserTest, ParsingFieldUpdateProcessorsTest, DeleteShardTest, DateMathParserTest, TestConfig, TestElisionMultitermQuery, FastVectorHighlighterTest, TestLFUCache, TestBadConfig, SystemInfoHandlerTest, OverseerRolesTest, TestRemoteStreaming, TestManagedSchema, OverseerCollectionProcessorTest, MBeansHandlerTest, SimpleFacetsTest, DistributedDebugComponentTest, TestDocumentBuilder, QueryParsingTest, UpdateRequestProcessorFactoryTest, TestPerFieldSimilarity, UnloadDistributedZkTest, CoreMergeIndexesAdminHandlerTest, TestSolrDeletionPolicy1, TestRealTimeGet, FileBasedSpellCheckerTest, TestPHPSerializedResponseWriter, TestSolrXmlPersistor, SynonymTokenizerTest, DocValuesTest, TestSolrXMLSerializer, TestLMJelinekMercerSimilarityFactory, EchoParamsTest, SoftAutoCommitTest, TestDocSet, TestFaceting, TestStressVersions, DistributedTermsComponentTest, ExternalFileFieldSortTest, TestWriterPerf, SliceStateTest, TestGroupingSearch, TestLRUCache, TestDefaultSearchFieldResource, TestCustomSort, TermVectorComponentTest, QueryElevationComponentTest, TriLevelCompositeIdRoutingTest, TermsComponentTest, ShowFileRequestHandlerTest, SearchHandlerTest, TestExtendedDismaxParser, ShardSplitTest, TestSystemIdResolver, TestCoreContainer, AliasIntegrationTest, CircularListTest, TestLuceneMatchVersion, URLClassifyProcessorTest, TestDocBasedVersionConstraints, TestSurroundQueryParser, TestAnalyzeInfixSuggestions, AnalysisErrorHandlingTest, BasicFunctionalityTest, PrimitiveFieldTypeTest, CSVRequestHandlerTest, DOMUtilTest, RemoteQueryErrorTest, ResourceLoaderTest, TestDynamicFieldCollectionResource, CollectionsAPIDistributedZkTest, OpenCloseCoreStressTest, OverseerTest]
   [junit4] Completed on J0 in 27.06s, 8 tests, 1 error <<< FAILURES!

[...truncated 588 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/build.xml:471: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/build.xml:451: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/solr/common-build.xml:490: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/lucene/common-build.xml:1278: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java6/lucene/common-build.xml:911: There were test failures: 363 suites, 1570 tests, 1 error, 36 ignored (8 assumptions)

Total time: 90 minutes 9 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Mime
View raw message