storm-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Stig Rohde Døssing (JIRA) <j...@apache.org>
Subject [jira] [Updated] (STORM-3354) LeaderElector is not shut down properly
Date Thu, 07 Mar 2019 11:18:00 GMT

     [ https://issues.apache.org/jira/browse/STORM-3354?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]

Stig Rohde Døssing updated STORM-3354:
--------------------------------------
    Description: 
Nimbus' LeaderElector is not shut down when Nimbus shuts down. This can cause test flakiness
because the elector callback may be called after the Zookeeper client is closed.

Additionally the LeaderListenerCallback may in some cases quit the leadership election if
Nimbus isn't ready to become master, but I don't see any code to re-enter the election.
 
{quote}java.lang.IllegalStateException: Client is not started
         at org.apache.storm.shade.org.apache.curator.shaded.com.google.common.base.Preconditions.checkState(Preconditions.java:444)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:139)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:602)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$17.call(CreateBuilderImpl.java:1191)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$17.call(CreateBuilderImpl.java:1158)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.curator.connection.StandardConnectionHandlingPolicy.callWithRetry(StandardConnectionHandlingPolicy.java:64)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:100)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.pathInForeground(CreateBuilderImpl.java:1155)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.protectedPathInForeground(CreateBuilderImpl.java:605)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.forPath(CreateBuilderImpl.java:595)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$3.forPath(CreateBuilderImpl.java:360)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$3.forPath(CreateBuilderImpl.java:308)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.zookeeper.ClientZookeeper.createNode(ClientZookeeper.java:98)
~[storm-client-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.nimbus.LeaderListenerCallback.setUpNimbusInfo(LeaderListenerCallback.java:154)
~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.nimbus.LeaderListenerCallback.leaderCallBack(LeaderListenerCallback.java:96)
~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.zookeeper.Zookeeper$1.isLeader(Zookeeper.java:123) ~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$9.apply(LeaderLatch.java:665)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$9.apply(LeaderLatch.java:661)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.curator.framework.listen.ListenerContainer$1.run(ListenerContainer.java:93)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.curator.shaded.com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:435)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.curator.framework.listen.ListenerContainer.forEach(ListenerContainer.java:85)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.setLeadership(LeaderLatch.java:660)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.checkLeadership(LeaderLatch.java:539)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.access$700(LeaderLatch.java:65)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$7.processResult(LeaderLatch.java:590)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.sendToBackgroundCallback(CuratorFrameworkImpl.java:865)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.processBackgroundOperation(CuratorFrameworkImpl.java:635)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.curator.framework.imps.WatcherRemovalFacade.processBackgroundOperation(WatcherRemovalFacade.java:152)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.curator.framework.imps.GetChildrenBuilderImpl$2.processResult(GetChildrenBuilderImpl.java:187)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:590)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
         at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT] 
{quote}

  was:
Nimbus' LeaderElector is not shut down when Nimbus shuts down. This can cause test flakiness
because the elector callback may be called after the Zookeeper client is closed.

 

{quote}

java.lang.IllegalStateException: Client is not started
        at org.apache.storm.shade.org.apache.curator.shaded.com.google.common.base.Preconditions.checkState(Preconditions.java:444)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:139)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:602)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$17.call(CreateBuilderImpl.java:1191)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$17.call(CreateBuilderImpl.java:1158)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.curator.connection.StandardConnectionHandlingPolicy.callWithRetry(StandardConnectionHandlingPolicy.java:64)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:100)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.pathInForeground(CreateBuilderImpl.java:1155)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.protectedPathInForeground(CreateBuilderImpl.java:605)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.forPath(CreateBuilderImpl.java:595)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$3.forPath(CreateBuilderImpl.java:360)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$3.forPath(CreateBuilderImpl.java:308)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.zookeeper.ClientZookeeper.createNode(ClientZookeeper.java:98)
~[storm-client-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.nimbus.LeaderListenerCallback.setUpNimbusInfo(LeaderListenerCallback.java:154)
~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.nimbus.LeaderListenerCallback.leaderCallBack(LeaderListenerCallback.java:96)
~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.zookeeper.Zookeeper$1.isLeader(Zookeeper.java:123) ~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$9.apply(LeaderLatch.java:665)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$9.apply(LeaderLatch.java:661)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.curator.framework.listen.ListenerContainer$1.run(ListenerContainer.java:93)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.curator.shaded.com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:435)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.curator.framework.listen.ListenerContainer.forEach(ListenerContainer.java:85)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.setLeadership(LeaderLatch.java:660)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.checkLeadership(LeaderLatch.java:539)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.access$700(LeaderLatch.java:65)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$7.processResult(LeaderLatch.java:590)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.sendToBackgroundCallback(CuratorFrameworkImpl.java:865)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.processBackgroundOperation(CuratorFrameworkImpl.java:635)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.curator.framework.imps.WatcherRemovalFacade.processBackgroundOperation(WatcherRemovalFacade.java:152)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.curator.framework.imps.GetChildrenBuilderImpl$2.processResult(GetChildrenBuilderImpl.java:187)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:590)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
        at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]

{quote}


> LeaderElector is not shut down properly
> ---------------------------------------
>
>                 Key: STORM-3354
>                 URL: https://issues.apache.org/jira/browse/STORM-3354
>             Project: Apache Storm
>          Issue Type: Bug
>    Affects Versions: 2.0.0
>            Reporter: Stig Rohde Døssing
>            Assignee: Stig Rohde Døssing
>            Priority: Major
>
> Nimbus' LeaderElector is not shut down when Nimbus shuts down. This can cause test flakiness
because the elector callback may be called after the Zookeeper client is closed.
> Additionally the LeaderListenerCallback may in some cases quit the leadership election
if Nimbus isn't ready to become master, but I don't see any code to re-enter the election.
>  
> {quote}java.lang.IllegalStateException: Client is not started
>          at org.apache.storm.shade.org.apache.curator.shaded.com.google.common.base.Preconditions.checkState(Preconditions.java:444)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:139)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:602)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$17.call(CreateBuilderImpl.java:1191)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$17.call(CreateBuilderImpl.java:1158)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.curator.connection.StandardConnectionHandlingPolicy.callWithRetry(StandardConnectionHandlingPolicy.java:64)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:100)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.pathInForeground(CreateBuilderImpl.java:1155)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.protectedPathInForeground(CreateBuilderImpl.java:605)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl.forPath(CreateBuilderImpl.java:595)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$3.forPath(CreateBuilderImpl.java:360)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.curator.framework.imps.CreateBuilderImpl$3.forPath(CreateBuilderImpl.java:308)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.zookeeper.ClientZookeeper.createNode(ClientZookeeper.java:98)
~[storm-client-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.nimbus.LeaderListenerCallback.setUpNimbusInfo(LeaderListenerCallback.java:154)
~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.nimbus.LeaderListenerCallback.leaderCallBack(LeaderListenerCallback.java:96)
~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.zookeeper.Zookeeper$1.isLeader(Zookeeper.java:123)
~[storm-server-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$9.apply(LeaderLatch.java:665)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$9.apply(LeaderLatch.java:661)
~[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.curator.framework.listen.ListenerContainer$1.run(ListenerContainer.java:93)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.curator.shaded.com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:435)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.curator.framework.listen.ListenerContainer.forEach(ListenerContainer.java:85)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.setLeadership(LeaderLatch.java:660)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.checkLeadership(LeaderLatch.java:539)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch.access$700(LeaderLatch.java:65)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.curator.framework.recipes.leader.LeaderLatch$7.processResult(LeaderLatch.java:590)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.sendToBackgroundCallback(CuratorFrameworkImpl.java:865)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.curator.framework.imps.CuratorFrameworkImpl.processBackgroundOperation(CuratorFrameworkImpl.java:635)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.curator.framework.imps.WatcherRemovalFacade.processBackgroundOperation(WatcherRemovalFacade.java:152)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.curator.framework.imps.GetChildrenBuilderImpl$2.processResult(GetChildrenBuilderImpl.java:187)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:590)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT]
>          at org.apache.storm.shade.org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
[shaded-deps-2.0.1-SNAPSHOT.jar:2.0.1-SNAPSHOT] 
> {quote}



--
This message was sent by Atlassian JIRA
(v7.6.3#76005)

Mime
View raw message