storm-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bo...@apache.org
Subject [29/50] [abbrv] storm git commit: Merge remote-tracking branch 'upstream/master' into STORM-166
Date Mon, 24 Aug 2015 13:52:14 GMT
Merge remote-tracking branch 'upstream/master' into STORM-166

Conflicts:
	storm-core/src/clj/backtype/storm/daemon/nimbus.clj
	storm-core/src/clj/backtype/storm/ui/core.clj


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/23ed5375
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/23ed5375
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/23ed5375

Branch: refs/heads/master
Commit: 23ed53754c47a2170e58b178beb1544616f0f08a
Parents: 63f29f3 8036109
Author: Parth Brahmbhatt <brahmbhatt.parth@gmail.com>
Authored: Thu Feb 12 16:44:06 2015 -0800
Committer: Parth Brahmbhatt <brahmbhatt.parth@gmail.com>
Committed: Thu Feb 12 16:44:06 2015 -0800

----------------------------------------------------------------------
 .gitignore                                      |   1 +
 CHANGELOG.md                                    |  25 +-
 README.markdown                                 |  13 +-
 SECURITY.md                                     |  14 +-
 STORM-UI-REST-API.md                            |  21 +
 bin/storm                                       |  36 +-
 bin/storm-config.cmd                            |  32 +-
 bin/storm.cmd                                   |  15 +-
 conf/defaults.yaml                              |   3 +-
 dev-tools/github/__init__.py                    |  11 +
 dev-tools/jira-github-join.py                   |   4 +-
 dev-tools/storm-merge.py                        |  31 +
 docs/README.md                                  |   9 +
 docs/documentation/Common-patterns.md           |  14 +-
 docs/documentation/Concepts.md                  |  13 +-
 docs/documentation/Contributing-to-Storm.md     |   2 +-
 docs/documentation/Home.md                      |   2 +-
 docs/documentation/Multilang-protocol.md        |   4 +-
 docs/documentation/Powered-By.md                |  15 +-
 docs/downloads.html                             |   3 +
 .../storm/starter/SkewedRollingTopWords.java    | 134 +++
 .../storm/starter/bolt/RollingCountAggBolt.java |  78 ++
 .../hbase/trident/state/HBaseMapState.java      |   2 +-
 .../storm/hbase/trident/state/HBaseState.java   |   2 +-
 external/storm-kafka/README.md                  |  24 +-
 external/storm-kafka/pom.xml                    |  21 +-
 .../ExponentialBackoffMsgRetryManager.java      | 167 ++++
 .../jvm/storm/kafka/FailedMsgRetryManager.java  |  26 +
 .../src/jvm/storm/kafka/PartitionManager.java   |  42 +-
 .../src/jvm/storm/kafka/SpoutConfig.java        |   8 +
 .../ExponentialBackoffMsgRetryManagerTest.java  | 194 ++++
 pom.xml                                         |  19 +-
 storm-buildtools/storm-maven-plugins/pom.xml    |  81 ++
 .../storm/maven/plugin/util/CommandExec.java    |  89 ++
 .../plugin/versioninfo/VersionInfoMojo.java     | 304 ++++++
 storm-core/pom.xml                              |  41 +-
 storm-core/src/clj/backtype/storm/cluster.clj   |  38 +-
 .../src/clj/backtype/storm/daemon/logviewer.clj |   5 +-
 .../src/clj/backtype/storm/daemon/nimbus.clj    |  41 +-
 .../clj/backtype/storm/daemon/supervisor.clj    |   5 +-
 .../src/clj/backtype/storm/daemon/worker.clj    |   6 +-
 storm-core/src/clj/backtype/storm/ui/core.clj   |  39 +-
 storm-core/src/clj/backtype/storm/util.clj      |  11 +-
 storm-core/src/jvm/backtype/storm/Config.java   |   6 +
 .../coordination/BatchSubtopologyBuilder.java   |  11 +
 .../storm/drpc/LinearDRPCInputDeclarer.java     |   5 +-
 .../storm/drpc/LinearDRPCTopologyBuilder.java   |  13 +-
 .../storm/generated/GetInfoOptions.java         | 350 +++++++
 .../jvm/backtype/storm/generated/Nimbus.java    | 974 +++++++++++++++++++
 .../storm/generated/NumErrorsChoice.java        |  64 ++
 .../storm/grouping/PartialKeyGrouping.java      |  82 ++
 .../backtype/storm/messaging/netty/Server.java  |   2 +
 .../backtype/storm/topology/InputDeclarer.java  |   3 +
 .../storm/topology/TopologyBuilder.java         |  11 +
 .../TransactionalTopologyBuilder.java           |  13 +-
 .../src/jvm/backtype/storm/utils/Monitor.java   |   8 +-
 .../jvm/backtype/storm/utils/VersionInfo.java   | 131 +++
 .../topology/TridentTopologyBuilder.java        |  13 +-
 storm-core/src/py/storm/DistributedRPC-remote   |   0
 .../py/storm/DistributedRPCInvocations-remote   |   0
 storm-core/src/py/storm/Nimbus-remote           |   7 +
 storm-core/src/py/storm/Nimbus.py               | 226 +++++
 storm-core/src/py/storm/ttypes.py               |  80 ++
 .../storm-core-version-info.properties          |  24 +
 storm-core/src/storm.thrift                     |  11 +
 .../templates/topology-page-template.html       |   8 +-
 .../clj/backtype/storm/integration_test.clj     |  10 +-
 .../scheduler/multitenant_scheduler_test.clj    |   2 +
 .../test/clj/backtype/storm/supervisor_test.clj |  34 +-
 .../storm/grouping/PartialKeyGroupingTest.java  |  66 ++
 storm-dist/binary/src/main/assembly/binary.xml  |   7 -
 71 files changed, 3647 insertions(+), 159 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/STORM-UI-REST-API.md
----------------------------------------------------------------------
diff --cc STORM-UI-REST-API.md
index 4801a51,deee134..e677c11
--- a/STORM-UI-REST-API.md
+++ b/STORM-UI-REST-API.md
@@@ -250,8 -219,10 +250,11 @@@ Response fields
  |bolts.errorWorkerLogLink| String | Link to the worker log that reported the exception |
  |bolts.emitted| Long |Number of tuples emitted|
  |antiForgeryToken| String | CSRF token|
 +|replicationCount| Integer |Number of nimbus hosts on which this topology code is replicated|
  
+ Caution: users need to unescape the antiForgeryToken value before using this token to make
POST calls(simple-json escapes forward slashes)
+ [ISSUE-8](https://code.google.com/p/json-simple/issues/detail?id=8)
+ 
  
  Examples:
  

http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/conf/defaults.yaml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/pom.xml
----------------------------------------------------------------------
diff --cc pom.xml
index b35a8d2,af86bd0..a5543b9
--- a/pom.xml
+++ b/pom.xml
@@@ -209,9 -210,9 +210,8 @@@
          <reply.version>0.3.0</reply.version>
          <conjure.version>2.1.3</conjure.version>
          <zookeeper.version>3.4.6</zookeeper.version>
-         <conjure.version>2.1.3</conjure.version>
          <clojure-data-codec.version>0.1.0</clojure-data-codec.version>
          <clojure-contrib.version>1.2.0</clojure-contrib.version>
 -
      </properties>
  
      <profiles>

http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/storm-core/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/storm-core/src/clj/backtype/storm/cluster.clj
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/daemon/nimbus.clj
index 26bd591,0c7612b..30d123d
--- a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
@@@ -24,13 -22,12 +24,14 @@@
    (:use [backtype.storm.scheduler.DefaultScheduler])
    (:import [backtype.storm.scheduler INimbus SupervisorDetails WorkerSlot TopologyDetails
              Cluster Topologies SchedulerAssignment SchedulerAssignmentImpl DefaultScheduler
ExecutorDetails])
 +  (:use [backtype.storm bootstrap util zookeeper])
-   (:import [backtype.storm.generated AuthorizationException])
 +  (:import [backtype.storm.nimbus NimbusInfo])
-   (:use [backtype.storm bootstrap util])
+   (:import [backtype.storm.generated AuthorizationException GetInfoOptions
+                                      NumErrorsChoice])
 -  (:use [backtype.storm bootstrap util])
    (:use [backtype.storm.config :only [validate-configs-with-schemas]])
    (:use [backtype.storm.daemon common])
    (:import [org.apache.zookeeper data.ACL ZooDefs$Ids ZooDefs$Perms])
++  (:import [backtype.storm.utils VersionInfo])
    (:gen-class
      :methods [^{:static true} [launch [backtype.storm.scheduler.INimbus] void]]))
  
@@@ -1031,24 -991,9 +1039,24 @@@
    (let [nimbus (nimbus-data conf inimbus)
         principal-to-local (AuthUtils/GetPrincipalToLocalPlugin conf)]
      (.prepare ^backtype.storm.nimbus.ITopologyValidator (:validator nimbus) conf)
 +
 +    ;add to nimbuses
 +    (.add-nimbus-host! (:storm-cluster-state nimbus)
 +      (.toHostPortString (:nimbus-host-port-info nimbus))
 +      {
 +        :host (.getHost (:nimbus-host-port-info nimbus))
 +        :port (.getPort (:nimbus-host-port-info nimbus))
 +        :start-time-secs (current-time-secs)
-         :version (read-storm-version)
++        :version (str (VersionInfo/getVersion))
 +        })
 +
 +    (.addToLeaderLockQueue (:leader-elector nimbus))
      (cleanup-corrupt-topologies! nimbus)
 -    (doseq [storm-id (.active-storms (:storm-cluster-state nimbus))]
 -      (transition! nimbus storm-id :startup))
 +    ;register call back for code-distributor
 +    (.code-distributor (:storm-cluster-state nimbus) (fn [] (sync-code conf nimbus)))
 +    (when (is-leader nimbus :throw-exception false)
 +      (doseq [storm-id (.active-storms (:storm-cluster-state nimbus))]
 +        (transition! nimbus storm-id :startup)))
      (schedule-recurring (:timer nimbus)
                          0
                          (conf NIMBUS-MONITOR-FREQ-SECS)
@@@ -1332,11 -1259,11 +1340,11 @@@
                                                 topo-summ
                                            ))]
            (ClusterSummary. supervisor-summaries
 -                           nimbus-uptime
 -                           topology-summaries)
 +                           topology-summaries
 +                           nimbuses)
            ))
        
-       (^TopologyInfo getTopologyInfo [this ^String storm-id]
+       (^TopologyInfo getTopologyInfoWithOpts [this ^String storm-id ^GetInfoOptions options]
          (let [storm-cluster-state (:storm-cluster-state nimbus)
                topology-conf (try-read-storm-conf conf storm-id)
                storm-name (topology-conf TOPOLOGY-NAME)
@@@ -1373,10 -1314,14 +1395,15 @@@
                             )]
              (when-let [owner (:owner base)] (.set_owner topo-info owner))
              (when-let [sched-status (.get @(:id->sched-status nimbus) storm-id)] (.set_sched_status
topo-info sched-status))
 +            (.set_replication_count topo-info (.getReplicationCount (:code-distributor nimbus)
storm-id))
              topo-info
            ))
-       
+ 
+       (^TopologyInfo getTopologyInfo [this ^String storm-id]
+         (.getTopologyInfoWithOpts this
+                                   storm-id
+                                   (doto (GetInfoOptions.) (.set_num_err_choice NumErrorsChoice/ALL))))
+ 
        Shutdownable
        (shutdown [this]
          (log-message "Shutting down master")

http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/ui/core.clj
index 8fd22a6,b36a7a9..726da73
--- a/storm-core/src/clj/backtype/storm/ui/core.clj
+++ b/storm-core/src/clj/backtype/storm/ui/core.clj
@@@ -498,7 -504,8 +502,7 @@@
                               (map #(.get_num_executors ^TopologySummary %))
                               (reduce +))]
         {"user" user
-         "stormVersion" (read-storm-version)
+         "stormVersion" (str (VersionInfo/getVersion))
 -        "nimbusUptime" (pretty-uptime-sec (.get_nimbus_uptime_secs summ))
          "supervisors" (count sups)
          "slotsTotal" total-slots
          "slotsUsed"  used-slots
@@@ -654,10 -643,13 +658,13 @@@
          "failed" (get-in stats [:failed k])})))
  
  (defn topology-page [id window include-sys? user]
 -  (with-nimbus nimbus
 +  (thrift/with-configured-nimbus-connection nimbus
      (let [window (if window window ":all-time")
            window-hint (window-hint window)
-           summ (.getTopologyInfo ^Nimbus$Client nimbus id)
+           summ (->> (doto
+                       (GetInfoOptions.)
+                       (.set_num_err_choice NumErrorsChoice/ONE))
+                     (.getTopologyInfoWithOpts ^Nimbus$Client nimbus id))
            topology (.getTopology ^Nimbus$Client nimbus id)
            topology-conf (from-json (.getTopologyConf ^Nimbus$Client nimbus id))
            spout-summs (filter (partial spout-summary? topology) (.get_executors summ))
@@@ -911,25 -898,36 +918,36 @@@
         (let [user (.getUserName http-creds-handler servlet-request)]
           (assert-authorized-user servlet-request "getTopology" (topology-config id))
           (json-response (component-page id component (:window m) (check-include-sys? (:sys
m)) user) (:callback m))))
+   (GET "/api/v1/token" [ & m]
+        (json-response (format "{\"antiForgeryToken\": \"%s\"}" *anti-forgery-token*) (:callback
m) :serialize-fn identity))
    (POST "/api/v1/topology/:id/activate" [:as {:keys [cookies servlet-request]} id]
 -    (with-nimbus nimbus
 +    (thrift/with-configured-nimbus-connection nimbus
-       (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+       (let [tplg (->> (doto
+                         (GetInfoOptions.)
+                         (.set_num_err_choice NumErrorsChoice/NONE))
+                       (.getTopologyInfoWithOpts ^Nimbus$Client nimbus id))
              name (.get_name tplg)]
          (assert-authorized-user servlet-request "activate" (topology-config id))
          (.activate nimbus name)
          (log-message "Activating topology '" name "'")))
-     (resp/redirect (str "/api/v1/topology/" id)))
+     (resp/redirect (str "/api/v1/topology/" (url-encode id))))
    (POST "/api/v1/topology/:id/deactivate" [:as {:keys [cookies servlet-request]} id]
 -    (with-nimbus nimbus
 +    (thrift/with-configured-nimbus-connection nimbus
-       (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+       (let [tplg (->> (doto
+                         (GetInfoOptions.)
+                         (.set_num_err_choice NumErrorsChoice/NONE))
+                       (.getTopologyInfoWithOpts ^Nimbus$Client nimbus id))
              name (.get_name tplg)]
          (assert-authorized-user servlet-request "deactivate" (topology-config id))
          (.deactivate nimbus name)
          (log-message "Deactivating topology '" name "'")))
      (resp/redirect (str "/api/v1/topology/" (url-encode id))))
    (POST "/api/v1/topology/:id/rebalance/:wait-time" [:as {:keys [cookies servlet-request]}
id wait-time]
 -    (with-nimbus nimbus
 +    (thrift/with-configured-nimbus-connection nimbus
-       (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+       (let [tplg (->> (doto
+                         (GetInfoOptions.)
+                         (.set_num_err_choice NumErrorsChoice/NONE))
+                       (.getTopologyInfoWithOpts ^Nimbus$Client nimbus id))
              name (.get_name tplg)
              options (RebalanceOptions.)]
          (assert-authorized-user servlet-request "rebalance" (topology-config id))
@@@ -938,8 -936,11 +956,11 @@@
          (log-message "Rebalancing topology '" name "' with wait time: " wait-time " secs")))
      (resp/redirect (str "/api/v1/topology/" (url-encode id))))
    (POST "/api/v1/topology/:id/kill/:wait-time" [:as {:keys [cookies servlet-request]} id
wait-time]
 -    (with-nimbus nimbus
 +    (thrift/with-configured-nimbus-connection nimbus
-       (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+       (let [tplg (->> (doto
+                         (GetInfoOptions.)
+                         (.set_num_err_choice NumErrorsChoice/NONE))
+                       (.getTopologyInfoWithOpts ^Nimbus$Client nimbus id))
              name (.get_name tplg)
              options (KillOptions.)]
          (assert-authorized-user servlet-request "killTopology" (topology-config id))

http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/storm-core/src/py/storm/ttypes.py
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/storm-core/src/storm.thrift
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/storm-core/src/ui/public/templates/topology-page-template.html
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/23ed5375/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------


Mime
View raw message