storm-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bo...@apache.org
Subject [47/50] storm git commit: Merge branch 'master' into security-upmerge
Date Thu, 13 Nov 2014 19:37:42 GMT
Merge branch 'master' into security-upmerge

Conflicts:
	storm-core/src/clj/backtype/storm/daemon/supervisor.clj
	storm-core/src/clj/backtype/storm/ui/core.clj
	storm-core/src/ui/public/templates/topology-page-template.html
	storm-core/test/clj/backtype/storm/supervisor_test.clj


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ea0b8a2a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ea0b8a2a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ea0b8a2a

Branch: refs/heads/master
Commit: ea0b8a2a51332020a569ed7aeb3e0647477af763
Parents: 4788c7e cfe7e63
Author: Robert (Bobby) Evans <evans@yahoo-inc.com>
Authored: Wed Oct 15 11:05:10 2014 -0500
Committer: Robert (Bobby) Evans <evans@yahoo-inc.com>
Committed: Wed Oct 15 11:05:10 2014 -0500

----------------------------------------------------------------------
 CHANGELOG.md                                    |  16 +-
 DEVELOPER.md                                    |  26 +-
 LICENSE                                         |  20 +-
 README.markdown                                 |   3 +
 bin/storm                                       |   3 +-
 examples/storm-starter/README.markdown          |   9 +-
 examples/storm-starter/pom.xml                  |   2 +-
 external/storm-hbase/LICENSE                    | 202 +++++++++++
 external/storm-hbase/README.md                  | 215 ++++++++++++
 external/storm-hbase/pom.xml                    |  78 +++++
 .../storm/hbase/bolt/AbstractHBaseBolt.java     |  69 ++++
 .../org/apache/storm/hbase/bolt/HBaseBolt.java  |  77 ++++
 .../storm/hbase/bolt/HBaseLookupBolt.java       |  83 +++++
 .../storm/hbase/bolt/mapper/HBaseMapper.java    |  48 +++
 .../bolt/mapper/HBaseProjectionCriteria.java    |  85 +++++
 .../hbase/bolt/mapper/HBaseValueMapper.java     |  43 +++
 .../hbase/bolt/mapper/SimpleHBaseMapper.java    |  90 +++++
 .../apache/storm/hbase/common/ColumnList.java   | 202 +++++++++++
 .../apache/storm/hbase/common/HBaseClient.java  | 135 +++++++
 .../org/apache/storm/hbase/common/IColumn.java  |  30 ++
 .../org/apache/storm/hbase/common/ICounter.java |  29 ++
 .../org/apache/storm/hbase/common/Utils.java    |  69 ++++
 .../storm/hbase/security/HBaseSecurityUtil.java |  52 +++
 .../mapper/SimpleTridentHBaseMapper.java        |  89 +++++
 .../trident/mapper/TridentHBaseMapper.java      |  31 ++
 .../hbase/trident/state/HBaseMapState.java      | 233 +++++++++++++
 .../storm/hbase/trident/state/HBaseQuery.java   |  40 +++
 .../storm/hbase/trident/state/HBaseState.java   | 164 +++++++++
 .../hbase/trident/state/HBaseStateFactory.java  |  40 +++
 .../storm/hbase/trident/state/HBaseUpdater.java |  32 ++
 .../storm/hbase/topology/LookupWordCount.java   |  79 +++++
 .../hbase/topology/PersistentWordCount.java     |  81 +++++
 .../storm/hbase/topology/TotalWordCounter.java  |  70 ++++
 .../storm/hbase/topology/WordCountClient.java   |  57 +++
 .../hbase/topology/WordCountValueMapper.java    |  70 ++++
 .../storm/hbase/topology/WordCounter.java       |  59 ++++
 .../apache/storm/hbase/topology/WordSpout.java  |  88 +++++
 .../storm/hbase/trident/PrintFunction.java      |  40 +++
 .../storm/hbase/trident/WordCountTrident.java   | 104 ++++++
 external/storm-hdfs/pom.xml                     |   2 +-
 external/storm-kafka/pom.xml                    |   2 +-
 .../jvm/storm/kafka/DynamicBrokersReader.java   |   2 +-
 .../storm/kafka/DynamicBrokersReaderTest.java   |   1 +
 .../src/test/storm/kafka/TridentKafkaTest.java  |   9 +-
 pom.xml                                         |  27 +-
 .../maven-shade-clojure-transformer/pom.xml     |   2 +-
 storm-core/pom.xml                              |   6 +-
 .../clj/backtype/storm/daemon/supervisor.clj    |   4 +-
 storm-core/src/clj/backtype/storm/ui/core.clj   |  51 +--
 storm-core/src/dev/resources/storm.py           | 248 ++++++++++++-
 storm-core/src/dev/resources/storm.rb           | 228 +++++++++++-
 .../src/jvm/backtype/storm/tuple/ITuple.java    | 136 ++++++++
 .../src/jvm/backtype/storm/tuple/Tuple.java     | 113 +-----
 .../storm/trident/testing/MockTridentTuple.java | 166 ---------
 .../jvm/storm/trident/tuple/TridentTuple.java   |  46 +--
 .../storm/trident/tuple/TridentTupleView.java   |  35 +-
 .../jvm/storm/trident/tuple/ValuePointer.java   |   1 -
 storm-core/src/multilang/js/storm.js            | 349 +++++++++++++++++++
 storm-core/src/ui/public/component.html         |   8 +-
 storm-core/src/ui/public/js/purl.js             | 267 --------------
 storm-core/src/ui/public/js/script.js           |   3 +-
 storm-core/src/ui/public/js/url.min.js          |   1 +
 storm-core/src/ui/public/js/visualization.js    |   2 +-
 .../templates/component-page-template.html      |   6 +-
 .../public/templates/index-page-template.html   |   2 +-
 .../templates/topology-page-template.html       |   6 +-
 storm-core/src/ui/public/topology.html          |  12 +-
 .../test/clj/backtype/storm/supervisor_test.clj |  12 +-
 .../test/clj/storm/trident/tuple_test.clj       |  28 +-
 storm-dist/binary/LICENSE                       |  10 +-
 storm-dist/binary/pom.xml                       |   2 +-
 storm-dist/binary/src/main/assembly/binary.xml  |  14 +
 storm-dist/source/pom.xml                       |   2 +-
 73 files changed, 3949 insertions(+), 717 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/README.markdown
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/bin/storm
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index 1fe5b2e,cce89bc..c580bc8
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@@ -605,9 -501,8 +605,9 @@@
  (defmethod launch-worker
      :distributed [supervisor storm-id port worker-id]
      (let [conf (:conf supervisor)
 +          run-worker-as-user (conf SUPERVISOR-RUN-WORKER-AS-USER)
            storm-home (System/getProperty "storm.home")
-           storm-log-dir (or (System/getProperty "storm.log.dir") (str storm-home "/logs"))
+           storm-log-dir (or (System/getProperty "storm.log.dir") (str storm-home file-path-separator
"logs"))
            stormroot (supervisor-stormdist-root conf storm-id)
            jlp (jlp stormroot conf)
            stormjar (supervisor-stormjar-path stormroot)
@@@ -638,7 -529,7 +638,7 @@@
                       (str "-Dlogfile.name=" logfilename)
                       (str "-Dstorm.home=" storm-home)
                       (str "-Dstorm.log.dir=" storm-log-dir)
-                      (str "-Dlogback.configurationFile=" storm-home "/logback/worker.xml")
 -                     (str "-Dlogback.configurationFile=" storm-home file-path-separator
"logback" file-path-separator "cluster.xml")
++                     (str "-Dlogback.configurationFile=" storm-home file-path-separator
"logback" file-path-separator "worker.xml")
                       (str "-Dstorm.id=" storm-id)
                       (str "-Dworker.id=" worker-id)
                       (str "-Dworker.port=" port)

http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/ui/core.clj
index 600c96a,969b514..e980a55
--- a/storm-core/src/clj/backtype/storm/ui/core.clj
+++ b/storm-core/src/clj/backtype/storm/ui/core.clj
@@@ -541,8 -519,9 +542,10 @@@
    ([summs]
     {"topologies"
      (for [^TopologySummary t summs]
-       {"id" (.get_id t)
+       {
+        "id" (.get_id t)
+        "encodedId" (url-encode (.get_id t))
 +       "owner" (.get_owner t)
         "name" (.get_name t)
         "status" (.get_status t)
         "uptime" (pretty-uptime-sec (.get_uptime_secs t))
@@@ -573,8 -551,9 +576,9 @@@
                        stats-seq include-sys?))
                last-error (most-recent-error (get errors id))
                error-host (get-error-host last-error)
 -              error-port (get-error-port last-error error-host top-id) ]]
 +              error-port (get-error-port last-error error-host top-id)]]
      {"spoutId" id
+      "encodedSpoutId" (url-encode id)
       "executors" (count summs)
       "tasks" (sum-tasks summs)
       "emitted" (get-in stats [:emitted window])
@@@ -596,8 -575,9 +600,9 @@@
                        stats-seq include-sys?))
                last-error (most-recent-error (get errors id))
                error-host (get-error-host last-error)
 -              error-port (get-error-port last-error error-host top-id) ]]
 +              error-port (get-error-port last-error error-host top-id)]]
      {"boltId" id
+      "encodedBoltId" (url-encode id)
       "executors" (count summs)
       "tasks" (sum-tasks summs)
       "emitted" (get-in stats [:emitted window])
@@@ -619,7 -599,7 +624,8 @@@
          workers (set (for [^ExecutorSummary e executors]
                         [(.get_host e) (.get_port e)]))]
        {"id" (.get_id summ)
+        "encodedId" (url-encode (.get_id summ))
 +       "owner" (.get_owner summ)
         "name" (.get_name summ)
         "status" (.get_status summ)
         "uptime" (pretty-uptime-sec (.get_uptime_secs summ))
@@@ -832,8 -811,8 +841,9 @@@
                       (= type :bolt) (bolt-stats window summ component summs include-sys?))
            errors (component-errors (get (.get_errors summ) component) topology-id)]
        (merge
 -       { "id" component
 +        {"user" user
 +         "id" component
+          "encodedId" (url-encode component)
           "name" (.get_name summ)
           "executors" (count summs)
           "tasks" (sum-tasks summs)
@@@ -869,67 -843,47 +880,59 @@@
    (GET "/api/v1/cluster/configuration" [& m]
         (json-response (cluster-configuration)
                        (:callback m) :serialize-fn identity))
 -  (GET "/api/v1/cluster/summary" [& m]
 -       (json-response (cluster-summary) (:callback m)))
 -  (GET "/api/v1/supervisor/summary" [& m]
 +  (GET "/api/v1/cluster/summary" [:as {:keys [cookies servlet-request]} & m]
 +       (let [user (.getUserName http-creds-handler servlet-request)]
 +         (assert-authorized-user servlet-request "getClusterInfo")
 +         (json-response (cluster-summary user) (:callback m))))
 +  (GET "/api/v1/supervisor/summary" [:as {:keys [cookies servlet-request]} & m]
 +       (assert-authorized-user servlet-request "getClusterInfo")
         (json-response (supervisor-summary) (:callback m)))
 -  (GET "/api/v1/topology/summary" [& m]
 +  (GET "/api/v1/topology/summary" [:as {:keys [cookies servlet-request]} & m]
 +       (assert-authorized-user servlet-request "getClusterInfo")
         (json-response (all-topologies-summary) (:callback m)))
 -  (GET  "/api/v1/topology/:id" [id & m]
 -          (json-response (topology-page id (:window m) (check-include-sys? (:sys m))) (:callback
m)))
 +  (GET  "/api/v1/topology/:id" [:as {:keys [cookies servlet-request]} id & m]
-         (let [id (url-decode id)
-               user (.getUserName http-creds-handler servlet-request)]
++        (let [user (.getUserName http-creds-handler servlet-request)]
 +          (assert-authorized-user servlet-request "getTopology" (topology-config id))
 +          (json-response (topology-page id (:window m) (check-include-sys? (:sys m)) user)
(:callback m))))
    (GET "/api/v1/topology/:id/visualization" [:as {:keys [cookies servlet-request]} id &
m]
-         (let [id (url-decode id)]
-           (assert-authorized-user servlet-request "getTopology" (topology-config id))
-           (json-response (mk-visualization-data id (:window m) (check-include-sys? (:sys
m))) (:callback m))))
 -       (json-response (mk-visualization-data id (:window m) (check-include-sys? (:sys m)))
(:callback m)))
 -  (GET "/api/v1/topology/:id/component/:component" [id component & m]
 -        (json-response (component-page id component (:window m) (check-include-sys? (:sys
m))) (:callback m)))
 -  (POST "/api/v1/topology/:id/activate" [id]
++        (assert-authorized-user servlet-request "getTopology" (topology-config id))
++        (json-response (mk-visualization-data id (:window m) (check-include-sys? (:sys m)))
(:callback m)))
 +  (GET "/api/v1/topology/:id/component/:component" [:as {:keys [cookies servlet-request]}
id component & m]
-        (let [id (url-decode id)
-              component (url-decode component)
-              user (.getUserName http-creds-handler servlet-request)]
++       (let [user (.getUserName http-creds-handler servlet-request)]
 +         (assert-authorized-user servlet-request "getTopology" (topology-config id))
 +         (json-response (component-page id component (:window m) (check-include-sys? (:sys
m)) user) (:callback m))))
 +  (POST "/api/v1/topology/:id/activate" [:as {:keys [cookies servlet-request]} id]
      (with-nimbus nimbus
-       (let [id (url-decode id)
-             tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+       (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
              name (.get_name tplg)]
 +        (assert-authorized-user servlet-request "activate" (topology-config id))
          (.activate nimbus name)
          (log-message "Activating topology '" name "'")))
 -    (resp/redirect (str "/api/v1/topology/" (url-encode id))))
 -
 -  (POST "/api/v1/topology/:id/deactivate" [id]
 +    (resp/redirect (str "/api/v1/topology/" id)))
 +  (POST "/api/v1/topology/:id/deactivate" [:as {:keys [cookies servlet-request]} id]
      (with-nimbus nimbus
-       (let [id (url-decode id)
-             tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+       (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
              name (.get_name tplg)]
 +        (assert-authorized-user servlet-request "deactivate" (topology-config id))
          (.deactivate nimbus name)
          (log-message "Deactivating topology '" name "'")))
-     (resp/redirect (str "/api/v1/topology/" id)))
+     (resp/redirect (str "/api/v1/topology/" (url-encode id))))
 -  (POST "/api/v1/topology/:id/rebalance/:wait-time" [id wait-time]
 +  (POST "/api/v1/topology/:id/rebalance/:wait-time" [:as {:keys [cookies servlet-request]}
id wait-time]
      (with-nimbus nimbus
-       (let [id (url-decode id)
-             tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+       (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
              name (.get_name tplg)
              options (RebalanceOptions.)]
 +        (assert-authorized-user servlet-request "rebalance" (topology-config id))
          (.set_wait_secs options (Integer/parseInt wait-time))
          (.rebalance nimbus name options)
          (log-message "Rebalancing topology '" name "' with wait time: " wait-time " secs")))
-     (resp/redirect (str "/api/v1/topology/" id)))
+     (resp/redirect (str "/api/v1/topology/" (url-encode id))))
 -  (POST "/api/v1/topology/:id/kill/:wait-time" [id wait-time]
 +  (POST "/api/v1/topology/:id/kill/:wait-time" [:as {:keys [cookies servlet-request]} id
wait-time]
      (with-nimbus nimbus
-       (let [id (url-decode id)
-             tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
+       (let [tplg (.getTopologyInfo ^Nimbus$Client nimbus id)
              name (.get_name tplg)
              options (KillOptions.)]
 +        (assert-authorized-user servlet-request "killTopology" (topology-config id))
          (.set_wait_secs options (Integer/parseInt wait-time))
          (.killTopologyWithOpts nimbus name options)
          (log-message "Killing topology '" name "' with wait time: " wait-time " secs")))

http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/src/ui/public/component.html
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/src/ui/public/js/script.js
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/src/ui/public/templates/index-page-template.html
----------------------------------------------------------------------
diff --cc storm-core/src/ui/public/templates/index-page-template.html
index 800b579,e3aab5e..9c48e83
--- a/storm-core/src/ui/public/templates/index-page-template.html
+++ b/storm-core/src/ui/public/templates/index-page-template.html
@@@ -128,9 -118,8 +128,9 @@@
      <tbody>
        {{#topologies}}
        <tr>
-         <td><a href="/topology.html?id={{id}}">{{name}}</a></td>
+         <td><a href="/topology.html?id={{encodedId}}">{{name}}</a></td>
          <td>{{id}}</td>
 +        <td>{{owner}}</td>
          <td>{{status}}</td>
          <td>{{uptime}}</td>
          <td>{{workersTotal}}</td>

http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/src/ui/public/templates/topology-page-template.html
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/src/ui/public/topology.html
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------
diff --cc storm-core/test/clj/backtype/storm/supervisor_test.clj
index 4261b4d,ba74d88..ebbf060
--- a/storm-core/test/clj/backtype/storm/supervisor_test.clj
+++ b/storm-core/test/clj/backtype/storm/supervisor_test.clj
@@@ -259,10 -253,10 +259,10 @@@
                                 opts
                                 topo-opts
                                 ["-Djava.library.path="
 -                                (str "-Dlogfile.name=worker-" mock-port ".log")
 +                                (str "-Dlogfile.name=" mock-storm-id "-worker-" mock-port
".log")
                                  "-Dstorm.home="
-                                 "-Dstorm.log.dir=/logs"
-                                 "-Dlogback.configurationFile=/logback/worker.xml"
+                                 (str "-Dstorm.log.dir=" file-path-separator "logs")
 -                                (str "-Dlogback.configurationFile=" file-path-separator
"logback" file-path-separator "cluster.xml")
++                                (str "-Dlogback.configurationFile=" file-path-separator
"logback" file-path-separator "worker.xml")
                                  (str "-Dstorm.id=" mock-storm-id)
                                  (str "-Dworker.id=" mock-worker-id)
                                  (str "-Dworker.port=" mock-port)
@@@ -322,10 -312,8 +322,10 @@@
            (stubbing [read-supervisor-storm-conf {TOPOLOGY-CLASSPATH topo-cp}
                       supervisor-stormdist-root nil
                       supervisor/jlp nil
 +                     set-worker-user! nil
 +                     supervisor/write-log-metadata! nil
                       launch-process nil
-                      current-classpath "/base"]
+                      current-classpath (str file-path-separator "base")]
                      (supervisor/launch-worker mock-supervisor
                                                mock-storm-id
                                                mock-port
@@@ -342,9 -329,7 +342,9 @@@
                       supervisor-stormdist-root nil
                       supervisor/jlp nil
                       launch-process nil
 +                     set-worker-user! nil
 +                     supervisor/write-log-metadata! nil
-                      current-classpath "/base"]
+                      current-classpath (str file-path-separator "base")]
                      (supervisor/launch-worker mock-supervisor
                                                mock-storm-id
                                                mock-port

http://git-wip-us.apache.org/repos/asf/storm/blob/ea0b8a2a/storm-dist/binary/src/main/assembly/binary.xml
----------------------------------------------------------------------


Mime
View raw message