storm-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bo...@apache.org
Subject [04/10] storm git commit: Make worker profiling optional and pluggable
Date Wed, 04 Nov 2015 17:18:47 GMT
Make worker profiling optional and pluggable


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/d2f9305a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/d2f9305a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/d2f9305a

Branch: refs/heads/master
Commit: d2f9305abb16ee3573abaab56de7b1e673168cd5
Parents: 0c2021e
Author: Kishor Patil <kpatil@yahoo-inc.com>
Authored: Mon Nov 2 23:51:33 2015 -0600
Committer: Kishor Patil <kpatil@yahoo-inc.com>
Committed: Mon Nov 2 23:51:33 2015 -0600

----------------------------------------------------------------------
 conf/defaults.yaml                              |  2 +
 docs/DYNAMIC_WORKER_PROFILING.md                |  5 +++
 .../clj/backtype/storm/daemon/supervisor.clj    | 39 ++++++++++----------
 storm-core/src/clj/backtype/storm/ui/core.clj   | 31 ++++++++++++----
 storm-core/src/jvm/backtype/storm/Config.java   | 15 ++++++++
 storm-core/src/ui/public/component.html         | 12 +++---
 6 files changed, 72 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/d2f9305a/conf/defaults.yaml
----------------------------------------------------------------------
diff --git a/conf/defaults.yaml b/conf/defaults.yaml
index 84babc3..322d386 100644
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@ -145,6 +145,8 @@ worker.heap.memory.mb: 768
 worker.childopts: "-Xmx%HEAP-MEM%m -XX:+PrintGCDetails -Xloggc:artifacts/gc.log -XX:+PrintGCDateStamps
-XX:+PrintGCTimeStamps -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=1M
-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=artifacts/heapdump"
 worker.gc.childopts: ""
 worker.profiler.childopts: "-XX:+UnlockCommercialFeatures -XX:+FlightRecorder"
+worker.profiler.enabled: true
+worker.profiler.command: "flight.bash"
 worker.heartbeat.frequency.secs: 1
 
 # check whether dynamic log levels can be reset from DEBUG to INFO in workers

http://git-wip-us.apache.org/repos/asf/storm/blob/d2f9305a/docs/DYNAMIC_WORKER_PROFILING.md
----------------------------------------------------------------------
diff --git a/docs/DYNAMIC_WORKER_PROFILING.md b/docs/DYNAMIC_WORKER_PROFILING.md
index 727322f..4b55a80 100644
--- a/docs/DYNAMIC_WORKER_PROFILING.md
+++ b/docs/DYNAMIC_WORKER_PROFILING.md
@@ -22,3 +22,8 @@ Click on "My Dump Files" to go the logviewer UI for list of worker specific
dump
 
 ![Dump Files Links for worker](images/dynamic_profiling_debugging_3.png "Dump Files Links
for worker")
 
+Configuration
+-------------
+
+The "worker.profiler.command" can be configured to point to specific pluggable profiler,
heapdump commands. The "worker.profiler.enabled" can be disabled if plugin is not available.
+

http://git-wip-us.apache.org/repos/asf/storm/blob/d2f9305a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index 8fe6eed..e5740cb 100644
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -530,25 +530,23 @@
 (defn java-cmd []
   (jvm-cmd "java"))
 
-(def PROFILE-CMD "flight.bash")
+(defn jmap-dump-cmd [profile-cmd pid target-dir]
+  [profile-cmd pid "jmap" target-dir])
 
-(defn jmap-dump-cmd [pid target-dir]
-  [PROFILE-CMD pid "jmap" target-dir])
+(defn jstack-dump-cmd [profile-cmd pid target-dir]
+  [profile-cmd pid "jstack" target-dir])
 
-(defn jstack-dump-cmd [pid target-dir]
-  [PROFILE-CMD pid "jstack" target-dir])
+(defn jprofile-start [profile-cmd pid]
+  [profile-cmd pid "start"])
 
-(defn jprofile-start [pid]
-  [PROFILE-CMD pid "start" ])
+(defn jprofile-stop [profile-cmd pid target-dir]
+  [profile-cmd pid "stop" target-dir])
 
-(defn jprofile-stop [pid target-dir]
-  [PROFILE-CMD pid "stop" target-dir])
+(defn jprofile-dump [profile-cmd pid workers-artifacts-directory]
+  [profile-cmd pid "dump" workers-artifacts-directory])
 
-(defn jprofile-dump [pid workers-artifacts-directory]
-  [PROFILE-CMD pid "dump" workers-artifacts-directory])
-
-(defn jprofile-jvm-restart [pid]
-  [PROFILE-CMD pid "kill" ])
+(defn jprofile-jvm-restart [profile-cmd pid]
+  [profile-cmd pid "kill"])
 
 (defn- delete-topology-profiler-action [storm-cluster-state storm-id profile-action]
   (log-message "Deleting profiler action.." profile-action)
@@ -587,6 +585,7 @@
             stormid->profiler-actions @(:stormid->profiler-actions supervisor)
             storm-cluster-state (:storm-cluster-state supervisor)
             hostname (:my-hostname supervisor)
+            profile-cmd (conf WORKER-PROFILER-COMMAND)
             new-assignment @(:curr-assignment supervisor)
             assigned-storm-ids (assigned-storm-ids-from-port-assignments new-assignment)]
         (doseq [[storm-id profiler-actions] stormid->profiler-actions]
@@ -605,14 +604,14 @@
                       ;; Until PROFILER_STOP action is invalid, keep launching profiler start
in case worker restarted
                       ;; The profiler plugin script validates if JVM is recording before
starting another recording.
                       command (cond
-                                (= action ProfileAction/JMAP_DUMP) (jmap-dump-cmd worker-pid
target-dir)
-                                (= action ProfileAction/JSTACK_DUMP) (jstack-dump-cmd worker-pid
target-dir)
-                                (= action ProfileAction/JPROFILE_DUMP) (jprofile-dump worker-pid
target-dir)
-                                (= action ProfileAction/JVM_RESTART) (jprofile-jvm-restart
worker-pid)
+                                (= action ProfileAction/JMAP_DUMP) (jmap-dump-cmd profile-cmd
worker-pid target-dir)
+                                (= action ProfileAction/JSTACK_DUMP) (jstack-dump-cmd profile-cmd
worker-pid target-dir)
+                                (= action ProfileAction/JPROFILE_DUMP) (jprofile-dump profile-cmd
worker-pid target-dir)
+                                (= action ProfileAction/JVM_RESTART) (jprofile-jvm-restart
profile-cmd worker-pid)
                                 (and (not stop?)
                                      (= action ProfileAction/JPROFILE_STOP))
-                                  (jprofile-start worker-pid) ;; Ensure the profiler is still
running
-                                (and stop? (= action ProfileAction/JPROFILE_STOP)) (jprofile-stop
worker-pid target-dir))
+                                  (jprofile-start profile-cmd worker-pid) ;; Ensure the profiler
is still running
+                                (and stop? (= action ProfileAction/JPROFILE_STOP)) (jprofile-stop
profile-cmd worker-pid target-dir))
                       action-on-exit (fn [exit-code]
                                        (log-message log-prefix " profile-action exited for
code: " exit-code)
                                        (if (and (= exit-code 0) stop?)

http://git-wip-us.apache.org/repos/asf/storm/blob/d2f9305a/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/backtype/storm/ui/core.clj b/storm-core/src/clj/backtype/storm/ui/core.clj
index c6ffe22..60404f0 100644
--- a/storm-core/src/clj/backtype/storm/ui/core.clj
+++ b/storm-core/src/clj/backtype/storm/ui/core.clj
@@ -83,6 +83,14 @@
            (throw (AuthorizationException.
                    (str "UI request '" op "' for '" user "' user is not authorized")))))))))
 
+
+(defn assert-authorized-profiler-action
+  [op]
+  (if-not (*STORM-CONF* WORKER-PROFILER-ENABLED)
+    (throw (AuthorizationException.
+             (str "UI request for profiler action '" op "' is disabled.")))))
+
+
 (defn executor-summary-type
   [topology ^ExecutorSummary s]
   (component-type topology (.get_component_id s)))
@@ -799,7 +807,10 @@
                                       (.get_eventlog_host comp-page-info)
                                       (.get_eventlog_port comp-page-info)
                                       secure?)
-       "profilerActive" (get-active-profile-actions nimbus topology-id component)))))
+       "profileActionEnabled" (*STORM-CONF* WORKER-PROFILER-ENABLED)
+       "profilerActive" (if (*STORM-CONF* WORKER-PROFILER-ENABLED)
+                          (get-active-profile-actions nimbus topology-id component)
+                          [])))))
     
 (defn- level-to-dict [level]
   (if level
@@ -1017,7 +1028,8 @@
          (let [user (.getUserName http-creds-handler servlet-request)
                topology-conf (from-json
                               (.getTopologyConf ^Nimbus$Client nimbus id))]
-           (assert-authorized-user "setWorkerProfiler" (topology-config id)))
+           (assert-authorized-user "setWorkerProfiler" (topology-config id))
+           (assert-authorized-profiler-action "start"))
 
          (let [[host, port] (split host-port #":")
                nodeinfo (NodeInfo. host (set [(Long. port)]))
@@ -1041,7 +1053,8 @@
          (let [user (.getUserName http-creds-handler servlet-request)
                topology-conf (from-json
                               (.getTopologyConf ^Nimbus$Client nimbus id))]
-           (assert-authorized-user "setWorkerProfiler" (topology-config id)))
+           (assert-authorized-user "setWorkerProfiler" (topology-config id))
+           (assert-authorized-profiler-action "stop"))
          (let [[host, port] (split host-port #":")
                nodeinfo (NodeInfo. host (set [(Long. port)]))
                timestamp 0
@@ -1059,7 +1072,8 @@
          (let [user (.getUserName http-creds-handler servlet-request)
                topology-conf (from-json
                               (.getTopologyConf ^Nimbus$Client nimbus id))]
-           (assert-authorized-user "setWorkerProfiler" (topology-config id)))
+           (assert-authorized-user "setWorkerProfiler" (topology-config id))
+           (assert-authorized-profiler-action "dumpprofile"))
          (let [[host, port] (split host-port #":")
                nodeinfo (NodeInfo. host (set [(Long. port)]))
                timestamp (System/currentTimeMillis)
@@ -1077,7 +1091,8 @@
          (let [user (.getUserName http-creds-handler servlet-request)
                topology-conf (from-json
                               (.getTopologyConf ^Nimbus$Client nimbus id))]
-           (assert-authorized-user "setWorkerProfiler" (topology-config id)))
+           (assert-authorized-user "setWorkerProfiler" (topology-config id))
+           (assert-authorized-profiler-action "dumpjstack"))
          (let [[host, port] (split host-port #":")
                nodeinfo (NodeInfo. host (set [(Long. port)]))
                timestamp (System/currentTimeMillis)
@@ -1095,7 +1110,8 @@
          (let [user (.getUserName http-creds-handler servlet-request)
                topology-conf (from-json
                               (.getTopologyConf ^Nimbus$Client nimbus id))]
-           (assert-authorized-user "setWorkerProfiler" (topology-config id)))
+           (assert-authorized-user "setWorkerProfiler" (topology-config id))
+           (assert-authorized-profiler-action "restartworker"))
          (let [[host, port] (split host-port #":")
                nodeinfo (NodeInfo. host (set [(Long. port)]))
                timestamp (System/currentTimeMillis)
@@ -1113,7 +1129,8 @@
          (let [user (.getUserName http-creds-handler servlet-request)
                topology-conf (from-json
                               (.getTopologyConf ^Nimbus$Client nimbus id))]
-           (assert-authorized-user "setWorkerProfiler" (topology-config id)))
+           (assert-authorized-user "setWorkerProfiler" (topology-config id))
+           (assert-authorized-profiler-action "dumpheap"))
          (let [[host, port] (split host-port #":")
                nodeinfo (NodeInfo. host (set [(Long. port)]))
                timestamp (System/currentTimeMillis)

http://git-wip-us.apache.org/repos/asf/storm/blob/d2f9305a/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java
index 303a3a0..ee6de1b 100644
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@ -1093,6 +1093,21 @@ public class Config extends HashMap<String, Object> {
     public static final String WORKER_PROFILER_CHILDOPTS = "worker.profiler.childopts";
 
     /**
+     * This configuration would enable or disable component page profiing and debugging for
workers.
+     */
+    @isBoolean
+    public static final String WORKER_PROFILER_ENABLED = "worker.profiler.enabled";
+
+    /**
+     * The command launched supervisor with worker arguments
+     * pid, action and [target_directory]
+     * Where action is - start profile, stop profile, jstack, heapdump and kill against pid
+     *
+     */
+    @isString
+    public static final String WORKER_PROFILER_COMMAND = "worker.profiler.command";
+
+    /**
      * The jvm opts provided to workers launched by this supervisor for GC. All "%ID%" substrings
are replaced
      * with an identifier for this worker.  Because the JVM complains about multiple GC opts
the topology
      * can override this default value by setting topology.worker.gc.childopts.

http://git-wip-us.apache.org/repos/asf/storm/blob/d2f9305a/storm-core/src/ui/public/component.html
----------------------------------------------------------------------
diff --git a/storm-core/src/ui/public/component.html b/storm-core/src/ui/public/component.html
index 60a85cf..ae2463c 100644
--- a/storm-core/src/ui/public/component.html
+++ b/storm-core/src/ui/public/component.html
@@ -166,11 +166,13 @@ $(document).ready(function() {
               componentActions.append(Mustache.render($(template).filter("#component-actions-template").html(),buttonJsonData));
             });
 
-            jsError(function() {
-                var part = $(template).filter('#profiler-active-partial').html();
-                var partials = {"profilerActive": part};
-                profilerControl.append(Mustache.render($(template).filter("#profiling-template").html(),
response, partials));
-            });
+            if(response["profileActionEnabled"] == true) {
+                jsError(function () {
+                    var part = $(template).filter('#profiler-active-partial').html();
+                    var partials = {"profilerActive": part};
+                    profilerControl.append(Mustache.render($(template).filter("#profiling-template").html(),
response, partials));
+                });
+            }
 
             if(response["componentType"] == "spout") {
                 componentStatsDetail.append(Mustache.render($(template).filter("#spout-stats-detail-template").html(),response));


Mime
View raw message