storm-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bo...@apache.org
Subject [06/10] storm git commit: Merge remote-tracking branch 'upstream/master' into STORM-1155
Date Wed, 11 Nov 2015 21:24:11 GMT
Merge remote-tracking branch 'upstream/master' into STORM-1155

Conflicts:
	storm-core/src/clj/backtype/storm/daemon/supervisor.clj


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/b3521c57
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/b3521c57
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/b3521c57

Branch: refs/heads/master
Commit: b3521c577bef6654dcf90269c4639064c3ae60eb
Parents: be9799e 316b23e
Author: Thomas Graves <tgraves@decadefade.corp.ne1.yahoo.com>
Authored: Thu Nov 5 19:08:45 2015 +0000
Committer: Thomas Graves <tgraves@decadefade.corp.ne1.yahoo.com>
Committed: Thu Nov 5 19:08:45 2015 +0000

----------------------------------------------------------------------
 CHANGELOG.md                                    |    5 +
 DISCLAIMER                                      |   10 -
 STORM-UI-REST-API.md                            |  735 -------
 bin/flight.bash                                 |  139 ++
 bin/storm.py                                    |   14 +-
 conf/defaults.yaml                              |    3 +
 docs/DYNAMIC_LOG_LEVEL_SETTINGS.md              |   41 -
 docs/documentation/Documentation.md             |    3 +
 .../Message-passing-implementation.md           |   34 +-
 .../documentation/dynamic-log-level-settings.md |   41 +
 docs/documentation/dynamic-worker-profiling.md  |   29 +
 .../images/dynamic_log_level_settings_1.png     |  Bin 0 -> 93689 bytes
 .../images/dynamic_log_level_settings_2.png     |  Bin 0 -> 78785 bytes
 .../images/dynamic_profiling_debugging_1.png    |  Bin 0 -> 93635 bytes
 .../images/dynamic_profiling_debugging_2.png    |  Bin 0 -> 138120 bytes
 .../images/dynamic_profiling_debugging_3.png    |  Bin 0 -> 96974 bytes
 docs/documentation/ui-rest-api.md               |  930 ++++++++
 docs/images/dynamic_log_level_settings_1.png    |  Bin 93689 -> 0 bytes
 docs/images/dynamic_log_level_settings_2.png    |  Bin 78785 -> 0 bytes
 external/storm-kafka/README.md                  |   11 +
 .../jvm/storm/kafka/DynamicBrokersReader.java   |   97 +-
 .../kafka/DynamicPartitionConnections.java      |   20 +-
 .../src/jvm/storm/kafka/KafkaSpout.java         |    2 +-
 .../src/jvm/storm/kafka/KafkaUtils.java         |   85 +-
 .../src/jvm/storm/kafka/Partition.java          |   26 +-
 .../src/jvm/storm/kafka/PartitionManager.java   |   18 +-
 .../src/jvm/storm/kafka/StaticCoordinator.java  |   11 +-
 .../storm/kafka/StringMultiSchemeWithTopic.java |   57 +
 .../src/jvm/storm/kafka/ZkCoordinator.java      |    2 +-
 .../jvm/storm/kafka/trident/Coordinator.java    |    7 +-
 .../trident/GlobalPartitionInformation.java     |   26 +-
 .../jvm/storm/kafka/trident/IBrokerReader.java  |    7 +-
 .../kafka/trident/OpaqueTridentKafkaSpout.java  |    5 +-
 .../storm/kafka/trident/StaticBrokerReader.java |   23 +-
 .../kafka/trident/TridentKafkaEmitter.java      |   36 +-
 .../jvm/storm/kafka/trident/ZkBrokerReader.java |   20 +-
 .../storm/kafka/DynamicBrokersReaderTest.java   |  114 +-
 .../src/test/storm/kafka/KafkaUtilsTest.java    |   58 +-
 .../src/test/storm/kafka/TestUtils.java         |   13 +-
 .../src/test/storm/kafka/ZkCoordinatorTest.java |    8 +-
 .../test/storm/kafka/bolt/KafkaBoltTest.java    |    4 +-
 storm-core/src/clj/backtype/storm/cluster.clj   |   57 +-
 storm-core/src/clj/backtype/storm/config.clj    |    4 +
 storm-core/src/clj/backtype/storm/converter.clj |   19 +-
 .../src/clj/backtype/storm/daemon/logviewer.clj |   45 +
 .../src/clj/backtype/storm/daemon/nimbus.clj    |   35 +-
 .../clj/backtype/storm/daemon/supervisor.clj    |  155 +-
 .../src/clj/backtype/storm/daemon/worker.clj    |    4 +-
 storm-core/src/clj/backtype/storm/stats.clj     |    9 +
 storm-core/src/clj/backtype/storm/ui/core.clj   |  166 +-
 storm-core/src/clj/backtype/storm/util.clj      |   10 +-
 storm-core/src/jvm/backtype/storm/Config.java   |   21 +
 .../src/jvm/backtype/storm/StormSubmitter.java  |   20 +-
 .../storm/coordination/BatchBoltExecutor.java   |    4 +-
 .../storm/coordination/CoordinatedBolt.java     |   14 +-
 .../src/jvm/backtype/storm/drpc/DRPCSpout.java  |   10 +-
 .../src/jvm/backtype/storm/drpc/JoinResult.java |    8 +-
 .../jvm/backtype/storm/generated/Nimbus.java    | 1977 +++++++++++++++++-
 .../backtype/storm/generated/ProfileAction.java |   74 +
 .../storm/generated/ProfileRequest.java         |  631 ++++++
 .../security/auth/ShellBasedGroupsMapping.java  |   10 +-
 .../auth/authorizer/DRPCAuthorizerBase.java     |    2 +-
 .../auth/authorizer/SimpleACLAuthorizer.java    |    8 +
 .../authorizer/SimpleWhitelistAuthorizer.java   |    7 +-
 .../storm/security/auth/kerberos/AutoTGT.java   |   10 +-
 .../serialization/BlowfishTupleSerializer.java  |    6 +-
 .../jvm/backtype/storm/spout/ShellSpout.java    |    4 +-
 .../src/jvm/backtype/storm/task/ShellBolt.java  |   42 +-
 .../storm/testing/MemoryTransactionalSpout.java |    9 +-
 .../testing/OpaqueMemoryTransactionalSpout.java |    8 +-
 .../storm/testing/TupleCaptureBolt.java         |    4 +-
 .../TransactionalSpoutBatchExecutor.java        |    4 +-
 .../src/jvm/backtype/storm/tuple/TupleImpl.java |    6 +-
 .../jvm/backtype/storm/utils/LocalState.java    |   22 +-
 .../jvm/backtype/storm/utils/ShellProcess.java  |    6 +-
 .../jvm/backtype/storm/utils/ShellUtils.java    |    2 +-
 .../src/jvm/backtype/storm/utils/Time.java      |    4 +-
 .../storm/validation/ConfigValidation.java      |   83 +-
 .../trident/spout/TridentSpoutExecutor.java     |    2 +-
 .../trident/topology/TridentBoltExecutor.java   |    6 +-
 .../storm/trident/tuple/TridentTupleView.java   |   12 +-
 .../src/native/worker-launcher/impl/main.c      |   10 +
 .../worker-launcher/impl/worker-launcher.c      |   47 +
 .../worker-launcher/impl/worker-launcher.h      |    2 +
 storm-core/src/py/storm/Nimbus-remote           |   14 +
 storm-core/src/py/storm/Nimbus.py               |  396 ++++
 storm-core/src/py/storm/ttypes.py               |  122 ++
 storm-core/src/storm.thrift                     |   20 +
 storm-core/src/ui/public/component.html         |  167 +-
 .../templates/component-page-template.html      |   53 +
 90 files changed, 5819 insertions(+), 1166 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/b3521c57/bin/storm.py
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/b3521c57/conf/defaults.yaml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/b3521c57/storm-core/src/clj/backtype/storm/config.clj
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/b3521c57/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index 64cbfd6,c86b73f..118e2c2
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@@ -26,9 -26,8 +26,9 @@@
    (:use [backtype.storm config util log timer local-state])
    (:import [backtype.storm.utils VersionInfo])
    (:import [backtype.storm Config])
-   (:import [backtype.storm.generated WorkerResources])
+   (:import [backtype.storm.generated WorkerResources ProfileAction])
    (:use [backtype.storm.daemon common])
 +  (:require [backtype.storm.command [healthcheck :as healthcheck]])
    (:require [backtype.storm.daemon [worker :as worker]]
              [backtype.storm [process-simulator :as psim] [cluster :as cluster] [event :as
event]]
              [clojure.set :as set])
@@@ -547,16 -674,11 +675,21 @@@
                            0
                            (conf SUPERVISOR-MONITOR-FREQUENCY-SECS)
                            (fn [] (.add processes-event-manager sync-processes)))
 +      (schedule-recurring (:event-timer supervisor)
 +                          (* 60 5)
 +                          (* 60 5)
 +                          (fn [] (let [health-code (healthcheck/health-check conf)
 +                                       ids (my-worker-ids conf)]
 +                                   (if (not (= health-code 0))
 +                                     (do
 +                                       (doseq [id ids]
 +                                         (shutdown-worker supervisor id))
 +                                       (throw (RuntimeException. "Supervisor failed health
check. Exiting."))))))))
+       ;; Launch a thread that Runs profiler commands . Starts with 30 seconds delay, every
30 seconds
+       (schedule-recurring (:event-timer supervisor)
+                           30
+                           30
+                           (fn [] (.add event-manager run-profiler-actions-fn))))
      (log-message "Starting supervisor with id " (:supervisor-id supervisor) " at host "
(:my-hostname supervisor))
      (reify
       Shutdownable

http://git-wip-us.apache.org/repos/asf/storm/blob/b3521c57/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------


Mime
View raw message