storm-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bo...@apache.org
Subject [16/50] [abbrv] storm git commit: Merge remote-tracking branch 'upstream/master' into STORM-166
Date Mon, 24 Aug 2015 13:52:01 GMT
Merge remote-tracking branch 'upstream/master' into STORM-166

Conflicts:
	storm-core/src/jvm/backtype/storm/Config.java


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/58667be9
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/58667be9
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/58667be9

Branch: refs/heads/master
Commit: 58667be951623848406f699f82cafa79a40cb496
Parents: 5658bbb e71e2a3
Author: Parth Brahmbhatt <brahmbhatt.parth@gmail.com>
Authored: Thu Jan 8 11:59:22 2015 -0500
Committer: Parth Brahmbhatt <brahmbhatt.parth@gmail.com>
Committed: Thu Jan 8 11:59:22 2015 -0500

----------------------------------------------------------------------
 CHANGELOG.md                                    |  14 +
 DEVELOPER.md                                    |   5 +-
 README.markdown                                 |  18 +-
 SECURITY.md                                     |  25 +-
 bin/storm                                       |  59 ++--
 conf/defaults.yaml                              |   1 +
 docs/about/integrates.md                        |   2 +-
 docs/about/multi-language.md                    |   6 +-
 docs/about/simple-api.md                        |   2 +-
 .../Acking-framework-implementation.md          |   8 +-
 docs/documentation/Clojure-DSL.md               |   6 +-
 docs/documentation/Common-patterns.md           |   2 +-
 docs/documentation/Configuration.md             |   4 +-
 .../Creating-a-new-Storm-project.md             |   6 +-
 .../DSLs-and-multilang-adapters.md              |   3 +-
 ...Defining-a-non-jvm-language-dsl-for-storm.md |   2 +-
 docs/documentation/Distributed-RPC.md           |   2 +-
 docs/documentation/FAQ.md                       |   4 +-
 docs/documentation/Kestrel-and-Storm.md         |   2 +-
 docs/documentation/Lifecycle-of-a-topology.md   |  70 ++---
 docs/documentation/Maven.md                     |  50 +---
 docs/documentation/Multilang-protocol.md        |  30 +-
 .../Serialization-(prior-to-0.6.0).md           |   2 +-
 .../documentation/Setting-up-a-Storm-cluster.md |   4 +-
 .../Setting-up-development-environment.md       |   2 +-
 docs/documentation/Structure-of-the-codebase.md |  88 +++---
 docs/documentation/Transactional-topologies.md  |  12 +-
 docs/documentation/Trident-API-Overview.md      |   4 +-
 docs/documentation/Trident-spouts.md            |   8 +-
 docs/documentation/Trident-state.md             |  12 +-
 docs/documentation/Tutorial.md                  |   4 +-
 docs/downloads.html                             |   2 +-
 external/storm-hbase/README.md                  |  49 +++-
 external/storm-hbase/pom.xml                    |  17 +-
 .../storm/hbase/bolt/AbstractHBaseBolt.java     |   9 +-
 .../apache/storm/hbase/security/AutoHBase.java  | 243 ++++++++++++++++
 .../storm/hbase/security/HBaseSecurityUtil.java |  32 ++-
 .../storm/hbase/trident/state/HBaseState.java   |   9 +-
 external/storm-hdfs/README.md                   |  45 +++
 .../storm/hdfs/bolt/format/SequenceFormat.java  |   5 +-
 .../storm/hdfs/common/security/AutoHDFS.java    | 281 +++++++++++++++++++
 .../hdfs/common/security/HdfsSecurityUtil.java  |  30 +-
 .../src/jvm/storm/kafka/KafkaUtils.java         |  12 +-
 .../src/jvm/storm/kafka/PartitionManager.java   |   6 +-
 .../kafka/TopicOffsetOutOfRangeException.java   |  25 ++
 .../jvm/storm/kafka/UpdateOffsetException.java  |  22 --
 .../FieldNameBasedTupleToKafkaMapper.java       |   2 +-
 .../kafka/trident/TridentKafkaEmitter.java      |  43 ++-
 .../src/test/storm/kafka/KafkaUtilsTest.java    |   2 +-
 .../src/clj/backtype/storm/daemon/logviewer.clj |  36 ++-
 .../src/clj/backtype/storm/daemon/nimbus.clj    |   7 +-
 .../clj/backtype/storm/daemon/supervisor.clj    |  22 +-
 storm-core/src/clj/backtype/storm/ui/core.clj   |   1 +
 storm-core/src/jvm/backtype/storm/Config.java   |  38 +--
 .../storm/security/auth/hadoop/AutoHDFS.java    | 262 -----------------
 .../jvm/backtype/storm/utils/ShellProcess.java  |  11 +-
 .../src/native/worker-launcher/configure.ac     |   2 +-
 .../worker-launcher/impl/worker-launcher.c      |   7 +-
 .../test/clj/backtype/storm/logviewer_test.clj  |  47 +++-
 .../backtype/storm/security/auth/auth_test.clj  |   8 +-
 .../test/clj/backtype/storm/supervisor_test.clj |   6 +-
 61 files changed, 1108 insertions(+), 630 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/58667be9/conf/defaults.yaml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/58667be9/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/58667be9/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/daemon/supervisor.clj
index c96dd88,9d2f313..3fcf7eb
--- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@@ -296,9 -294,12 +296,13 @@@
                                 (log-error t "Error when processing event")
                                 (exit-process! 20 "Error when processing an event")
                                 ))
+    :event-timer (mk-timer :kill-fn (fn [t]
+                                          (log-error t "Error when processing event")
+                                          (exit-process! 20 "Error when processing an event")
+                                          ))
     :assignment-versions (atom {})
     :sync-retry (atom 0)
 +   :bt-tracker (mk-bt-tracker conf)
     })
  
  (defn sync-processes [supervisor]

http://git-wip-us.apache.org/repos/asf/storm/blob/58667be9/storm-core/src/clj/backtype/storm/ui/core.clj
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/58667be9/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --cc storm-core/src/jvm/backtype/storm/Config.java
index 4678177,3a241ba..df330a0
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@@ -1234,54 -1246,6 +1240,36 @@@ public class Config extends HashMap<Str
      public static final String TOPOLOGY_ISOLATED_MACHINES = "topology.isolate.machines";
      public static final Object TOPOLOGY_ISOLATED_MACHINES_SCHEMA = Number.class;
  
 +    /**
-      * HDFS information, used to get the delegation token on behalf of the topology
-      * submitter user and renew the tokens. see {@link backtype.storm.security.auth.hadoop.AutoHDFS}
-      * kerberos principal name with realm should be provided.
-      */
-     public static final Object TOPOLOGY_HDFS_PRINCIPAL = "topology.hdfs.user";
-     public static final Object TOPOLOGY_HDFS_PRINCIPAL_SCHEMA = String.class;
- 
-     /**
-      * The HDFS URI to be used by AutoHDFS.java to grab the delegation token on topology
-      * submitter user's behalf by the nimbus. If this is not provided the default URI provided
-      * in the hdfs configuration files will be used.
-      */
-     public static final Object TOPOLOGY_HDFS_URI = "topology.hdfs.uri";
-     public static final Object TOPOLOGY_HDFS_URI_SCHEMA = String.class;
- 
-     /**
 +     * Which implementation of {@link backtype.storm.codedistributor.ICodeDistributor} should
be used by storm for code
 +     * distribution.
 +     */
 +    public static final String STORM_CODE_DISTRIBUTOR_CLASS = "storm.codedistributor.class";
 +    public static final Object STORM_CODE_DISTRIBUTOR_CLASS_SCHEMA = String.class;
 +
 +    /**
 +     * Minimum number of nimbus hosts where the code must be replicated before leader nimbus
 +     * is allowed to perform topology activation tasks like setting up heartbeats/assignments
 +     * and marking the topology as active. default is 0.
 +     */
 +    public static final String NIMBUS_MIN_REPLICATION_COUNT = "nimbus.min.replication.count";
 +    public static final Object NIMBUS_MIN_REPLICATION_COUNT_SCHEMA = Number.class;
 +
 +    /**
 +     * Maximum wait time for the nimbus host replication to achieve the nimbus.min.replication.count.
 +     * Once this time is elapsed nimbus will go ahead and perform topology activation tasks
even
 +     * if required nimbus.min.replication.count is not achieved. The default is 0 seconds,
a value of
 +     * -1 indicates to wait for ever.
 +     */
 +    public static final String NIMBUS_MAX_REPLICATION_WAIT_TIME_SEC = "nimbus.max.replication.wait.time.sec";
 +    public static final Object NIMBUS_MAX_REPLICATION_WAIT_TIME_SEC_SCHEMA = Number.class;
 +
 +    /**
-      * How often nimbus should wake the cleanup thread to clean the inbox.
-      * @see NIMBUS_INBOX_JAR_EXPIRATION_SECS
++     * How often nimbus's background thread to sync code for missing topologies should run.
 +     */
 +    public static final String NIMBUS_CODE_SYNC_FREQ_SECS = "nimbus.code.sync.freq.secs";
 +    public static final Object NIMBUS_CODE_SYNC_FREQ_SECS_SCHEMA = ConfigValidation.IntegerValidator;
 +
- 
      public static void setClasspath(Map conf, String cp) {
          conf.put(Config.TOPOLOGY_CLASSPATH, cp);
      }

http://git-wip-us.apache.org/repos/asf/storm/blob/58667be9/storm-core/test/clj/backtype/storm/security/auth/auth_test.clj
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/58667be9/storm-core/test/clj/backtype/storm/supervisor_test.clj
----------------------------------------------------------------------


Mime
View raw message