storm-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bo...@apache.org
Subject [08/16] storm git commit: Merge remote-tracking branch 'upstream/master' into ru
Date Wed, 18 Mar 2015 19:39:15 GMT
Merge remote-tracking branch 'upstream/master' into ru

Conflicts:
	storm-core/src/clj/backtype/storm/bootstrap.clj
	storm-core/src/clj/backtype/storm/config.clj
	storm-core/src/clj/backtype/storm/daemon/executor.clj
	storm-core/src/clj/backtype/storm/daemon/nimbus.clj
	storm-core/src/jvm/backtype/storm/generated/ExecutorStats.java
	storm-core/src/py/storm/ttypes.py


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/5b67b571
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/5b67b571
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/5b67b571

Branch: refs/heads/master
Commit: 5b67b571fe45e14fd52b835bc1de056538de1e70
Parents: f23b3c8 5eff2e7
Author: Parth Brahmbhatt <brahmbhatt.parth@gmail.com>
Authored: Thu Mar 5 17:05:33 2015 -0800
Committer: Parth Brahmbhatt <brahmbhatt.parth@gmail.com>
Committed: Thu Mar 5 17:05:33 2015 -0800

----------------------------------------------------------------------
 .gitignore                                      |    4 +-
 BYLAWS.md                                       |   31 +-
 CHANGELOG.md                                    |   19 +
 DEVELOPER.md                                    |   24 +-
 README.markdown                                 |    3 +
 SECURITY.md                                     |   10 +-
 STORM-UI-REST-API.md                            |   45 +-
 bin/storm-config.cmd                            |   14 +-
 bin/storm.cmd                                   |   33 +-
 conf/defaults.yaml                              |    2 +-
 docs/documentation/Powered-By.md                |   83 +
 docs/documentation/Trident-API-Overview.md      |    2 +-
 docs/images/architecture.png                    |  Bin 0 -> 69825 bytes
 docs/images/architecture.svg                    | 1458 +++
 docs/images/bolt.png                            |  Bin 0 -> 24796 bytes
 docs/images/bolt.svg                            |  743 ++
 docs/images/security.png                        |  Bin 0 -> 72415 bytes
 docs/images/security.svg                        | 1779 ++++
 docs/images/spout.png                           |  Bin 0 -> 22911 bytes
 docs/images/spout.svg                           |  833 ++
 docs/images/storm.svg                           | 1326 +++
 docs/images/topology.png                        |  Bin 59837 -> 64740 bytes
 docs/images/topology.svg                        | 1044 ++
 docs/images/topology_dark.png                   |  Bin 0 -> 49692 bytes
 docs/images/topology_dark.svg                   | 1101 +++
 .../jvm/storm/starter/BasicDRPCTopology.java    |    3 +-
 external/storm-hdfs/pom.xml                     |   18 +-
 .../storm/hdfs/bolt/HdfsFileTopology.java       |    6 +-
 .../storm/hdfs/bolt/SequenceFileTopology.java   |    4 +-
 .../storm/hdfs/trident/TridentFileTopology.java |    2 +-
 .../hdfs/trident/TridentSequenceTopology.java   |    6 +-
 external/storm-hive/README.md                   |  113 +
 external/storm-hive/pom.xml                     |  143 +
 .../org/apache/storm/hive/bolt/HiveBolt.java    |  289 +
 .../bolt/mapper/DelimitedRecordHiveMapper.java  |  143 +
 .../storm/hive/bolt/mapper/HiveMapper.java      |   81 +
 .../hive/bolt/mapper/JsonRecordHiveMapper.java  |  132 +
 .../apache/storm/hive/common/HiveOptions.java   |  146 +
 .../org/apache/storm/hive/common/HiveUtils.java |   76 +
 .../apache/storm/hive/common/HiveWriter.java    |  420 +
 .../apache/storm/hive/trident/HiveState.java    |  306 +
 .../storm/hive/trident/HiveStateFactory.java    |   31 +
 .../apache/storm/hive/trident/HiveUpdater.java  |   14 +
 .../apache/storm/hive/bolt/HiveSetupUtil.java   |  220 +
 .../apache/storm/hive/bolt/HiveTopology.java    |  150 +
 .../hive/bolt/HiveTopologyPartitioned.java      |  153 +
 .../apache/storm/hive/bolt/TestHiveBolt.java    |  330 +
 .../storm/hive/common/TestHiveWriter.java       |  193 +
 .../storm/hive/trident/TridentHiveTopology.java |  190 +
 external/storm-jdbc/LICENSE                     |  202 +
 external/storm-jdbc/README.md                   |  240 +
 external/storm-jdbc/pom.xml                     |  125 +
 .../storm/jdbc/bolt/AbstractJdbcBolt.java       |   57 +
 .../apache/storm/jdbc/bolt/JdbcInsertBolt.java  |   71 +
 .../apache/storm/jdbc/bolt/JdbcLookupBolt.java  |   76 +
 .../org/apache/storm/jdbc/common/Column.java    |  111 +
 .../apache/storm/jdbc/common/JdbcClient.java    |  228 +
 .../java/org/apache/storm/jdbc/common/Util.java |   75 +
 .../storm/jdbc/mapper/JdbcLookupMapper.java     |   26 +
 .../apache/storm/jdbc/mapper/JdbcMapper.java    |   33 +
 .../jdbc/mapper/SimpleJdbcLookupMapper.java     |   46 +
 .../storm/jdbc/mapper/SimpleJdbcMapper.java     |   92 +
 .../storm/jdbc/trident/state/JdbcQuery.java     |   40 +
 .../storm/jdbc/trident/state/JdbcState.java     |  145 +
 .../jdbc/trident/state/JdbcStateFactory.java    |   40 +
 .../storm/jdbc/trident/state/JdbcUpdater.java   |   32 +
 .../storm/jdbc/common/JdbcClientTest.java       |   88 +
 .../org/apache/storm/jdbc/common/UtilTest.java  |   69 +
 .../org/apache/storm/jdbc/spout/UserSpout.java  |   90 +
 .../jdbc/topology/AbstractUserTopology.java     |  106 +
 .../jdbc/topology/UserPersistanceTopology.java  |   48 +
 .../UserPersistanceTridentTopology.java         |   61 +
 external/storm-jdbc/src/test/sql/test.sql       |    1 +
 .../src/jvm/storm/kafka/PartitionManager.java   |    5 +-
 external/storm-redis/LICENSE                    |  202 +
 external/storm-redis/README.md                  |  137 +
 external/storm-redis/pom.xml                    |   65 +
 .../storm/redis/bolt/AbstractRedisBolt.java     |   67 +
 .../trident/mapper/TridentTupleMapper.java      |   27 +
 .../trident/state/RedisClusterMapState.java     |  294 +
 .../redis/trident/state/RedisClusterState.java  |   80 +
 .../trident/state/RedisClusterStateQuerier.java |   78 +
 .../trident/state/RedisClusterStateUpdater.java |   76 +
 .../redis/trident/state/RedisMapState.java      |  323 +
 .../storm/redis/trident/state/RedisState.java   |   83 +
 .../redis/trident/state/RedisStateQuerier.java  |   70 +
 .../state/RedisStateSetCountQuerier.java        |   74 +
 .../trident/state/RedisStateSetUpdater.java     |   80 +
 .../redis/trident/state/RedisStateUpdater.java  |   75 +
 .../redis/util/config/JedisClusterConfig.java   |   82 +
 .../redis/util/config/JedisPoolConfig.java      |   97 +
 .../util/container/JedisClusterContainer.java   |   47 +
 .../JedisCommandsContainerBuilder.java          |   38 +
 .../JedisCommandsInstanceContainer.java         |   25 +
 .../redis/util/container/JedisContainer.java    |   65 +
 .../storm/redis/topology/LookupWordCount.java   |  127 +
 .../redis/topology/PersistentWordCount.java     |  117 +
 .../storm/redis/topology/WordCounter.java       |   58 +
 .../apache/storm/redis/topology/WordSpout.java  |   88 +
 .../storm/redis/trident/PrintFunction.java      |   40 +
 .../redis/trident/WordCountTridentRedis.java    |   97 +
 .../trident/WordCountTridentRedisCluster.java   |  103 +
 .../WordCountTridentRedisClusterMap.java        |  101 +
 .../redis/trident/WordCountTridentRedisMap.java |   95 +
 .../redis/trident/WordCountTupleMapper.java     |   16 +
 pom.xml                                         |   16 +-
 storm-core/pom.xml                              |   13 +-
 storm-core/src/clj/backtype/storm/bootstrap.clj |   64 -
 storm-core/src/clj/backtype/storm/clojure.clj   |    2 +-
 storm-core/src/clj/backtype/storm/config.clj    |    3 +-
 .../src/clj/backtype/storm/daemon/drpc.clj      |    6 +-
 .../src/clj/backtype/storm/daemon/executor.clj  |   30 +-
 .../src/clj/backtype/storm/daemon/nimbus.clj    |   29 +-
 .../clj/backtype/storm/daemon/supervisor.clj    |  191 +-
 .../src/clj/backtype/storm/daemon/task.clj      |   20 +-
 .../src/clj/backtype/storm/daemon/worker.clj    |   70 +-
 .../src/clj/backtype/storm/messaging/local.clj  |    2 +-
 storm-core/src/clj/backtype/storm/tuple.clj     |    4 +-
 storm-core/src/clj/backtype/storm/ui/core.clj   |   63 +-
 .../src/clj/backtype/storm/ui/helpers.clj       |    1 +
 storm-core/src/clj/backtype/storm/util.clj      |    3 +
 .../storm/drpc/DRPCInvocationsClient.java       |    6 +
 .../src/jvm/backtype/storm/drpc/DRPCSpout.java  |   14 +-
 .../jvm/backtype/storm/drpc/ReturnResults.java  |    8 +-
 .../storm/generated/AlreadyAliveException.java  |  149 +-
 .../storm/generated/AuthorizationException.java |  149 +-
 .../src/jvm/backtype/storm/generated/Bolt.java  |  194 +-
 .../jvm/backtype/storm/generated/BoltStats.java |  908 +-
 .../storm/generated/ClusterSummary.java         |  339 +-
 .../storm/generated/ComponentCommon.java        |  426 +-
 .../storm/generated/ComponentObject.java        |   86 +-
 .../backtype/storm/generated/Credentials.java   |  220 +-
 .../storm/generated/DRPCExecutionException.java |  149 +-
 .../backtype/storm/generated/DRPCRequest.java   |  185 +-
 .../storm/generated/DistributedRPC.java         |  529 +-
 .../generated/DistributedRPCInvocations.java    | 1199 ++-
 .../jvm/backtype/storm/generated/ErrorInfo.java |  300 +-
 .../backtype/storm/generated/ExecutorInfo.java  |  198 +-
 .../storm/generated/ExecutorSpecificStats.java  |   72 +-
 .../backtype/storm/generated/ExecutorStats.java |  609 +-
 .../storm/generated/ExecutorSummary.java        |  371 +-
 .../storm/generated/GetInfoOptions.java         |  166 +-
 .../storm/generated/GlobalStreamId.java         |  185 +-
 .../jvm/backtype/storm/generated/Grouping.java  |  163 +-
 .../generated/InvalidTopologyException.java     |  149 +-
 .../backtype/storm/generated/JavaObject.java    |  239 +-
 .../backtype/storm/generated/JavaObjectArg.java |  108 +-
 .../backtype/storm/generated/KillOptions.java   |  176 +-
 .../jvm/backtype/storm/generated/Nimbus.java    | 9177 +++++++++++++-----
 .../storm/generated/NotAliveException.java      |  149 +-
 .../backtype/storm/generated/NullStruct.java    |  112 +-
 .../storm/generated/NumErrorsChoice.java        |    3 +-
 .../storm/generated/RebalanceOptions.java       |  348 +-
 .../storm/generated/ShellComponent.java         |  202 +-
 .../jvm/backtype/storm/generated/SpoutSpec.java |  194 +-
 .../backtype/storm/generated/SpoutStats.java    |  614 +-
 .../storm/generated/StateSpoutSpec.java         |  194 +-
 .../backtype/storm/generated/StormTopology.java |  410 +-
 .../backtype/storm/generated/StreamInfo.java    |  249 +-
 .../backtype/storm/generated/SubmitOptions.java |  208 +-
 .../storm/generated/SupervisorSummary.java      |  309 +-
 .../backtype/storm/generated/TopologyInfo.java  |  609 +-
 .../storm/generated/TopologyInitialStatus.java  |    3 +-
 .../storm/generated/TopologySummary.java        |  486 +-
 .../storm/messaging/ConnectionWithStatus.java   |   32 +
 .../backtype/storm/messaging/netty/Client.java  |  712 +-
 .../messaging/netty/SaslStormClientHandler.java |    5 +-
 .../backtype/storm/messaging/netty/Server.java  |  182 +-
 .../netty/StormClientPipelineFactory.java       |    5 +-
 .../security/auth/SimpleTransportPlugin.java    |    2 +-
 .../trident/drpc/ReturnResultsReducer.java      |    4 +-
 .../trident/spout/RichSpoutBatchExecutor.java   |    1 +
 storm-core/src/py/storm/DistributedRPC-remote   |   35 +-
 storm-core/src/py/storm/DistributedRPC.py       |   38 +-
 .../py/storm/DistributedRPCInvocations-remote   |   43 +-
 .../src/py/storm/DistributedRPCInvocations.py   |   95 +-
 storm-core/src/py/storm/Nimbus-remote           |  111 +-
 storm-core/src/py/storm/Nimbus.py               |  640 +-
 storm-core/src/py/storm/constants.py            |    6 +-
 storm-core/src/py/storm/ttypes.py               | 1290 +--
 storm-core/src/ui/public/index.html             |    1 +
 .../test/clj/backtype/storm/clojure_test.clj    |    9 +-
 .../test/clj/backtype/storm/drpc_test.clj       |    8 +-
 .../test/clj/backtype/storm/grouping_test.clj   |    9 +-
 .../clj/backtype/storm/integration_test.clj     |   10 +-
 .../storm/messaging/netty_integration_test.clj  |    5 +-
 .../storm/messaging/netty_unit_test.clj         |   73 +-
 .../test/clj/backtype/storm/messaging_test.clj  |    6 +-
 .../test/clj/backtype/storm/metrics_test.clj    |    8 +-
 .../test/clj/backtype/storm/multilang_test.clj  |    6 +-
 .../test/clj/backtype/storm/nimbus_test.clj     |   21 +-
 .../scheduler/multitenant_scheduler_test.clj    |    4 +-
 .../test/clj/backtype/storm/scheduler_test.clj  |    4 +-
 .../storm/security/auth/AuthUtils_test.clj      |    6 +-
 .../backtype/storm/security/auth/auth_test.clj  |   13 +-
 .../storm/security/auth/drpc_auth_test.clj      |    7 +-
 .../storm/security/auth/nimbus_auth_test.clj    |   10 +-
 .../clj/backtype/storm/subtopology_test.clj     |    9 +-
 .../test/clj/backtype/storm/supervisor_test.clj |   16 +-
 .../test/clj/backtype/storm/tick_tuple_test.clj |    7 +-
 .../clj/backtype/storm/transactional_test.clj   |   14 +-
 .../test/clj/backtype/storm/worker_test.clj     |   37 +
 storm-dist/binary/src/main/assembly/binary.xml  |   38 +-
 203 files changed, 32972 insertions(+), 8388 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/5b67b571/conf/defaults.yaml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/5b67b571/storm-core/src/clj/backtype/storm/config.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/config.clj
index 00396c7,483c6ff..5e7893b
--- a/storm-core/src/clj/backtype/storm/config.clj
+++ b/storm-core/src/clj/backtype/storm/config.clj
@@@ -212,8 -211,7 +212,7 @@@
    (let [stormroot (supervisor-stormdist-root conf storm-id)
          conf-path (supervisor-stormconf-path stormroot)
          topology-path (supervisor-stormcode-path stormroot)]
-     (merge conf (Utils/javaDeserialize (FileUtils/readFileToByteArray (File. conf-path))
java.util.Map))
-     ))
 -    (merge conf (clojurify-structure (Utils/deserialize (FileUtils/readFileToByteArray (File.
conf-path)))))))
++    (merge conf (clojurify-structure (Utils/javaDeserialize (FileUtils/readFileToByteArray
(File. conf-path)))))))
  
  (defn read-supervisor-topology
    [conf storm-id]

http://git-wip-us.apache.org/repos/asf/storm/blob/5b67b571/storm-core/src/clj/backtype/storm/daemon/executor.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/daemon/executor.clj
index a1ecc4a,ec68d45..0e083ea
--- a/storm-core/src/clj/backtype/storm/daemon/executor.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/executor.clj
@@@ -15,23 -15,29 +15,31 @@@
  ;; limitations under the License.
  (ns backtype.storm.daemon.executor
    (:use [backtype.storm.daemon common])
-   (:use [backtype.storm bootstrap])
-   (:import [backtype.storm ICredentialsListener]
-            [backtype.storm.generated Grouping]
++  (:import [backtype.storm.generated Grouping]
 +           [java.io Serializable])
+   (:use [backtype.storm util config log timer stats])
+   (:import [java.util List Random HashMap ArrayList LinkedList Map])
+   (:import [backtype.storm ICredentialsListener])
    (:import [backtype.storm.hooks ITaskHook])
-   (:import [backtype.storm.tuple Tuple])
-   (:import [backtype.storm.spout ISpoutWaitStrategy])
+   (:import [backtype.storm.tuple Tuple Fields TupleImpl MessageId])
+   (:import [backtype.storm.spout ISpoutWaitStrategy ISpout SpoutOutputCollector ISpoutOutputCollector])
    (:import [backtype.storm.hooks.info SpoutAckInfo SpoutFailInfo
              EmitInfo BoltFailInfo BoltAckInfo BoltExecuteInfo])
-   (:import [backtype.storm.metric.api IMetric IMetricsConsumer$TaskInfo IMetricsConsumer$DataPoint])
-   (:import [backtype.storm Config])
+   (:import [backtype.storm.grouping CustomStreamGrouping])
+   (:import [backtype.storm.task WorkerTopologyContext IBolt OutputCollector IOutputCollector])
+   (:import [backtype.storm.generated GlobalStreamId])
+   (:import [backtype.storm.utils Utils MutableObject RotatingMap RotatingMap$ExpiredCallback
MutableLong Time])
+   (:import [com.lmax.disruptor InsufficientCapacityException])
+   (:import [backtype.storm.serialization KryoTupleSerializer KryoTupleDeserializer])
+   (:import [backtype.storm.daemon Shutdownable])
+   (:import [backtype.storm.metric.api IMetric IMetricsConsumer$TaskInfo IMetricsConsumer$DataPoint
StateMetric])
+   (:import [backtype.storm Config Constants])
    (:import [java.util.concurrent ConcurrentLinkedQueue])
-   (:require [backtype.storm [tuple :as tuple]])
+   (:require [backtype.storm [tuple :as tuple] [thrift :as thrift]
+              [cluster :as cluster] [disruptor :as disruptor] [stats :as stats]])
    (:require [backtype.storm.daemon [task :as task]])
-   (:require [backtype.storm.daemon.builtin-metrics :as builtin-metrics]))
- 
- (bootstrap)
+   (:require [backtype.storm.daemon.builtin-metrics :as builtin-metrics])
+   (:require [clojure.set :as set]))
  
  (defn- mk-fields-grouper [^Fields out-fields ^Fields group-fields ^List target-tasks]
    (let [num-tasks (count target-tasks)

http://git-wip-us.apache.org/repos/asf/storm/blob/5b67b571/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
----------------------------------------------------------------------
diff --cc storm-core/src/clj/backtype/storm/daemon/nimbus.clj
index d79b542,f1b4082..2c7b82c
--- a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
+++ b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj
@@@ -14,10 -14,14 +14,14 @@@
  ;; See the License for the specific language governing permissions and
  ;; limitations under the License.
  (ns backtype.storm.daemon.nimbus
-   (:import [java.nio ByteBuffer]
-            [java.util Collections]
-            [backtype.storm.generated StormTopology])
-   (:import [java.io FileNotFoundException])
+   (:import [org.apache.thrift.server THsHaServer THsHaServer$Args])
+   (:import [org.apache.thrift.protocol TBinaryProtocol TBinaryProtocol$Factory])
+   (:import [org.apache.thrift.exception])
+   (:import [org.apache.thrift.transport TNonblockingServerTransport TNonblockingServerSocket])
+   (:import [org.apache.commons.io FileUtils])
 -  (:import [java.nio ByteBuffer]
++  (:import [java.nio ByteBuffer])
+            [java.util Collections HashMap])
+   (:import [java.io FileNotFoundException File FileOutputStream])
    (:import [java.nio.channels Channels WritableByteChannel])
    (:import [backtype.storm.security.auth ThriftServer ThriftConnectionType ReqContext AuthUtils])
    (:use [backtype.storm.scheduler.DefaultScheduler])

http://git-wip-us.apache.org/repos/asf/storm/blob/5b67b571/storm-core/src/clj/backtype/storm/daemon/worker.clj
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/storm/blob/5b67b571/storm-core/test/clj/backtype/storm/nimbus_test.clj
----------------------------------------------------------------------
diff --cc storm-core/test/clj/backtype/storm/nimbus_test.clj
index 7671f58,99d1e32..f9502c2
--- a/storm-core/test/clj/backtype/storm/nimbus_test.clj
+++ b/storm-core/test/clj/backtype/storm/nimbus_test.clj
@@@ -15,18 -15,25 +15,25 @@@
  ;; limitations under the License.
  (ns backtype.storm.nimbus-test
    (:use [clojure test])
 -  (:require [backtype.storm [util :as util]])
 +  (:require [backtype.storm [util :as util] [stats :as stats]])
    (:require [backtype.storm.daemon [nimbus :as nimbus]])
-   (:import [backtype.storm.testing TestWordCounter TestWordSpout TestGlobalCount TestAggregatesCounter])
+   (:import [backtype.storm.testing TestWordCounter TestWordSpout TestGlobalCount
+             TestAggregatesCounter TestPlannerSpout TestPlannerBolt])
    (:import [backtype.storm.scheduler INimbus])
-   (:import [backtype.storm.generated Credentials])
-   (:use [backtype.storm bootstrap testing MockAutoCred])
+   (:import [backtype.storm.generated Credentials NotAliveException SubmitOptions
+             TopologyInitialStatus AlreadyAliveException KillOptions RebalanceOptions
+             InvalidTopologyException AuthorizationException])
+   (:import [java.util HashMap])
+   (:import [java.io File])
+   (:import [backtype.storm.utils Time])
+   (:import [org.apache.commons.io FileUtils])
+   (:use [backtype.storm testing MockAutoCred util config log timer])
    (:use [backtype.storm.daemon common])
    (:require [conjure.core])
-   (:use [conjure core])
-   )
- 
- (bootstrap)
+   (:require [backtype.storm
+              [thrift :as thrift]
+              [cluster :as cluster]])
+   (:use [conjure core]))
  
  (defn storm-component->task-info [cluster storm-name]
    (let [storm-id (get-storm-id (:storm-cluster-state cluster) storm-name)


Mime
View raw message