storm-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ptgo...@apache.org
Subject [37/50] [abbrv] git commit: STORM-241: making thrift binary chunk size configurable
Date Thu, 20 Mar 2014 21:23:00 GMT
STORM-241: making thrift binary chunk size configurable


Project: http://git-wip-us.apache.org/repos/asf/incubator-storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-storm/commit/02109413
Tree: http://git-wip-us.apache.org/repos/asf/incubator-storm/tree/02109413
Diff: http://git-wip-us.apache.org/repos/asf/incubator-storm/diff/02109413

Branch: refs/heads/master
Commit: 021094131c13909bd9195cb3e3b9c9775ee2fcf5
Parents: 254ec13
Author: supercargo <adam@acuitysds.com>
Authored: Sun Feb 16 08:23:46 2014 -0500
Committer: supercargo <adam@acuitysds.com>
Committed: Sun Feb 16 08:23:46 2014 -0500

----------------------------------------------------------------------
 conf/defaults.yaml                                    | 1 +
 storm-core/src/jvm/backtype/storm/Config.java         | 6 ++++++
 storm-core/src/jvm/backtype/storm/StormSubmitter.java | 5 ++++-
 3 files changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/02109413/conf/defaults.yaml
----------------------------------------------------------------------
diff --git a/conf/defaults.yaml b/conf/defaults.yaml
index 2dbba24..d0b87a0 100644
--- a/conf/defaults.yaml
+++ b/conf/defaults.yaml
@@ -35,6 +35,7 @@ storm.zookeeper.retry.intervalceiling.millis: 30000
 storm.cluster.mode: "distributed" # can be distributed or local
 storm.local.mode.zmq: false
 storm.thrift.transport: "backtype.storm.security.auth.SimpleTransportPlugin"
+storm.thrift.chunk.size: 307200
 storm.messaging.transport: "backtype.storm.messaging.netty.Context"
 
 ### nimbus.* configs are for the master

http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/02109413/storm-core/src/jvm/backtype/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java
index 281ae52..1a308c2 100644
--- a/storm-core/src/jvm/backtype/storm/Config.java
+++ b/storm-core/src/jvm/backtype/storm/Config.java
@@ -137,6 +137,12 @@ public class Config extends HashMap<String, Object> {
     public static final Object STORM_THRIFT_TRANSPORT_PLUGIN_SCHEMA = String.class;
 
     /**
+     * The chunk size in bytes to use when sending binary blobs over the Thrift protocol
+     */
+    public static final String STORM_THRIFT_CHUNK_SIZE = "storm.thrift.chunk.size";
+    public static final Object STORM_THRIFT_CHUNK_SIZE_SCHEMA = Number.class;
+    
+    /**
      * The serializer class for ListDelegate (tuple payload). 
      * The default serializer will be ListDelegateSerializer
      */

http://git-wip-us.apache.org/repos/asf/incubator-storm/blob/02109413/storm-core/src/jvm/backtype/storm/StormSubmitter.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/backtype/storm/StormSubmitter.java b/storm-core/src/jvm/backtype/storm/StormSubmitter.java
index 8073456..b5081e8 100644
--- a/storm-core/src/jvm/backtype/storm/StormSubmitter.java
+++ b/storm-core/src/jvm/backtype/storm/StormSubmitter.java
@@ -148,10 +148,13 @@ public class StormSubmitter {
             throw new RuntimeException("Must submit topologies using the 'storm' client script
so that StormSubmitter knows which jar to upload.");
         }
         NimbusClient client = NimbusClient.getConfiguredClient(conf);
+        
+        int thriftChunkSize = ((Number) conf.get(Config.STORM_THRIFT_CHUNK_SIZE)).intValue();
+
         try {
             String uploadLocation = client.getClient().beginFileUpload();
             LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation);
-            BufferFileInputStream is = new BufferFileInputStream(localJar);
+            BufferFileInputStream is = new BufferFileInputStream(localJar, thriftChunkSize);
             while(true) {
                 byte[] toSubmit = is.read();
                 if(toSubmit.length==0) break;


Mime
View raw message