storm-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kabh...@apache.org
Subject [1/2] storm git commit: STORM-2660 The Nimbus storm-local directory is relative to the working directory of the shell executing "storm nimbus"
Date Sun, 03 Sep 2017 23:09:19 GMT
Repository: storm
Updated Branches:
  refs/heads/master 758a1d2d6 -> aefc72520


STORM-2660 The Nimbus storm-local directory is relative to the working directory of the shell
executing "storm nimbus"

* Let Nimbus uses same storm-local directory as Supervisor which is known to be valid
* also addresses other places as well (review comment from @srdo)


Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/96a08665
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/96a08665
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/96a08665

Branch: refs/heads/master
Commit: 96a08665ca5cceda541514872e904079e59b486a
Parents: 758a1d2
Author: Jungtaek Lim <kabhwan@gmail.com>
Authored: Thu Aug 31 18:26:59 2017 +0900
Committer: Jungtaek Lim <kabhwan@gmail.com>
Committed: Mon Sep 4 08:08:37 2017 +0900

----------------------------------------------------------------------
 storm-client/src/jvm/org/apache/storm/Config.java     |  6 ++++--
 .../src/jvm/org/apache/storm/utils/ConfigUtils.java   | 14 ++++++++++++++
 .../org/apache/storm/blobstore/LocalFsBlobStore.java  |  7 ++-----
 .../scheduler/utils/ArtifactoryConfigLoader.java      | 14 +++++---------
 .../org/apache/storm/utils/ServerConfigUtils.java     |  2 +-
 5 files changed, 26 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/storm/blob/96a08665/storm-client/src/jvm/org/apache/storm/Config.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/Config.java b/storm-client/src/jvm/org/apache/storm/Config.java
index 2965b79..13b8a22 100644
--- a/storm-client/src/jvm/org/apache/storm/Config.java
+++ b/storm-client/src/jvm/org/apache/storm/Config.java
@@ -1031,7 +1031,8 @@ public class Config extends HashMap<String, Object> {
     /**
      * What directory to use for the blobstore. The directory is expected to be an
      * absolute path when using HDFS blobstore, for LocalFsBlobStore it could be either
-     * absolute or relative.
+     * absolute or relative. If the setting is a relative directory, it is relative to
+     * root directory of Storm installation.
      */
     @isString
     public static final String BLOBSTORE_DIR = "blobstore.dir";
@@ -1424,7 +1425,8 @@ public class Config extends HashMap<String, Object> {
     /**
      * A directory on the local filesystem used by Storm for any local
      * filesystem usage it needs. The directory must exist and the Storm daemons must
-     * have permission to read/write from this location.
+     * have permission to read/write from this location. It could be either absolute or relative.
+     * If the setting is a relative directory, it is relative to root directory of Storm
installation.
      */
     @isString
     public static final String STORM_LOCAL_DIR = "storm.local.dir";

http://git-wip-us.apache.org/repos/asf/storm/blob/96a08665/storm-client/src/jvm/org/apache/storm/utils/ConfigUtils.java
----------------------------------------------------------------------
diff --git a/storm-client/src/jvm/org/apache/storm/utils/ConfigUtils.java b/storm-client/src/jvm/org/apache/storm/utils/ConfigUtils.java
index 6ea79dd..4a54ac7 100644
--- a/storm-client/src/jvm/org/apache/storm/utils/ConfigUtils.java
+++ b/storm-client/src/jvm/org/apache/storm/utils/ConfigUtils.java
@@ -215,6 +215,20 @@ public class ConfigUtils {
         }
     }
 
+    public static String absoluteStormBlobStoreDir(Map<String, Object> conf) {
+        String stormHome = System.getProperty("storm.home");
+        String blobStoreDir = (String) conf.get(Config.BLOBSTORE_DIR);
+        if (blobStoreDir == null) {
+            return ConfigUtils.absoluteStormLocalDir(conf);
+        } else {
+            if (new File(blobStoreDir).isAbsolute()) {
+                return blobStoreDir;
+            } else {
+                return (stormHome + FILE_SEPARATOR + blobStoreDir);
+            }
+        }
+    }
+
     public static StormTopology readSupervisorStormCodeGivenPath(String stormCodePath, AdvancedFSOps
ops) throws IOException {
         return Utils.deserialize(ops.slurp(new File(stormCodePath)), StormTopology.class);
     }

http://git-wip-us.apache.org/repos/asf/storm/blob/96a08665/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStore.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStore.java b/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStore.java
index 87c8dce..2f73334 100644
--- a/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStore.java
+++ b/storm-server/src/main/java/org/apache/storm/blobstore/LocalFsBlobStore.java
@@ -17,7 +17,6 @@
  */
 package org.apache.storm.blobstore;
 
-import org.apache.storm.Config;
 import org.apache.storm.generated.SettableBlobMeta;
 import org.apache.storm.generated.AuthorizationException;
 import org.apache.storm.generated.KeyAlreadyExistsException;
@@ -25,6 +24,7 @@ import org.apache.storm.generated.KeyNotFoundException;
 import org.apache.storm.generated.ReadableBlobMeta;
 
 import org.apache.storm.nimbus.NimbusInfo;
+import org.apache.storm.utils.ConfigUtils;
 import org.apache.storm.utils.Utils;
 import org.apache.zookeeper.KeeperException.NoNodeException;
 import org.apache.curator.framework.CuratorFramework;
@@ -85,10 +85,7 @@ public class LocalFsBlobStore extends BlobStore {
         this.nimbusInfo = nimbusInfo;
         zkClient = BlobStoreUtils.createZKClient(conf);
         if (overrideBase == null) {
-            overrideBase = (String)conf.get(Config.BLOBSTORE_DIR);
-            if (overrideBase == null) {
-                overrideBase = (String) conf.get(Config.STORM_LOCAL_DIR);
-            }
+            overrideBase = ConfigUtils.absoluteStormBlobStoreDir(conf);
         }
         File baseDir = new File(overrideBase, BASE_BLOBS_DIR_NAME);
         try {

http://git-wip-us.apache.org/repos/asf/storm/blob/96a08665/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoader.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoader.java
b/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoader.java
index 3afa903..87f39cd 100644
--- a/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoader.java
+++ b/storm-server/src/main/java/org/apache/storm/scheduler/utils/ArtifactoryConfigLoader.java
@@ -37,6 +37,8 @@ import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.util.EntityUtils;
 import org.apache.storm.Config;
 import org.apache.storm.DaemonConfig;
+import org.apache.storm.utils.ConfigUtils;
+import org.apache.storm.utils.ServerConfigUtils;
 import org.apache.storm.utils.Time;
 import org.apache.storm.utils.Utils;
 import org.json.simple.JSONArray;
@@ -330,15 +332,9 @@ public class ArtifactoryConfigLoader implements IConfigLoader {
         }
     }
 
-    private void makeArtifactoryCache(String location) {
-        // Just make sure appropriate directories exist
-        String localDirName = (String) conf.get(Config.STORM_LOCAL_DIR);
-        if (localDirName == null) {
-            return;
-        }
-
+    private void makeArtifactoryCache(String location) throws IOException {
         // First make the cache dir
-        localDirName = localDirName + File.separator + "nimbus" + File.separator + LOCAL_ARTIFACT_DIR;
+        String localDirName = ServerConfigUtils.masterLocalDir(conf) + File.separator + LOCAL_ARTIFACT_DIR;
         File dir = new File(localDirName);
         if (! dir.exists()) {
             dir.mkdirs();
@@ -352,7 +348,7 @@ public class ArtifactoryConfigLoader implements IConfigLoader {
         cacheInitialized = true;
     }
 
-    private Map loadFromURI(URI uri) {
+    private Map loadFromURI(URI uri) throws IOException {
         String host = uri.getHost();
         Integer port = uri.getPort();
         String location = uri.getPath();

http://git-wip-us.apache.org/repos/asf/storm/blob/96a08665/storm-server/src/main/java/org/apache/storm/utils/ServerConfigUtils.java
----------------------------------------------------------------------
diff --git a/storm-server/src/main/java/org/apache/storm/utils/ServerConfigUtils.java b/storm-server/src/main/java/org/apache/storm/utils/ServerConfigUtils.java
index 426e190..480d523 100644
--- a/storm-server/src/main/java/org/apache/storm/utils/ServerConfigUtils.java
+++ b/storm-server/src/main/java/org/apache/storm/utils/ServerConfigUtils.java
@@ -55,7 +55,7 @@ public class ServerConfigUtils {
     }
 
     public static String masterLocalDir(Map<String, Object> conf) throws IOException
{
-        String ret = String.valueOf(conf.get(Config.STORM_LOCAL_DIR)) + FILE_SEPARATOR +
"nimbus";
+        String ret = ConfigUtils.absoluteStormLocalDir(conf) + FILE_SEPARATOR + "nimbus";
         FileUtils.forceMkdir(new File(ret));
         return ret;
     }


Mime
View raw message