hadoop-mapreduce-commits mailing list archives

Author: bobby
Date: Fri Jul  6 17:19:13 2012
New Revision: 1358305

URL: http://svn.apache.org/viewvc?rev=1358305&view=rev
Log:
MAPREDUCE-4379. Node Manager throws java.lang.OutOfMemoryError: Java heap space due to org.apache.hadoop.fs.LocalDirAllocator.contexts
(Devaraj K via bobby)

Modified:
    hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java

Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1358305&r1=1358304&r2=1358305&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Fri Jul  6 17:19:13 2012
@@ -643,6 +643,10 @@ Release 0.23.3 - UNRELEASED
     MAPREDUCE-4387. RM gets fatal error and exits during TestRM 
     (Kihwal Lee via tgraves)
 
+    MAPREDUCE-4379. Node Manager throws java.lang.OutOfMemoryError: Java heap
+    space due to org.apache.hadoop.fs.LocalDirAllocator.contexts (Devaraj K
+    via bobby)
+
 Release 0.23.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java?rev=1358305&r1=1358304&r2=1358305&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
Fri Jul  6 17:19:13 2012
@@ -91,6 +91,7 @@ public class ContainerLocalizer {
   private final LocalDirAllocator userDirs;
   private final RecordFactory recordFactory;
   private final Map<LocalResource,Future<Path>> pendingResources;
+  private final String appCacheDirContextName;
 
   public ContainerLocalizer(FileContext lfs, String user, String appId,
       String localizerId, List<Path> localDirs,
@@ -108,10 +109,9 @@ public class ContainerLocalizer {
     this.localizerId = localizerId;
     this.recordFactory = recordFactory;
     this.conf = new Configuration();
-    this.appDirs =
-      new LocalDirAllocator(String.format(APPCACHE_CTXT_FMT, appId));
-    this.userDirs =
-      new LocalDirAllocator(String.format(USERCACHE_CTXT_FMT, appId));
+    this.appCacheDirContextName = String.format(APPCACHE_CTXT_FMT, appId);
+    this.appDirs = new LocalDirAllocator(appCacheDirContextName);
+    this.userDirs = new LocalDirAllocator(String.format(USERCACHE_CTXT_FMT, user));
     this.pendingResources = new HashMap<LocalResource,Future<Path>>();
   }
 
@@ -121,6 +121,7 @@ public class ContainerLocalizer {
       rpc.getProxy(LocalizationProtocol.class, nmAddr, conf);
   }
 
+  @SuppressWarnings("deprecation")
   public int runLocalization(final InetSocketAddress nmAddr)
       throws IOException, InterruptedException {
     // load credentials
@@ -177,6 +178,7 @@ public class ContainerLocalizer {
       if (exec != null) {
         exec.shutdownNow();
       }
+      LocalDirAllocator.removeContext(appCacheDirContextName);
     }
   }
 
@@ -373,7 +375,7 @@ public class ContainerLocalizer {
       lfs.mkdir(appFileCacheDir, null, false);
     }
     conf.setStrings(String.format(APPCACHE_CTXT_FMT, appId), appsFileCacheDirs);
-    conf.setStrings(String.format(USERCACHE_CTXT_FMT, appId), usersFileCacheDirs);
+    conf.setStrings(String.format(USERCACHE_CTXT_FMT, user), usersFileCacheDirs);
   }
 
 }