hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bo...@apache.org
Subject svn commit: r1360735 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/ hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/ hadoop-yarn/hadoop-yarn-...
Date Thu, 12 Jul 2012 15:52:13 GMT
Author: bobby
Date: Thu Jul 12 15:52:12 2012
New Revision: 1360735

URL: http://svn.apache.org/viewvc?rev=1360735&view=rev
Log:
MAPREDUCE-4416. Some tests fail if Clover is enabled (Kihwal Lee via bobby)

Modified:
    hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
    hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml

Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1360735&r1=1360734&r2=1360735&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Thu Jul 12 15:52:12 2012
@@ -221,6 +221,8 @@ Branch-2 ( Unreleased changes )
     MAPREDUCE-3993. Graceful handling of codec errors during decompression 
     (kkambatl via tucu)
 
+    MAPREDUCE-4416. Some tests fail if Clover is enabled (Kihwal Lee via bobby)
+
 Release 2.0.0-alpha - 05-23-2012
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java?rev=1360735&r1=1360734&r2=1360735&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestIndexCache.java
Thu Jul 12 15:52:12 2012
@@ -35,16 +35,23 @@ import org.apache.hadoop.mapreduce.serve
 import junit.framework.TestCase;
 
 public class TestIndexCache extends TestCase {
+  private JobConf conf;
+  private FileSystem fs;
+  private Path p;
+
+  @Override
+  public void setUp() throws IOException {
+    conf = new JobConf();
+    fs = FileSystem.getLocal(conf).getRaw();
+    p =  new Path(System.getProperty("test.build.data", "/tmp"),
+        "cache").makeQualified(fs.getUri(), fs.getWorkingDirectory());
+  }
 
   public void testLRCPolicy() throws Exception {
     Random r = new Random();
     long seed = r.nextLong();
     r.setSeed(seed);
     System.out.println("seed: " + seed);
-    JobConf conf = new JobConf();
-    FileSystem fs = FileSystem.getLocal(conf).getRaw();
-    Path p = new Path(System.getProperty("test.build.data", "/tmp"),
-        "cache").makeQualified(fs);
     fs.delete(p, true);
     conf.setInt(TTConfig.TT_INDEX_CACHE, 1);
     final int partsPerMap = 1000;
@@ -115,10 +122,6 @@ public class TestIndexCache extends Test
 
   public void testBadIndex() throws Exception {
     final int parts = 30;
-    JobConf conf = new JobConf();
-    FileSystem fs = FileSystem.getLocal(conf).getRaw();
-    Path p = new Path(System.getProperty("test.build.data", "/tmp"),
-        "cache").makeQualified(fs);
     fs.delete(p, true);
     conf.setInt(TTConfig.TT_INDEX_CACHE, 1);
     IndexCache cache = new IndexCache(conf);
@@ -150,10 +153,6 @@ public class TestIndexCache extends Test
   }
 
   public void testInvalidReduceNumberOrLength() throws Exception {
-    JobConf conf = new JobConf();
-    FileSystem fs = FileSystem.getLocal(conf).getRaw();
-    Path p = new Path(System.getProperty("test.build.data", "/tmp"),
-                      "cache").makeQualified(fs);
     fs.delete(p, true);
     conf.setInt(TTConfig.TT_INDEX_CACHE, 1);
     final int partsPerMap = 1000;
@@ -199,10 +198,6 @@ public class TestIndexCache extends Test
     // This test case may not repeatable. But on my macbook this test 
     // fails with probability of 100% on code before MAPREDUCE-2541,
     // so it is repeatable in practice.
-    JobConf conf = new JobConf();
-    FileSystem fs = FileSystem.getLocal(conf).getRaw();
-    Path p = new Path(System.getProperty("test.build.data", "/tmp"),
-                      "cache").makeQualified(fs);
     fs.delete(p, true);
     conf.setInt(TTConfig.TT_INDEX_CACHE, 10);
     // Make a big file so removeMapThread almost surely runs faster than 
@@ -247,6 +242,66 @@ public class TestIndexCache extends Test
     }      
   }
   
+  public void testCreateRace() throws Exception {
+    fs.delete(p, true);
+    conf.setInt(TTConfig.TT_INDEX_CACHE, 1);
+    final int partsPerMap = 1000;
+    final int bytesPerFile = partsPerMap * 24;
+    final IndexCache cache = new IndexCache(conf);
+    
+    final Path racy = new Path(p, "racyIndex");
+    final String user =  
+      UserGroupInformation.getCurrentUser().getShortUserName();
+    writeFile(fs, racy, bytesPerFile, partsPerMap);
+
+    // run multiple instances
+    Thread[] getInfoThreads = new Thread[50];
+    for (int i = 0; i < 50; i++) {
+      getInfoThreads[i] = new Thread() {
+        @Override
+        public void run() {
+          try {
+            cache.getIndexInformation("racyIndex", partsPerMap, racy, user);
+            cache.removeMap("racyIndex");
+          } catch (Exception e) {
+            // should not be here
+          }
+        }
+      };
+    }
+
+    for (int i = 0; i < 50; i++) {
+      getInfoThreads[i].start();
+    }
+
+    final Thread mainTestThread = Thread.currentThread();
+
+    Thread timeoutThread = new Thread() {
+      @Override
+      public void run() {
+        try {
+          Thread.sleep(15000);
+          mainTestThread.interrupt();
+        } catch (InterruptedException ie) {
+          // we are done;
+        }
+      }
+    };
+
+    for (int i = 0; i < 50; i++) {
+      try {
+        getInfoThreads[i].join();
+      } catch (InterruptedException ie) {
+        // we haven't finished in time. Potential deadlock/race.
+        fail("Unexpectedly long delay during concurrent cache entry creations");
+      }
+    }
+    // stop the timeoutThread. If we get interrupted before stopping, there
+    // must be something wrong, although it wasn't a deadlock. No need to
+    // catch and swallow.
+    timeoutThread.interrupt();
+  }
+
   private static void checkRecord(IndexRecord rec, long fill) {
     assertEquals(fill, rec.startOffset);
     assertEquals(fill, rec.rawLength);

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml?rev=1360735&r1=1360734&r2=1360735&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
Thu Jul 12 15:52:12 2012
@@ -99,6 +99,23 @@
     </dependency>
   </dependencies>
 
+ <profiles>
+  <profile>
+    <id>clover</id>
+    <activation>
+      <activeByDefault>false</activeByDefault>
+      <property>
+        <name>clover</name>
+      </property>
+    </activation>
+    <dependencies>
+      <dependency>
+        <groupId>com.cenqua.clover</groupId>
+        <artifactId>clover</artifactId>
+      </dependency>
+    </dependencies>
+  </profile>
+</profiles>
   <build>
     <plugins>
       <plugin>

Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml?rev=1360735&r1=1360734&r2=1360735&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
(original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
Thu Jul 12 15:52:12 2012
@@ -31,4 +31,21 @@
   <modules>
     <module>hadoop-yarn-applications-distributedshell</module>
   </modules>
+ <profiles>
+  <profile>
+    <id>clover</id>
+    <activation>
+      <activeByDefault>false</activeByDefault>
+      <property>
+        <name>clover</name>
+      </property>
+    </activation>
+    <dependencies>
+      <dependency>
+        <groupId>com.cenqua.clover</groupId>
+        <artifactId>clover</artifactId>
+      </dependency>
+    </dependencies>
+  </profile>
+</profiles>
 </project>



Mime
View raw message