hama-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From edwardy...@apache.org
Subject svn commit: r1171537 - in /incubator/hama/trunk: ./ core/src/main/java/org/apache/hama/bsp/ core/src/main/java/org/apache/hama/util/ core/src/main/webapp/bspmaster/ core/src/test/java/org/apache/hama/bsp/
Date Fri, 16 Sep 2011 13:05:11 GMT
Author: edwardyoon
Date: Fri Sep 16 13:05:11 2011
New Revision: 1171537

URL: http://svn.apache.org/viewvc?rev=1171537&view=rev
Log:
Web Interface does not update Superstep Count

Modified:
    incubator/hama/trunk/CHANGES.txt
    incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/BSPMaster.java
    incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/ClusterStatus.java
    incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/LocalBSPRunner.java
    incubator/hama/trunk/core/src/main/java/org/apache/hama/util/BSPServletUtil.java
    incubator/hama/trunk/core/src/main/webapp/bspmaster/machines.jsp
    incubator/hama/trunk/core/src/test/java/org/apache/hama/bsp/TestClusterStatus.java

Modified: incubator/hama/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/CHANGES.txt?rev=1171537&r1=1171536&r2=1171537&view=diff
==============================================================================
--- incubator/hama/trunk/CHANGES.txt (original)
+++ incubator/hama/trunk/CHANGES.txt Fri Sep 16 13:05:11 2011
@@ -9,6 +9,7 @@ Release 0.4 - Unreleased
 
   BUG FIXES
 
+    HAMA-436: Web Interface does not update Superstep Count (Thomas Jungblut)
     HAMA-429: Groom statuses should be reported periodically (ChiaHung Lin via edwardyoon)
     HAMA-421: Maven build issues using proxy (Joe Crobak via edwardyoon)
 

Modified: incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/BSPMaster.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/BSPMaster.java?rev=1171537&r1=1171536&r2=1171537&view=diff
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/BSPMaster.java (original)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/BSPMaster.java Fri Sep 16
13:05:11 2011
@@ -72,7 +72,7 @@ public class BSPMaster implements JobSub
 
   private ZooKeeper zk = null;
   private String bspRoot = null;
-  
+
   /**
    * Constants for BSPMaster's status.
    */
@@ -343,7 +343,7 @@ public class BSPMaster implements JobSub
       LOG.error("Fail to register GroomServer " + status.getGroomName(), e);
       return false;
     }
-    LOG.info(status.getGroomName()+" is added.");
+    LOG.info(status.getGroomName() + " is added.");
     return true;
   }
 
@@ -426,7 +426,7 @@ public class BSPMaster implements JobSub
     BSPMaster result = new BSPMaster(conf, identifier);
     result.taskScheduler.setGroomServerManager(result);
     result.taskScheduler.start();
-    
+
     // init zk root and child nodes
     result.initZK(conf);
 
@@ -440,8 +440,8 @@ public class BSPMaster implements JobSub
    */
   private void initZK(HamaConfiguration conf) {
     try {
-      zk = new ZooKeeper(QuorumPeer.getZKQuorumServersString(conf), conf
-          .getInt(Constants.ZOOKEEPER_SESSION_TIMEOUT, 1200000), this);
+      zk = new ZooKeeper(QuorumPeer.getZKQuorumServersString(conf),
+          conf.getInt(Constants.ZOOKEEPER_SESSION_TIMEOUT, 1200000), this);
     } catch (IOException e) {
       LOG.error("Exception during reinitialization!", e);
     }
@@ -499,7 +499,7 @@ public class BSPMaster implements JobSub
       for (String node : zk.getChildren("/" + string, this)) {
         zk.delete("/" + string + "/" + node, 0);
       }
-      
+
       zk.delete("/" + string, 0);
     } catch (InterruptedException e) {
       e.printStackTrace();
@@ -507,7 +507,7 @@ public class BSPMaster implements JobSub
       e.printStackTrace();
     }
   }
-  
+
   public static InetSocketAddress getAddress(Configuration conf) {
     String hamaMasterStr = conf.get("bsp.master.address", "localhost");
     int defaultPort = conf.getInt("bsp.master.port", 40000);
@@ -594,16 +594,16 @@ public class BSPMaster implements JobSub
 
   @Override
   public ClusterStatus getClusterStatus(boolean detailed) {
-    Map<String, String> groomsMap = null;
+    Map<String, GroomServerStatus> groomsMap = null;
 
     // give the caller a snapshot of the cluster status
     int numGroomServers = groomServers.size();
     if (detailed) {
-      groomsMap = new HashMap<String, String>();
+      groomsMap = new HashMap<String, GroomServerStatus>();
       for (Map.Entry<GroomServerStatus, GroomProtocol> entry : groomServers
           .entrySet()) {
         GroomServerStatus s = entry.getKey();
-        groomsMap.put(s.getGroomName(), s.getGroomHostName());
+        groomsMap.put(s.getGroomName(), s);
       }
 
     }

Modified: incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/ClusterStatus.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/ClusterStatus.java?rev=1171537&r1=1171536&r2=1171537&view=diff
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/ClusterStatus.java (original)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/ClusterStatus.java Fri Sep
16 13:05:11 2011
@@ -20,10 +20,9 @@ package org.apache.hama.bsp;
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableUtils;
@@ -31,56 +30,58 @@ import org.apache.hadoop.io.WritableUtil
 /**
  * Status information on the current state of the BSP cluster.
  * 
- * <p><code>ClusterStatus</code> provides clients with information such
as:
+ * <p>
+ * <code>ClusterStatus</code> provides clients with information such as:
  * <ol>
- *   <li>
- *   Size of the cluster. 
- *   </li>
- *   <li>
- *   Name of the grooms. 
- *   </li>
- *   <li>
- *   Task capacity of the cluster. 
- *   </li>
- *   <li>
- *   The number of currently running bsp tasks.
- *   </li>
- *   <li>
- *   State of the <code>BSPMaster</code>.
- *   </li>
- * </ol></p>
+ * <li>
+ * Size of the cluster.</li>
+ * <li>
+ * Name of the grooms.</li>
+ * <li>
+ * Task capacity of the cluster.</li>
+ * <li>
+ * The number of currently running bsp tasks.</li>
+ * <li>
+ * State of the <code>BSPMaster</code>.</li>
+ * </ol>
+ * </p>
  * 
- * <p>Clients can query for the latest <code>ClusterStatus</code>, via

- * {@link BSPJobClient#getClusterStatus(boolean)}.</p>
+ * <p>
+ * Clients can query for the latest <code>ClusterStatus</code>, via
+ * {@link BSPJobClient#getClusterStatus(boolean)}.
+ * </p>
  * 
  * @see BSPMaster
  */
 public class ClusterStatus implements Writable {
 
   private int numActiveGrooms;
-  private Map<String, String> activeGrooms = new HashMap<String, String>();
+  private Map<String, GroomServerStatus> activeGrooms = new HashMap<String, GroomServerStatus>();
+  private Map<String, String> cachedActiveGroomNames = null;
   private int tasks;
   private int maxTasks;
   private BSPMaster.State state;
-  
+
   /**
    * 
    */
-  public ClusterStatus() {}
-    
-  public ClusterStatus(int grooms, int tasks, int maxTasks, BSPMaster.State state) {
+  public ClusterStatus() {
+  }
+
+  public ClusterStatus(int grooms, int tasks, int maxTasks,
+      BSPMaster.State state) {
     this.numActiveGrooms = grooms;
     this.tasks = tasks;
     this.maxTasks = maxTasks;
     this.state = state;
   }
-  
-  public ClusterStatus(Map<String, String> activeGrooms, int tasks, int maxTasks,
-      BSPMaster.State state) {
+
+  public ClusterStatus(Map<String, GroomServerStatus> activeGrooms, int tasks,
+      int maxTasks, BSPMaster.State state) {
     this(activeGrooms.size(), tasks, maxTasks, state);
     this.activeGrooms = activeGrooms;
   }
-  
+
   /**
    * Get the number of groom servers in the cluster.
    * 
@@ -89,16 +90,34 @@ public class ClusterStatus implements Wr
   public int getGroomServers() {
     return numActiveGrooms;
   }
-  
+
   /**
    * Get the names of groom servers, and their hostnames, in the cluster.
    * 
    * @return the active groom servers in the cluster.
-   */  
+   */
   public Map<String, String> getActiveGroomNames() {
+    if (cachedActiveGroomNames == null) {
+      if (activeGrooms != null) {
+        Map<String, String> map = new HashMap<String, String>();
+        for (Entry<String, GroomServerStatus> entry : activeGrooms.entrySet()) {
+          map.put(entry.getKey(), entry.getValue().getGroomHostName());
+        }
+        cachedActiveGroomNames = map;
+      }
+    }
+    return cachedActiveGroomNames;
+  }
+
+  /**
+   * Get the names of groom servers, and their current status in the cluster.
+   * 
+   * @return the active groom servers in the cluster.
+   */
+  public Map<String, GroomServerStatus> getActiveGroomServerStatus() {
     return activeGrooms;
   }
-  
+
   /**
    * Get the number of currently running tasks in the cluster.
    * 
@@ -107,7 +126,7 @@ public class ClusterStatus implements Wr
   public int getTasks() {
     return tasks;
   }
-  
+
   /**
    * Get the maximum capacity for running tasks in the cluster.
    * 
@@ -116,20 +135,20 @@ public class ClusterStatus implements Wr
   public int getMaxTasks() {
     return maxTasks;
   }
-  
+
   /**
-   * Get the current state of the <code>BSPMaster</code>, 
-   * as {@link BSPMaster.State}
+   * Get the current state of the <code>BSPMaster</code>, as
+   * {@link BSPMaster.State}
    * 
    * @return the current state of the <code>BSPMaster</code>.
    */
   public BSPMaster.State getBSPMasterState() {
     return state;
   }
-  
-  //////////////////////////////////////////////
+
+  // ////////////////////////////////////////////
   // Writable
-  //////////////////////////////////////////////
+  // ////////////////////////////////////////////
   @Override
   public void write(DataOutput out) throws IOException {
     if (activeGrooms.isEmpty()) {
@@ -139,33 +158,30 @@ public class ClusterStatus implements Wr
       out.writeInt(activeGrooms.size());
       out.writeBoolean(true);
 
-      String[] groomNames = activeGrooms.keySet().toArray(new String[0]);
-      List<String> peerNames = new ArrayList<String>();
-
-      for (String groomName : groomNames) {
-        peerNames.add(activeGrooms.get(groomName));
+      for (Entry<String, GroomServerStatus> entry : activeGrooms.entrySet()) {
+        out.writeUTF(entry.getKey());
+        entry.getValue().write(out);
       }
 
-      WritableUtils.writeCompressedStringArray(out, groomNames);
-      WritableUtils.writeCompressedStringArray(out, peerNames.toArray(new String[0]));
     }
     out.writeInt(tasks);
     out.writeInt(maxTasks);
     WritableUtils.writeEnum(out, state);
   }
-  
+
   @Override
   public void readFields(DataInput in) throws IOException {
     numActiveGrooms = in.readInt();
     boolean groomListFollows = in.readBoolean();
 
     if (groomListFollows) {
-      String[] groomNames = WritableUtils.readCompressedStringArray(in);
-      String[] peerNames = WritableUtils.readCompressedStringArray(in);
-      activeGrooms = new HashMap<String, String>(groomNames.length);
+      activeGrooms = new HashMap<String, GroomServerStatus>(numActiveGrooms);
 
-      for (int i = 0; i < groomNames.length; i++) {
-        activeGrooms.put(groomNames[i], peerNames[i]);
+      for (int i = 0; i < numActiveGrooms; i++) {
+        final String groomName = in.readUTF();
+        final GroomServerStatus status = new GroomServerStatus();
+        status.readFields(in);
+        activeGrooms.put(groomName, status);
       }
     }
 

Modified: incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/LocalBSPRunner.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/LocalBSPRunner.java?rev=1171537&r1=1171536&r2=1171537&view=diff
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/LocalBSPRunner.java (original)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/bsp/LocalBSPRunner.java Fri Sep
16 13:05:11 2011
@@ -18,6 +18,7 @@
 package org.apache.hama.bsp;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.Map;
@@ -58,7 +59,6 @@ public class LocalBSPRunner implements J
   protected static CyclicBarrier barrier;
 
   static {
-    threadPoolSize = Runtime.getRuntime().availableProcessors();
     barrier = new CyclicBarrier(threadPoolSize);
     threadPool = (ThreadPoolExecutor) Executors
         .newFixedThreadPool(threadPoolSize);
@@ -125,9 +125,9 @@ public class LocalBSPRunner implements J
 
   @Override
   public ClusterStatus getClusterStatus(boolean detailed) throws IOException {
-    Map<String, String> map = new HashMap<String, String>();
+    Map<String, GroomServerStatus> map = new HashMap<String, GroomServerStatus>();
     for (Entry<String, LocalGroom> entry : localGrooms.entrySet()) {
-      map.put(entry.getKey(), entry.getValue().getPeerName());
+      map.put(entry.getKey(), new GroomServerStatus(entry.getKey(), new ArrayList<TaskStatus>(0),
0, 0, "", entry.getKey()));
     }
     return new ClusterStatus(map, threadPoolSize, threadPoolSize, State.RUNNING);
   }

Modified: incubator/hama/trunk/core/src/main/java/org/apache/hama/util/BSPServletUtil.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/java/org/apache/hama/util/BSPServletUtil.java?rev=1171537&r1=1171536&r2=1171537&view=diff
==============================================================================
--- incubator/hama/trunk/core/src/main/java/org/apache/hama/util/BSPServletUtil.java (original)
+++ incubator/hama/trunk/core/src/main/java/org/apache/hama/util/BSPServletUtil.java Fri Sep
16 13:05:11 2011
@@ -18,10 +18,14 @@
 package org.apache.hama.util;
 
 import java.io.IOException;
-import java.util.Date;
 import java.util.Calendar;
+import java.util.Date;
+import java.util.Map.Entry;
 
 import org.apache.hadoop.util.ServletUtil;
+import org.apache.hama.bsp.BSPMaster;
+import org.apache.hama.bsp.ClusterStatus;
+import org.apache.hama.bsp.GroomServerStatus;
 import org.apache.hama.bsp.JobStatus;
 
 public class BSPServletUtil extends ServletUtil {
@@ -59,14 +63,15 @@ public class BSPServletUtil extends Serv
       sb.append("<tr><th>Jobid</th>" + "<th>User</th>" + "<th>Name</th>"
           + "<th>SuperStep</th>" + "<th>Starttime</th>" + "</tr>\n");
       for (JobStatus status : jobs) {
-        sb.append("<tr><td><a href=\"bspjob.jsp?jobid="+status.getJobID()+
"\">");
+        sb.append("<tr><td><a href=\"bspjob.jsp?jobid=" + status.getJobID()
+            + "\">");
         sb.append(status.getJobID());
         sb.append("</a></td><td>");
         sb.append(status.getUsername());
         sb.append("</td><td>");
         sb.append(status.getName());
         sb.append("</td><td>");
-        sb.append(status.getSuperstepCount());
+        sb.append(status.progress());
         sb.append("</td><td>");
         sb.append(new Date(status.getStartTime()));
         sb.append("</td></tr>\n");
@@ -79,4 +84,32 @@ public class BSPServletUtil extends Serv
     return sb.toString();
   }
 
+  public static String generateGroomsTable(String type, ClusterStatus status,
+      BSPMaster master) throws IOException {
+    StringBuilder sb = new StringBuilder();
+    sb.append("<center>\n");
+    sb.append("<table border=\"2\" cellpadding=\"5\" cellspacing=\"2\">\n");
+    sb.append("<tr><td align=\"center\" colspan=\"6\"><b>Groom Servers</b></td></tr>\n");
+    sb.append("<tr><td><b>Name</b></td>"
+        + "<td><b>Host</b></td>"
+        + "<td><b># maximum tasks</b></td><td><b># current
running tasks</b></td>" +
+        		"<td><b># current failures</b></td>" +
+        		"<td><b>Last seen</b></td>" +
+        		"</tr>\n");
+    for (Entry<String, GroomServerStatus> entry : status
+        .getActiveGroomServerStatus().entrySet()) {
+      sb.append("<tr><td>");
+      sb.append(entry.getKey() + "</td><td>");
+      sb.append(entry.getValue().getGroomHostName() + "</td>" + "<td>"
+          + entry.getValue().getMaxTasks() + "</td><td>");
+      sb.append(entry.getValue().countTasks() + "</td><td>");
+      sb.append(entry.getValue().getFailures() + "</td><td>");
+      sb.append(entry.getValue().getLastSeen() + "</td>");
+      sb.append("</tr>\n");
+    }
+    sb.append("</table>\n");
+    sb.append("</center>\n");
+    return sb.toString();
+  }
+
 }

Modified: incubator/hama/trunk/core/src/main/webapp/bspmaster/machines.jsp
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/main/webapp/bspmaster/machines.jsp?rev=1171537&r1=1171536&r2=1171537&view=diff
==============================================================================
--- incubator/hama/trunk/core/src/main/webapp/bspmaster/machines.jsp (original)
+++ incubator/hama/trunk/core/src/main/webapp/bspmaster/machines.jsp Fri Sep 16 13:05:11 2011
@@ -26,23 +26,6 @@
 	String trackerName = tracker.getBSPMasterName();
 	String type = request.getParameter("type");
 %>
-<%!public void generateGroomsTable(JspWriter out, String type,
-			ClusterStatus status, BSPMaster master) throws IOException {
-	out.print("<center>\n");
-    out.print("<table border=\"2\" cellpadding=\"5\" cellspacing=\"2\">\n");
-    out.print("<tr><td align=\"center\" colspan=\"6\"><b>Groom Servers</b></td></tr>\n");
-    out.print("<tr><td><b>Name</b></td>" + "<td><b>Host</b></td>"
-        + "<td><b># running tasks</b></td></tr>\n");
-    for (Map.Entry<String, String> entry : status.getActiveGroomNames()
-        .entrySet()) {
-      out.print("<tr><td><a href=\"http://");
-      out.print(entry.getKey() + ":" + master.getHttpPort() + "/\">");
-      out.print(entry.getValue() + "</a></td><td>");
-      out.print(entry.getValue() + "</td>" + "<td>" + 1 + "</td></tr>\n");
-    }
-    out.print("</table>\n");
-    out.print("</center>\n");
-  }%>
 
 <html>
 
@@ -53,9 +36,6 @@
 
 <h2>Grooms</h2>
 <%
-  generateGroomsTable(out, type, status, tracker);
-%>
-
-<%
+  out.println(BSPServletUtil.generateGroomsTable(type, status, tracker));
   out.println(BSPServletUtil.htmlFooter());
 %>
\ No newline at end of file

Modified: incubator/hama/trunk/core/src/test/java/org/apache/hama/bsp/TestClusterStatus.java
URL: http://svn.apache.org/viewvc/incubator/hama/trunk/core/src/test/java/org/apache/hama/bsp/TestClusterStatus.java?rev=1171537&r1=1171536&r2=1171537&view=diff
==============================================================================
--- incubator/hama/trunk/core/src/test/java/org/apache/hama/bsp/TestClusterStatus.java (original)
+++ incubator/hama/trunk/core/src/test/java/org/apache/hama/bsp/TestClusterStatus.java Fri
Sep 16 13:05:11 2011
@@ -18,16 +18,16 @@
 package org.apache.hama.bsp;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Random;
 
 import junit.framework.TestCase;
 
 import org.apache.hadoop.io.DataInputBuffer;
 import org.apache.hadoop.io.DataOutputBuffer;
-import org.apache.hama.bsp.BSPMaster;
-import org.apache.hama.bsp.ClusterStatus;
 
 public class TestClusterStatus extends TestCase {
   Random rnd = new Random();
@@ -41,13 +41,13 @@ public class TestClusterStatus extends T
     DataInputBuffer in = new DataInputBuffer();
 
     ClusterStatus status1;
-    Map<String, String> grooms = new HashMap<String, String>();
+    Map<String, GroomServerStatus> grooms = new HashMap<String, GroomServerStatus>();
 
     for (int i = 0; i < 10; i++) {
       int num = rnd.nextInt();
       String groomName = "groom_" + num;
       String peerName = "peerhost:" + num;
-      grooms.put(groomName, peerName);
+      grooms.put(groomName, new GroomServerStatus(peerName, new ArrayList<TaskStatus>(0),
25, 2));
     }
 
     int tasks = rnd.nextInt(100);
@@ -61,11 +61,16 @@ public class TestClusterStatus extends T
 
     ClusterStatus status2 = new ClusterStatus();
     status2.readFields(in);
+    
+    for(Entry<String, GroomServerStatus> entry : status2.getActiveGroomServerStatus().entrySet()){
+      assertEquals(entry.getValue().getMaxTasks(),2);
+      assertEquals(entry.getValue().getFailures(),25);
+    }
 
-    Map<String, String> grooms_s = new HashMap<String, String>(status1
-        .getActiveGroomNames());
-    Map<String, String> grooms_o = new HashMap<String, String>(status2
-        .getActiveGroomNames());
+    Map<String, String> grooms_s = new HashMap<String, String>(
+        status1.getActiveGroomNames());
+    Map<String, String> grooms_o = new HashMap<String, String>(
+        status2.getActiveGroomNames());
 
     assertEquals(status1.getGroomServers(), status2.getGroomServers());
 



Mime
View raw message