carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vimal...@apache.org
Subject [1/2] incubator-carbondata git commit: clear query statistics map
Date Tue, 06 Sep 2016 06:45:27 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 713d2c110 -> d174d59e1


clear query statistics map


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/dc5dfebd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/dc5dfebd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/dc5dfebd

Branch: refs/heads/master
Commit: dc5dfebd04834496e43ed25e6245db23aae66ada
Parents: 713d2c1
Author: foryou2030 <foryou2030@126.com>
Authored: Fri Sep 2 18:22:03 2016 +0800
Committer: Vimal Das Kammath <vkammath@VIMALDK-M01.vmware.com>
Committed: Tue Sep 6 12:12:37 2016 +0530

----------------------------------------------------------------------
 .../DriverQueryStatisticsRecorder.java          | 234 +++++++++++--------
 .../QueryStatisticsConstants.java               |   3 +
 .../apache/carbondata/core/util/CarbonUtil.java |  14 ++
 3 files changed, 158 insertions(+), 93 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dc5dfebd/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorder.java
b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorder.java
index 9e83c22..534852f 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorder.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/DriverQueryStatisticsRecorder.java
@@ -19,15 +19,19 @@
 package org.apache.carbondata.core.carbon.querystatistics;
 
 import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 
+import static org.apache.carbondata.core.util.CarbonUtil.add;
 import static org.apache.carbondata.core.util.CarbonUtil.printLine;
 
+import org.apache.commons.lang3.StringUtils;
+
 /**
  * Class will be used to record and log the query statistics
  */
@@ -47,7 +51,8 @@ public class DriverQueryStatisticsRecorder {
   private static final Object lock = new Object();
 
   private DriverQueryStatisticsRecorder() {
-    queryStatisticsMap = new HashMap<String, List<QueryStatistic>>();
+    // use ConcurrentHashMap, it is thread-safe
+    queryStatisticsMap = new ConcurrentHashMap<String, List<QueryStatistic>>();
   }
 
   private static DriverQueryStatisticsRecorder carbonLoadStatisticsImplInstance =
@@ -78,9 +83,32 @@ public class DriverQueryStatisticsRecorder {
    */
   public void logStatisticsAsTableDriver() {
     synchronized (lock) {
-      String tableInfo = collectDriverStatistics();
-      if (null != tableInfo) {
-        LOGGER.statistic(tableInfo);
+      Iterator<Map.Entry<String, List<QueryStatistic>>> entries =
+              queryStatisticsMap.entrySet().iterator();
+      while (entries.hasNext()) {
+        Map.Entry<String, List<QueryStatistic>> entry = entries.next();
+        String queryId = entry.getKey();
+        // clear the unknown query statistics
+        if(StringUtils.isEmpty(queryId)) {
+          entries.remove();
+        } else {
+          // clear the timeout query statistics
+          long interval = System.nanoTime() - Long.parseLong(queryId);
+          if (interval > QueryStatisticsConstants.CLEAR_STATISTICS_TIMEOUT) {
+            entries.remove();
+          } else {
+            // print sql_parse_t,load_meta_t,block_allocation_t,block_identification_t
+            // or just print block_allocation_t,block_identification_t
+            if (entry.getValue().size() >= 2) {
+              String tableInfo = collectDriverStatistics(entry.getValue(), queryId);
+              if (null != tableInfo) {
+                LOGGER.statistic(tableInfo);
+                // clear the statistics that has been printed
+                entries.remove();
+              }
+            }
+          }
+        }
       }
     }
   }
@@ -88,96 +116,116 @@ public class DriverQueryStatisticsRecorder {
   /**
    * Below method will parse queryStatisticsMap and put time into table
    */
-  public String collectDriverStatistics() {
-    for (String key: queryStatisticsMap.keySet()) {
-      try {
-        // TODO: get the finished query, and print Statistics
-        if (queryStatisticsMap.get(key).size() > 3) {
-          String sql_parse_time = "";
-          String load_meta_time = "";
-          String block_allocation_time = "";
-          String block_identification_time = "";
-          Double driver_part_time_tmp = 0.0;
-          String splitChar = " ";
-          // get statistic time from the QueryStatistic
-          for (QueryStatistic statistic : queryStatisticsMap.get(key)) {
-            switch (statistic.getMessage()) {
-              case QueryStatisticsConstants.SQL_PARSE:
-                sql_parse_time += statistic.getTimeTaken() + splitChar;
-                driver_part_time_tmp += statistic.getTimeTaken();
-                break;
-              case QueryStatisticsConstants.LOAD_META:
-                load_meta_time += statistic.getTimeTaken() + splitChar;
-                driver_part_time_tmp += statistic.getTimeTaken();
-                break;
-              case QueryStatisticsConstants.BLOCK_ALLOCATION:
-                block_allocation_time += statistic.getTimeTaken() + splitChar;
-                driver_part_time_tmp += statistic.getTimeTaken();
-                break;
-              case QueryStatisticsConstants.BLOCK_IDENTIFICATION:
-                block_identification_time += statistic.getTimeTaken() + splitChar;
-                driver_part_time_tmp += statistic.getTimeTaken();
-                break;
-              default:
-                break;
-            }
-          }
-          String driver_part_time = driver_part_time_tmp + splitChar;
-          // structure the query statistics info table
-          StringBuilder tableInfo = new StringBuilder();
-          int len1 = 8;
-          int len2 = 20;
-          int len3 = 21;
-          int len4 = 22;
-          String line = "+" + printLine("-", len1) + "+" + printLine("-", len2) + "+" +
-              printLine("-", len3) + "+" + printLine("-", len4) + "+";
-          String line2 = "|" + printLine(" ", len1) + "+" + printLine("-", len2) + "+" +
-              printLine(" ", len3) + "+" + printLine("-", len4) + "+";
-          // table header
-          tableInfo.append(line).append("\n");
-          tableInfo.append("|" + printLine(" ", (len1 - "Module".length())) + "Module" +
"|" +
-              printLine(" ", (len2 - "Operation Step".length())) + "Operation Step" + "|"
+
-              printLine(" ", (len3 + len4 + 1 - "Query Cost".length())) +
-              "Query Cost" + "|" + "\n");
-          // driver part
-          tableInfo.append(line).append("\n");
-          tableInfo.append("|" + printLine(" ", len1) + "|" +
-              printLine(" ", (len2 - "SQL parse".length())) + "SQL parse" + "|" +
-              printLine(" ", len3) + "|" +
-              printLine(" ", (len4 - sql_parse_time.length())) + sql_parse_time + "|" + "\n");
-          tableInfo.append(line2).append("\n");
-          tableInfo.append("|" +printLine(" ", (len1 - "Driver".length())) + "Driver" + "|"
+
-              printLine(" ", (len2 - "Load meta data".length())) + "Load meta data" + "|"
+
-              printLine(" ", (len3 - driver_part_time.length())) + driver_part_time + "|"
+
-              printLine(" ", (len4 - load_meta_time.length())) +
-              load_meta_time + "|" + "\n");
-          tableInfo.append(line2).append("\n");
-          tableInfo.append("|" +
-              printLine(" ", (len1 - "Part".length())) + "Part" + "|" +
-              printLine(" ", (len2 - "Block allocation".length())) +
-              "Block allocation" + "|" +
-              printLine(" ", len3) + "|" +
-              printLine(" ", (len4 - block_allocation_time.length())) +
-              block_allocation_time + "|" + "\n");
-          tableInfo.append(line2).append("\n");
-          tableInfo.append("|" +
-              printLine(" ", len1) + "|" +
-              printLine(" ", (len2 - "Block identification".length())) +
-              "Block identification" + "|" +
-              printLine(" ", len3) + "|" +
-              printLine(" ", (len4 - block_identification_time.length())) +
-              block_identification_time + "|" + "\n");
-          tableInfo.append(line).append("\n");
-
-          // once the statistics be printed, remove it from the map
-          queryStatisticsMap.remove(key);
-          // show query statistic as "query id" + "table"
-          return "Print query statistic for query id: " + key + "\n" + tableInfo.toString();
+  public String collectDriverStatistics(List<QueryStatistic> statisticsList, String
queryId) {
+    String sql_parse_time = "";
+    String load_meta_time = "";
+    String block_allocation_time = "";
+    String block_identification_time = "";
+    Double driver_part_time_tmp = 0.0;
+    Double driver_part_time_tmp2 = 0.0;
+    String splitChar = " ";
+    try {
+      // get statistic time from the QueryStatistic
+      for (QueryStatistic statistic : statisticsList) {
+        switch (statistic.getMessage()) {
+          case QueryStatisticsConstants.SQL_PARSE:
+            sql_parse_time += statistic.getTimeTaken() + splitChar;
+            driver_part_time_tmp = add(driver_part_time_tmp, statistic.getTimeTaken());
+            break;
+          case QueryStatisticsConstants.LOAD_META:
+            load_meta_time += statistic.getTimeTaken() + splitChar;
+            driver_part_time_tmp = add(driver_part_time_tmp, statistic.getTimeTaken());
+            break;
+          case QueryStatisticsConstants.BLOCK_ALLOCATION:
+            block_allocation_time += statistic.getTimeTaken() + splitChar;
+            driver_part_time_tmp = add(driver_part_time_tmp, statistic.getTimeTaken());
+            driver_part_time_tmp2 = add(driver_part_time_tmp2, statistic.getTimeTaken());
+            break;
+          case QueryStatisticsConstants.BLOCK_IDENTIFICATION:
+            block_identification_time += statistic.getTimeTaken() + splitChar;
+            driver_part_time_tmp = add(driver_part_time_tmp, statistic.getTimeTaken());
+            driver_part_time_tmp2 = add(driver_part_time_tmp2, statistic.getTimeTaken());
+            break;
+          default:
+            break;
         }
-      } catch (Exception ex) {
-        return "Put statistics into table failed, catch exception: " + ex.getMessage();
       }
+      String driver_part_time = driver_part_time_tmp + splitChar;
+      // structure the query statistics info table
+      StringBuilder tableInfo = new StringBuilder();
+      int len1 = 8;
+      int len2 = 20;
+      int len3 = 21;
+      int len4 = 22;
+      String line = "+" + printLine("-", len1) + "+" + printLine("-", len2) + "+" +
+          printLine("-", len3) + "+" + printLine("-", len4) + "+";
+      String line2 = "|" + printLine(" ", len1) + "+" + printLine("-", len2) + "+" +
+          printLine(" ", len3) + "+" + printLine("-", len4) + "+";
+      // table header
+      tableInfo.append(line).append("\n");
+      tableInfo.append("|" + printLine(" ", (len1 - "Module".length())) + "Module" + "|"
+
+          printLine(" ", (len2 - "Operation Step".length())) + "Operation Step" + "|" +
+          printLine(" ", (len3 + len4 + 1 - "Query Cost".length())) + "Query Cost" + "|"
+ "\n");
+      tableInfo.append(line).append("\n");
+      // print sql_parse_t,load_meta_t,block_allocation_t,block_identification_t
+      if (!StringUtils.isEmpty(sql_parse_time) &&
+          !StringUtils.isEmpty(load_meta_time) &&
+          !StringUtils.isEmpty(block_allocation_time) &&
+          !StringUtils.isEmpty(block_identification_time)) {
+        tableInfo.append("|" + printLine(" ", len1) + "|" +
+            printLine(" ", (len2 - "SQL parse".length())) + "SQL parse" + "|" +
+            printLine(" ", len3) + "|" +
+            printLine(" ", (len4 - sql_parse_time.length())) + sql_parse_time + "|" + "\n");
+        tableInfo.append(line2).append("\n");
+        tableInfo.append("|" + printLine(" ", (len1 - "Driver".length())) + "Driver" + "|"
+
+            printLine(" ", (len2 - "Load meta data".length())) + "Load meta data" + "|" +
+            printLine(" ", (len3 - driver_part_time.length())) + driver_part_time + "|" +
+            printLine(" ", (len4 - load_meta_time.length())) +
+            load_meta_time + "|" + "\n");
+        tableInfo.append(line2).append("\n");
+        tableInfo.append("|" + printLine(" ", (len1 - "Part".length())) + "Part" + "|" +
+            printLine(" ", (len2 - "Block allocation".length())) + "Block allocation" + "|"
+
+            printLine(" ", len3) + "|" +
+            printLine(" ", (len4 - block_allocation_time.length())) +
+            block_allocation_time + "|" + "\n");
+        tableInfo.append(line2).append("\n");
+        tableInfo.append("|" +
+            printLine(" ", len1) + "|" +
+            printLine(" ", (len2 - "Block identification".length())) +
+            "Block identification" + "|" +
+            printLine(" ", len3) + "|" +
+            printLine(" ", (len4 - block_identification_time.length())) +
+            block_identification_time + "|" + "\n");
+        tableInfo.append(line).append("\n");
+
+        // show query statistic as "query id" + "table"
+        return "Print query statistic for query id: " + queryId + "\n" + tableInfo.toString();
+      } else if (!StringUtils.isEmpty(block_allocation_time) &&
+          !StringUtils.isEmpty(block_identification_time)) {
+        // when we can't get sql parse time, we only print the last two
+        driver_part_time = driver_part_time_tmp2 + splitChar;
+        tableInfo.append("|" + printLine(" ", (len1 - "Driver".length())) + "Driver" + "|"
+
+            printLine(" ", (len2 - "Block allocation".length())) + "Block allocation" + "|"
+
+            printLine(" ", (len3 - driver_part_time.length())) + driver_part_time + "|" +
+            printLine(" ", (len4 - block_allocation_time.length())) +
+            block_allocation_time + "|" + "\n");
+        tableInfo.append(line2).append("\n");
+        tableInfo.append("|" +
+            printLine(" ", (len1 - "Part".length())) + "Part" + "|" +
+            printLine(" ", (len2 - "Block identification".length())) +
+            "Block identification" + "|" +
+            printLine(" ", len3) + "|" +
+            printLine(" ", (len4 - block_identification_time.length())) +
+            block_identification_time + "|" + "\n");
+        tableInfo.append(line).append("\n");
+
+        // show query statistic as "query id" + "table"
+        return "Print query statistic for query id: " + queryId + "\n" + tableInfo.toString();
+      }
+
+      return null;
+    } catch (Exception ex) {
+      return "Put statistics into table failed, catch exception: " + ex.getMessage();
     }
-    return null;
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dc5dfebd/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsConstants.java
b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsConstants.java
index 540cf1e..118dadb 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/querystatistics/QueryStatisticsConstants.java
@@ -48,6 +48,9 @@ public interface QueryStatisticsConstants {
 
   String RESULT_SIZE = "The size of query result";
 
+  // clear no-use statistics timeout
+  long CLEAR_STATISTICS_TIMEOUT = 60 * 1000 * 1000000L;
+
 }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/dc5dfebd/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 77d041c..c98456e 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -28,6 +28,7 @@ import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.math.BigDecimal;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
 import java.nio.charset.Charset;
@@ -1430,5 +1431,18 @@ public final class CarbonUtil {
     }
     return builder.toString();
   }
+
+  /**
+   * Below method will for double plus double
+   *
+   * @param v1
+   * @param v2
+   */
+  public static double add(double v1, double v2)
+  {
+    BigDecimal b1 = new BigDecimal(Double.toString(v1));
+    BigDecimal b2 = new BigDecimal(Double.toString(v2));
+    return  b1.add(b2).doubleValue();
+  }
 }
 


Mime
View raw message