carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [02/50] [abbrv] incubator-carbondata git commit: Improve Carbon index file loading file in case of big cluster
Date Tue, 02 May 2017 14:12:26 GMT
Improve Carbon index file loading file in case of big cluster


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/8fc9c7bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/8fc9c7bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/8fc9c7bf

Branch: refs/heads/12-dev
Commit: 8fc9c7bf67c1e5678cfa955266dfe005fed4a5d5
Parents: 174bfad
Author: kumarvishal <kumarvishal.1802@gmail.com>
Authored: Fri Apr 21 20:42:10 2017 +0530
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Sat Apr 22 00:30:32 2017 +0530

----------------------------------------------------------------------
 .../apache/carbondata/core/util/CarbonUtil.java |  8 +++--
 .../core/util/path/CarbonTablePath.java         | 33 ++++++++++++++++++++
 2 files changed, 39 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8fc9c7bf/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 7d57fb8..8956549 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -990,7 +990,9 @@ public final class CarbonUtil {
     //TODO need to pass proper partition number when partiton will be supported
     String carbonIndexFilePath = carbonTablePath
         .getCarbonIndexFilePath(taskId, "0", tableBlockInfoList.get(0).getSegmentId(),
-            bucketNumber);
+            bucketNumber, CarbonTablePath.DataFileUtil
+                .getTimeStampFromFileName(tableBlockInfoList.get(0).getFilePath()),
+            tableBlockInfoList.get(0).getVersion());
     CarbonFile carbonFile = FileFactory
         .getCarbonFile(carbonIndexFilePath, FileFactory.getFileType(carbonIndexFilePath));
     // in case of carbonIndex file whole file is meta only so reading complete file.
@@ -1232,7 +1234,9 @@ public final class CarbonUtil {
     //TODO need to pass proper partition number when partiton will be supported
     String carbonIndexFilePath = carbonTablePath
         .getCarbonIndexFilePath(taskId, "0", tableBlockInfoList.get(0).getSegmentId(),
-            bucketNumber);
+            bucketNumber, CarbonTablePath.DataFileUtil
+                .getTimeStampFromFileName(tableBlockInfoList.get(0).getFilePath()),
+            tableBlockInfoList.get(0).getVersion());
     DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
     // read the index info and return
     return fileFooterConverter.getIndexInfo(carbonIndexFilePath, tableBlockInfoList);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8fc9c7bf/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
index 3e81fbb..918772a 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/path/CarbonTablePath.java
@@ -23,6 +23,7 @@ import org.apache.carbondata.core.datastore.filesystem.CarbonFile;
 import org.apache.carbondata.core.datastore.filesystem.CarbonFileFilter;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
 import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
+import org.apache.carbondata.core.metadata.ColumnarFormatVersion;
 
 import org.apache.hadoop.fs.Path;
 
@@ -274,6 +275,38 @@ public class CarbonTablePath extends Path {
           + "]");
     }
   }
+
+  /**
+   * Below method will be used to get the carbon index file path
+   * @param taskId
+   *        task id
+   * @param partitionId
+   *        partition id
+   * @param segmentId
+   *        segment id
+   * @param bucketNumber
+   *        bucket number
+   * @param timeStamp
+   *        timestamp
+   * @return carbon index file path
+   */
+  public String getCarbonIndexFilePath(String taskId, String partitionId, String segmentId,
+      String bucketNumber, String timeStamp, ColumnarFormatVersion columnarFormatVersion)
{
+    switch (columnarFormatVersion) {
+      case V1:
+      case V2:
+        return getCarbonIndexFilePath(taskId, partitionId, segmentId, bucketNumber);
+      default:
+        String segmentDir = getSegmentDir(partitionId, segmentId);
+        return segmentDir + File.separator + getCarbonIndexFileName(taskId,
+            Integer.parseInt(bucketNumber), timeStamp);
+    }
+  }
+
+  private static String getCarbonIndexFileName(String taskNo, int bucketNumber,
+      String factUpdatedtimeStamp) {
+    return taskNo + "-" + bucketNumber + "-" + factUpdatedtimeStamp + INDEX_FILE_EXT;
+  }
   /**
    * Below method will be used to get the index file present in the segment folder
    * based on task id


Mime
View raw message