carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [1/2] incubator-carbondata git commit: Segment folders are deleted for multiple loads
Date Thu, 07 Jul 2016 21:31:08 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 61a4355f6 -> 8dfa67238


Segment folders are deleted for multiple loads


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/a156737e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/a156737e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/a156737e

Branch: refs/heads/master
Commit: a156737e1dc992f42a796d15e1891a2e6ba5388e
Parents: 61a4355
Author: ravipesala <ravi.pesala@gmail.com>
Authored: Fri Jul 8 00:04:14 2016 +0530
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Fri Jul 8 00:04:14 2016 +0530

----------------------------------------------------------------------
 .../carbondata/spark/load/CarbonLoaderUtil.java | 23 ++++++++++++--------
 1 file changed, 14 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/a156737e/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
b/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
index af880f5..52f8fb7 100644
--- a/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
+++ b/integration/spark/src/main/java/org/carbondata/spark/load/CarbonLoaderUtil.java
@@ -307,25 +307,30 @@ public final class CarbonLoaderUtil {
     String metaDataLocation = carbonTable.getMetaDataFilepath();
     SegmentStatusManager segmentStatusManager =
         new SegmentStatusManager(carbonTable.getAbsoluteTableIdentifier());
-    LoadMetadataDetails[] details = segmentStatusManager.readLoadMetadata(metaDataLocation);
+    final LoadMetadataDetails[] details = segmentStatusManager.readLoadMetadata(metaDataLocation);
     CarbonTablePath carbonTablePath = CarbonStorePath
         .getCarbonTablePath(loadModel.getStorePath(), carbonTable.getCarbonTableIdentifier());
-    final List<String> loadFolders = new ArrayList<String>();
-    for (LoadMetadataDetails loadMetadata : details) {
-      loadFolders.add(carbonTablePath
-          .getCarbonDataDirectoryPath(loadMetadata.getPartitionCount(), loadMetadata.getLoadName())
-          .replace("\\", "/"));
-    }
 
     //delete folder which metadata no exist in tablestatus
     for (int i = 0; i < carbonTable.getPartitionCount(); i++) {
-      String partitionPath = carbonTablePath.getPartitionDir(i + "");
+      final String partitionCount = i + "";
+      String partitionPath = carbonTablePath.getPartitionDir(partitionCount);
       FileType fileType = FileFactory.getFileType(partitionPath);
       if (FileFactory.isFileExist(partitionPath, fileType)) {
         CarbonFile carbonFile = FileFactory.getCarbonFile(partitionPath, fileType);
         CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {
           @Override public boolean accept(CarbonFile path) {
-            return !loadFolders.contains(path.getAbsolutePath().replace("\\", "/"));
+            String segmentId =
+                CarbonTablePath.DataPathUtil.getSegmentId(path.getAbsolutePath() + "/dummy");
+            boolean found = false;
+            for (int j = 0; j < details.length; j++) {
+              if (details[j].getLoadName().equals(segmentId)
+                  && details[j].getPartitionCount().equals(partitionCount)) {
+                found = true;
+                break;
+              }
+            }
+            return !found;
           }
         });
         for (int k = 0; k < listFiles.length; k++) {


Mime
View raw message