carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vimal...@apache.org
Subject [2/3] incubator-carbondata git commit: Problem: Code contains currentRestructureNumber variable which is not used in the code now
Date Sat, 03 Sep 2016 11:41:46 GMT
Problem: Code contains currentRestructureNumber variable which is not used in the code now

Impact area: Data load flow

Fix: Remove the usage of currentRestructureNumber variable everywhere in the code


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/5aedf544
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/5aedf544
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/5aedf544

Branch: refs/heads/master
Commit: 5aedf54437757e645bb360a913c424418c433b45
Parents: ec58755
Author: manishgupta88 <tomanishgupta18@gmail.com>
Authored: Fri Sep 2 16:06:05 2016 +0530
Committer: Vimal Das Kammath <vkammath@VIMALDK-M01.vmware.com>
Committed: Sat Sep 3 17:09:33 2016 +0530

----------------------------------------------------------------------
 .../apache/carbondata/core/util/CarbonUtil.java |  76 ---
 .../hadoop/test/util/StoreCreator.java          |  36 +-
 .../carbondata/spark/load/CarbonLoaderUtil.java | 526 +------------------
 .../spark/load/DeleteLoadFolders.java           |  96 ----
 .../carbondata/spark/util/LoadMetadataUtil.java |  50 --
 .../spark/rdd/CarbonDataLoadRDD.scala           |  33 +-
 .../spark/rdd/CarbonDataRDDFactory.scala        |  61 +--
 .../spark/rdd/CarbonDeleteLoadByDateRDD.scala   |   3 +-
 .../processing/csvreaderstep/CsvInputMeta.java  |  26 -
 .../graphgenerator/GraphGenerator.java          |  24 +-
 .../configuration/GraphConfigurationInfo.java   |  12 -
 .../processing/mdkeygen/MDKeyGenStepMeta.java   |  24 -
 .../merger/step/CarbonSliceMergerStepMeta.java  |  24 -
 .../sortdatastep/SortKeyStepMeta.java           |  25 -
 .../csvbased/CarbonCSVBasedSeqGenMeta.java      |  17 -
 15 files changed, 30 insertions(+), 1003 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 6f2bcc8..77d041c 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -64,7 +64,6 @@ import org.apache.carbondata.core.datastorage.store.columnar.UnBlockIndexer;
 import org.apache.carbondata.core.datastorage.store.compression.MeasureMetaDataModel;
 import org.apache.carbondata.core.datastorage.store.compression.ValueCompressionModel;
 import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
 import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
 import org.apache.carbondata.core.keygenerator.mdkey.NumberCompressor;
 import org.apache.carbondata.core.metadata.ValueEncoderMeta;
@@ -148,67 +147,6 @@ public final class CarbonUtil {
   }
 
   /**
-   * This method checks whether Restructure Folder exists or not
-   * and if not exist then return the number with which folder need to created.
-   *
-   * @param baseStorePath -
-   *                      baselocation where folder will be created.
-   * @return counter
-   * counter with which folder will be created.
-   */
-  public static int checkAndReturnCurrentRestructFolderNumber(String baseStorePath,
-      final String filterType, final boolean isDirectory) {
-    if (null == baseStorePath || 0 == baseStorePath.length()) {
-      return -1;
-    }
-    // change the slashes to /
-    baseStorePath = baseStorePath.replace("\\", "/");
-
-    // check if string wnds with / then remove that.
-    if (baseStorePath.charAt(baseStorePath.length() - 1) == '/') {
-      baseStorePath = baseStorePath.substring(0, baseStorePath.lastIndexOf("/"));
-    }
-    int retValue = createBaseStoreFolders(baseStorePath);
-    if (-1 == retValue) {
-      return retValue;
-    }
-
-    CarbonFile carbonFile =
-        FileFactory.getCarbonFile(baseStorePath, FileFactory.getFileType(baseStorePath));
-
-    // List of directories
-    CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {
-      @Override public boolean accept(CarbonFile pathname) {
-        if (isDirectory && pathname.isDirectory()) {
-          if (pathname.getAbsolutePath().indexOf(filterType) > -1) {
-            return true;
-          }
-        } else {
-          if (pathname.getAbsolutePath().indexOf(filterType) > -1) {
-            return true;
-          }
-        }
-
-        return false;
-      }
-    });
-
-    int counter = -1;
-
-    // if no folder exists then return -1
-    if (listFiles.length == 0) {
-      return counter;
-    }
-
-    counter = findCounterValue(filterType, listFiles, counter);
-    return counter;
-  }
-
-  public static int checkAndReturnCurrentLoadFolderNumber(String baseStorePath) {
-    return checkAndReturnCurrentRestructFolderNumber(baseStorePath, "Load_", true);
-  }
-
-  /**
    * @param filterType
    * @param listFiles
    * @param counter
@@ -870,20 +808,6 @@ public final class CarbonUtil {
   }
 
   /**
-   * @param location
-   * @param factTableName
-   * @return
-   */
-  public static int getRestructureNumber(String location, String factTableName) {
-    String restructName =
-        location.substring(location.indexOf(CarbonCommonConstants.RESTRUCTRE_FOLDER));
-    int factTableIndex = restructName.indexOf(factTableName) - 1;
-    String restructNumber =
-        restructName.substring(CarbonCommonConstants.RESTRUCTRE_FOLDER.length(), factTableIndex);
-    return Integer.parseInt(restructNumber);
-  }
-
-  /**
    * Below method will be used to get the aggregator type
    * CarbonCommonConstants.SUM_COUNT_VALUE_MEASURE will return when value is double measure
    * CarbonCommonConstants.BYTE_VALUE_MEASURE will be returned when value is byte array

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
index 7960940..a48e6ad 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
@@ -18,21 +18,7 @@
  */
 package org.apache.carbondata.hadoop.test.util;
 
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.UUID;
-
+import com.google.gson.Gson;
 import org.apache.carbondata.core.cache.Cache;
 import org.apache.carbondata.core.cache.CacheProvider;
 import org.apache.carbondata.core.cache.CacheType;
@@ -83,7 +69,9 @@ import org.apache.carbondata.processing.graphgenerator.GraphGenerator;
 import org.apache.carbondata.processing.graphgenerator.GraphGeneratorException;
 import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
 
-import com.google.gson.Gson;
+import java.io.*;
+import java.text.SimpleDateFormat;
+import java.util.*;
 
 /**
  * This class will create store file based on provided schema
@@ -124,7 +112,6 @@ public class StoreCreator {
           absoluteTableIdentifier.getStorePath());
 
       String kettleHomePath = "../processing/carbonplugins";
-      int currentRestructureNumber = 0;
       CarbonTable table = createTable();
       writeDictionary(factFilePath, table);
       CarbonDataLoadSchema schema = new CarbonDataLoadSchema(table);
@@ -137,8 +124,7 @@ public class StoreCreator {
       loadModel.setFactFilePath(factFilePath);
       loadModel.setLoadMetadataDetails(new ArrayList<LoadMetadataDetails>());
 
-      executeGraph(loadModel, absoluteTableIdentifier.getStorePath(), kettleHomePath,
-          currentRestructureNumber);
+      executeGraph(loadModel, absoluteTableIdentifier.getStorePath(), kettleHomePath);
     } catch (Exception e) {
       e.printStackTrace();
     }
@@ -327,11 +313,10 @@ public class StoreCreator {
    * @param loadModel
    * @param storeLocation
    * @param kettleHomePath
-   * @param currentRestructNumber
    * @throws Exception
    */
-  public static void executeGraph(LoadModel loadModel, String storeLocation, String kettleHomePath,
-      int currentRestructNumber) throws Exception {
+  public static void executeGraph(LoadModel loadModel, String storeLocation, String kettleHomePath)
+      throws Exception {
     System.setProperty("KETTLE_HOME", kettleHomePath);
     new File(storeLocation).mkdirs();
     String outPutLoc = storeLocation + "/etl";
@@ -372,7 +357,7 @@ public class StoreCreator {
     info.setTableName(tableName);
 
     generateGraph(schmaModel, info, loadModel.getTableName(), "0", loadModel.getSchema(), null,
-        currentRestructNumber, loadModel.getLoadMetadataDetails());
+        loadModel.getLoadMetadataDetails());
 
     DataGraphExecuter graphExecuter = new DataGraphExecuter(schmaModel);
     graphExecuter
@@ -460,13 +445,12 @@ public class StoreCreator {
    * @param partitionID
    * @param schema
    * @param factStoreLocation
-   * @param currentRestructNumber
    * @param loadMetadataDetails
    * @throws GraphGeneratorException
    */
   private static void generateGraph(IDataProcessStatus schmaModel, SchemaInfo info,
       String tableName, String partitionID, CarbonDataLoadSchema schema, String factStoreLocation,
-      int currentRestructNumber, List<LoadMetadataDetails> loadMetadataDetails)
+      List<LoadMetadataDetails> loadMetadataDetails)
       throws GraphGeneratorException {
     DataLoadModel model = new DataLoadModel();
     model.setCsvLoad(null != schmaModel.getCsvFilePath() || null != schmaModel.getFilesToProcess());
@@ -489,7 +473,7 @@ public class StoreCreator {
         .getProperty("store_output_location", "../carbon-store/system/carbon/etl");
     GraphGenerator generator =
         new GraphGenerator(model, hdfsReadMode, partitionID, factStoreLocation,
-            currentRestructNumber, allocate, schema, "0", outputLocation);
+            allocate, schema, "0", outputLocation);
     generator.generateGraph();
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java b/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
index 54c8f5e..246597c 100644
--- a/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
+++ b/integration/spark/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
@@ -19,7 +19,6 @@
 package org.apache.carbondata.spark.load;
 
 import java.io.BufferedWriter;
-import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.File;
 import java.io.IOException;
@@ -55,8 +54,6 @@ import org.apache.carbondata.core.carbon.datastore.block.TableBlockInfo;
 import org.apache.carbondata.core.carbon.metadata.CarbonMetadata;
 import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
 import org.apache.carbondata.core.carbon.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
-import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure;
 import org.apache.carbondata.core.carbon.path.CarbonStorePath;
 import org.apache.carbondata.core.carbon.path.CarbonTablePath;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
@@ -85,9 +82,6 @@ import org.apache.carbondata.spark.merger.NodeBlockRelation;
 import org.apache.carbondata.spark.merger.NodeMultiBlockRelation;
 
 import com.google.gson.Gson;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
 import org.apache.spark.SparkConf;
 import org.apache.spark.util.Utils;
 
@@ -112,19 +106,12 @@ public final class CarbonLoaderUtil {
     }
   }
 
-  /**
-   * dfs.bytes-per-checksum
-   * HDFS checksum length, block size for a file should be exactly divisible
-   * by this value
-   */
-  private static final int HDFS_CHECKSUM_LENGTH = 512;
-
   private CarbonLoaderUtil() {
 
   }
 
   private static void generateGraph(IDataProcessStatus schmaModel, SchemaInfo info,
-      int currentRestructNumber, CarbonLoadModel loadModel, String outputLocation)
+      CarbonLoadModel loadModel, String outputLocation)
       throws GraphGeneratorException {
     DataLoadModel model = new DataLoadModel();
     model.setCsvLoad(null != schmaModel.getCsvFilePath() || null != schmaModel.getFilesToProcess());
@@ -146,13 +133,13 @@ public final class CarbonLoaderUtil {
         schmaModel.getCsvFilePath() != null && schmaModel.getCsvFilePath().startsWith("hdfs:");
     int allocate = null != schmaModel.getCsvFilePath() ? 1 : schmaModel.getFilesToProcess().size();
     GraphGenerator generator = new GraphGenerator(model, hdfsReadMode, loadModel.getPartitionId(),
-        loadModel.getStorePath(), currentRestructNumber, allocate,
+        loadModel.getStorePath(), allocate,
         loadModel.getCarbonDataLoadSchema(), loadModel.getSegmentId(), outputLocation);
     generator.generateGraph();
   }
 
   public static void executeGraph(CarbonLoadModel loadModel, String storeLocation,
-      String hdfsStoreLocation, String kettleHomePath, int currentRestructNumber) throws Exception {
+      String hdfsStoreLocation, String kettleHomePath) throws Exception {
     System.setProperty("KETTLE_HOME", kettleHomePath);
     if (!new File(storeLocation).mkdirs()) {
       LOGGER.error("Error while creating the temp store path: " + storeLocation);
@@ -202,7 +189,7 @@ public final class CarbonLoaderUtil {
     info.setComplexDelimiterLevel2(loadModel.getComplexDelimiterLevel2());
     info.setSerializationNullFormat(loadModel.getSerializationNullFormat());
 
-    generateGraph(schmaModel, info, currentRestructNumber, loadModel, outPutLoc);
+    generateGraph(schmaModel, info, loadModel, outPutLoc);
 
     DataGraphExecuter graphExecuter = new DataGraphExecuter(schmaModel);
     graphExecuter
@@ -210,89 +197,11 @@ public final class CarbonLoaderUtil {
             info, loadModel.getPartitionId(), loadModel.getCarbonDataLoadSchema());
   }
 
-  public static String[] getStorelocs(String databaseName, String tableName, String factTableName,
-      String hdfsStoreLocation, int currentRestructNumber) {
-    String[] loadFolders;
-
-    String baseStorelocation =
-        hdfsStoreLocation + File.separator + databaseName + File.separator + tableName;
-
-    String factStorepath =
-        baseStorelocation + File.separator + CarbonCommonConstants.RESTRUCTRE_FOLDER
-            + currentRestructNumber + File.separator + factTableName;
-
-    // Change to LOCAL for testing in local
-    CarbonFile file =
-        FileFactory.getCarbonFile(factStorepath, FileFactory.getFileType(factStorepath));
-
-    if (!file.exists()) {
-      return new String[0];
-    }
-    CarbonFile[] listFiles = file.listFiles(new CarbonFileFilter() {
-      @Override public boolean accept(CarbonFile path) {
-        return path.getName().startsWith(CarbonCommonConstants.LOAD_FOLDER) && !path.getName()
-            .endsWith(CarbonCommonConstants.FILE_INPROGRESS_STATUS);
-      }
-    });
-
-    loadFolders = new String[listFiles.length];
-    int count = 0;
-
-    for (CarbonFile loadFile : listFiles) {
-      loadFolders[count++] = loadFile.getAbsolutePath();
-    }
-
-    return loadFolders;
-  }
-
   public static List<String> addNewSliceNameToList(String newSlice, List<String> activeSlices) {
     activeSlices.add(newSlice);
     return activeSlices;
   }
 
-  public static String getAggLoadFolderLocation(String loadFolderName, String databaseName,
-      String tableName, String aggTableName, String hdfsStoreLocation, int currentRestructNumber) {
-    for (int i = currentRestructNumber; i >= 0; i--) {
-      String aggTableLocation =
-          getTableLocation(databaseName, tableName, aggTableName, hdfsStoreLocation, i);
-      String aggStorepath = aggTableLocation + File.separator + loadFolderName;
-      try {
-        if (FileFactory.isFileExist(aggStorepath, FileFactory.getFileType(aggStorepath))) {
-          return aggStorepath;
-        }
-      } catch (IOException e) {
-        LOGGER.error("Problem checking file existence :: " + e.getMessage());
-      }
-    }
-    return null;
-  }
-
-  public static String getTableLocation(String databaseName, String tableName, String aggTableName,
-      String hdfsStoreLocation, int currentRestructNumber) {
-    String baseStorelocation =
-        hdfsStoreLocation + File.separator + databaseName + File.separator + tableName;
-    String aggTableLocation =
-        baseStorelocation + File.separator + CarbonCommonConstants.RESTRUCTRE_FOLDER
-            + currentRestructNumber + File.separator + aggTableName;
-    return aggTableLocation;
-  }
-
-  public static void deleteTable(int partitionCount, String databaseName, String tableName,
-      String aggTableName, String hdfsStoreLocation, int currentRestructNumber) {
-    String aggTableLoc = null;
-    String partitionDatabaseName = null;
-    String partitionTableName = null;
-    for (int i = 0; i < partitionCount; i++) {
-      partitionDatabaseName = databaseName + '_' + i;
-      partitionTableName = tableName + '_' + i;
-      for (int j = currentRestructNumber; j >= 0; j--) {
-        aggTableLoc = getTableLocation(partitionDatabaseName, partitionTableName, aggTableName,
-            hdfsStoreLocation, j);
-        deleteStorePath(aggTableLoc);
-      }
-    }
-  }
-
   public static void deleteSegment(CarbonLoadModel loadModel, int currentLoad) {
     CarbonTable carbonTable = loadModel.getCarbonDataLoadSchema().getCarbonTable();
     CarbonTablePath carbonTablePath = CarbonStorePath
@@ -304,22 +213,6 @@ public final class CarbonLoaderUtil {
     }
   }
 
-  public static void deleteSlice(int partitionCount, String databaseName, String tableName,
-      String hdfsStoreLocation, int currentRestructNumber, String loadFolder) {
-    String tableLoc = null;
-    String partitionDatabaseName = null;
-    String partitionTableName = null;
-    for (int i = 0; i < partitionCount; i++) {
-      partitionDatabaseName = databaseName + '_' + i;
-      partitionTableName = tableName + '_' + i;
-      tableLoc =
-          getTableLocation(partitionDatabaseName, partitionTableName, tableName, hdfsStoreLocation,
-              currentRestructNumber);
-      tableLoc = tableLoc + File.separator + loadFolder;
-      deleteStorePath(tableLoc);
-    }
-  }
-
   public static void deletePartialLoadDataIfExist(CarbonLoadModel loadModel,
       final boolean isCompactionFlow) throws IOException {
     CarbonTable carbonTable = loadModel.getCarbonDataLoadSchema().getCarbonTable();
@@ -383,24 +276,6 @@ public final class CarbonLoaderUtil {
     }
   }
 
-  public static boolean isSliceValid(String loc, List<String> activeSlices,
-      List<String> updatedSlices, String factTableName) {
-    String loadFolderName = loc.substring(loc.indexOf(CarbonCommonConstants.LOAD_FOLDER));
-    String sliceNum = loadFolderName.substring(CarbonCommonConstants.LOAD_FOLDER.length());
-    if (activeSlices.contains(loadFolderName) || updatedSlices.contains(sliceNum)) {
-      String factFileLoc =
-          loc + File.separator + factTableName + "_0" + CarbonCommonConstants.FACT_FILE_EXT;
-      try {
-        if (FileFactory.isFileExist(factFileLoc, FileFactory.getFileType(factFileLoc))) {
-          return true;
-        }
-      } catch (IOException e) {
-        LOGGER.error("Problem checking file existence :: " + e.getMessage());
-      }
-    }
-    return false;
-  }
-
   public static List<String> getListOfValidSlices(LoadMetadataDetails[] details) {
     List<String> activeSlices =
         new ArrayList<String>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
@@ -439,24 +314,6 @@ public final class CarbonLoaderUtil {
     // TODO: Remove from memory
   }
 
-  public static void createEmptyLoadFolder(CarbonLoadModel model, String factLoadFolderLocation,
-      String hdfsStoreLocation, int currentRestructNumber) {
-    String loadFolderName = factLoadFolderLocation
-        .substring(factLoadFolderLocation.indexOf(CarbonCommonConstants.LOAD_FOLDER));
-    String aggLoadFolderLocation =
-        getTableLocation(model.getDatabaseName(), model.getTableName(), model.getAggTableName(),
-            hdfsStoreLocation, currentRestructNumber);
-    aggLoadFolderLocation = aggLoadFolderLocation + File.separator + loadFolderName;
-    FileType fileType = FileFactory.getFileType(hdfsStoreLocation);
-    try {
-      FileFactory.mkdirs(aggLoadFolderLocation, fileType);
-    } catch (IOException e) {
-      LOGGER
-          .error("Problem creating empty folder created for aggregation table: " + e.getMessage());
-    }
-    LOGGER.info("Empty folder created for aggregation table");
-  }
-
   /**
    * This method will delete the local data load folder location after data load is complete
    *
@@ -484,48 +341,6 @@ public final class CarbonLoaderUtil {
   }
 
   /**
-   * This method will copy the current segment load to cgiven carbon store path
-   *
-   * @param loadModel
-   * @param segmentName
-   * @param updatedSlices
-   * @throws IOException
-   * @throws CarbonUtilException
-   */
-  public static void copyCurrentLoadToHDFS(CarbonLoadModel loadModel, String segmentName,
-      List<String> updatedSlices) throws IOException, CarbonUtilException {
-    //Copy the current load folder to carbon store
-    boolean copyStore =
-        Boolean.valueOf(CarbonProperties.getInstance().getProperty("dataload.hdfs.copy", "true"));
-    String databaseName = loadModel.getDatabaseName();
-    String tableName = loadModel.getTableName();
-    String aggTableName = loadModel.getAggTableName();
-    if (copyStore) {
-      CarbonTableIdentifier carbonTableIdentifier =
-          loadModel.getCarbonDataLoadSchema().getCarbonTable().getCarbonTableIdentifier();
-      String segmentId = segmentName.substring(CarbonCommonConstants.LOAD_FOLDER.length());
-      // form carbon store location
-      String carbonStoreLocation = getStoreLocation(
-          CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION_HDFS),
-          carbonTableIdentifier, segmentId, loadModel.getPartitionId());
-      String tempLocationKey = databaseName + '_' + tableName;
-      // form local store location
-      String localStoreLocation = getStoreLocation(CarbonProperties.getInstance()
-              .getProperty(tempLocationKey, CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL),
-          carbonTableIdentifier, segmentId, loadModel.getPartitionId());
-      localStoreLocation = localStoreLocation + File.separator + loadModel.getTaskNo();
-      boolean isUpdate = false;
-      if (loadModel.isAggLoadRequest() && null != aggTableName) {
-        if (updatedSlices.contains(segmentId)) {
-          isUpdate = true;
-        }
-      }
-      copyToHDFS(carbonStoreLocation, localStoreLocation, isUpdate);
-      CarbonUtil.deleteFoldersAndFiles(new File[] { new File(localStoreLocation) });
-    }
-  }
-
-  /**
    * This method will get the store location for the given path, segemnt id and partition id
    *
    * @param storePath
@@ -543,205 +358,6 @@ public final class CarbonLoaderUtil {
   }
 
   /**
-   * This method will copy the carbon data files form local store segment folder
-   * to carbon store segment location
-   *
-   * @param carbonStoreLocation
-   * @param localStoreLocation
-   * @param isUpdate
-   * @throws IOException
-   */
-  private static void copyToHDFS(String carbonStoreLocation, String localStoreLocation,
-      boolean isUpdate) throws IOException {
-    //If the carbon store and the local store configured differently, then copy
-    if (carbonStoreLocation != null && !carbonStoreLocation.equals(localStoreLocation)) {
-      long copyStartTime = System.currentTimeMillis();
-      // isUpdate will be true only when the below 2 conditions are satisfied
-      // 1. slice is marked for update, 2. request is for aggregate table creation
-      if (isUpdate) {
-        renameFactFile(localStoreLocation);
-      }
-      LOGGER.info("Copying " + localStoreLocation + " --> " + carbonStoreLocation);
-      CarbonUtil.checkAndCreateFolder(carbonStoreLocation);
-      long blockSize = getBlockSize();
-      int bufferSize = CarbonCommonConstants.BYTEBUFFER_SIZE;
-      CarbonFile carbonFile = FileFactory
-          .getCarbonFile(localStoreLocation, FileFactory.getFileType(localStoreLocation));
-      CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {
-        @Override public boolean accept(CarbonFile path) {
-          return path.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT) && !path.getName()
-              .endsWith(CarbonCommonConstants.FILE_INPROGRESS_STATUS);
-        }
-      });
-      for (int i = 0; i < listFiles.length; i++) {
-        String localFilePath = listFiles[i].getCanonicalPath();
-        CarbonFile localCarbonFile =
-            FileFactory.getCarbonFile(localFilePath, FileFactory.getFileType(localFilePath));
-        String carbonFilePath = carbonStoreLocation + localFilePath
-            .substring(localFilePath.lastIndexOf(File.separator));
-        copyLocalFileToHDFS(carbonFilePath, localFilePath, bufferSize,
-            getMaxOfBlockAndFileSize(blockSize, localCarbonFile.getSize()));
-      }
-      LOGGER.info("Total copy time (ms):  " + (System.currentTimeMillis() - copyStartTime));
-    } else {
-      LOGGER.info("Separate carbon.storelocation.hdfs is not configured for carbon store path");
-    }
-  }
-
-  /**
-   * This method will return max of block size and file size
-   *
-   * @param blockSize
-   * @param fileSize
-   * @return
-   */
-  private static long getMaxOfBlockAndFileSize(long blockSize, long fileSize) {
-    long maxSize = blockSize;
-    if (fileSize > blockSize) {
-      maxSize = fileSize;
-    }
-    // block size should be exactly divisible by 512 which is  maintained by HDFS as bytes
-    // per checksum, dfs.bytes-per-checksum=512 must divide block size
-    long remainder = maxSize % HDFS_CHECKSUM_LENGTH;
-    if (remainder > 0) {
-      maxSize = maxSize + HDFS_CHECKSUM_LENGTH - remainder;
-    }
-    return maxSize;
-  }
-
-  /**
-   * This method will return the block size for file to be copied in HDFS
-   *
-   * @return
-   */
-  private static long getBlockSize() {
-    CarbonProperties carbonProperties = CarbonProperties.getInstance();
-    long blockSizeInBytes = Long.parseLong(carbonProperties
-        .getProperty(CarbonCommonConstants.MAX_FILE_SIZE,
-            CarbonCommonConstants.MAX_FILE_SIZE_DEFAULT_VAL))
-        * CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR
-        * CarbonCommonConstants.BYTE_TO_KB_CONVERSION_FACTOR * 1L;
-    return blockSizeInBytes;
-  }
-
-  /**
-   * This method will read the local carbon data file and write to carbon data file in HDFS
-   *
-   * @param carbonStoreFilePath
-   * @param localFilePath
-   * @param bufferSize
-   * @param blockSize
-   * @throws IOException
-   */
-  private static void copyLocalFileToHDFS(String carbonStoreFilePath, String localFilePath,
-      int bufferSize, long blockSize) throws IOException {
-    DataOutputStream dataOutputStream = null;
-    DataInputStream dataInputStream = null;
-    try {
-      LOGGER.debug(
-          "HDFS file block size for file: " + carbonStoreFilePath + " is " + blockSize + " (bytes");
-      dataOutputStream = FileFactory
-          .getDataOutputStream(carbonStoreFilePath, FileFactory.getFileType(carbonStoreFilePath),
-              bufferSize, blockSize);
-      dataInputStream = FileFactory
-          .getDataInputStream(localFilePath, FileFactory.getFileType(localFilePath), bufferSize);
-      IOUtils.copyBytes(dataInputStream, dataOutputStream, bufferSize);
-    } finally {
-      CarbonUtil.closeStreams(dataInputStream, dataOutputStream);
-    }
-  }
-
-  private static void renameFactFile(String localStoreLocation) {
-    FileType fileType = FileFactory.getFileType(localStoreLocation);
-    try {
-      if (FileFactory.isFileExist(localStoreLocation, fileType)) {
-        CarbonFile carbonFile = FileFactory.getCarbonFile(localStoreLocation, fileType);
-        CarbonFile[] listFiles = carbonFile.listFiles(new CarbonFileFilter() {
-          @Override public boolean accept(CarbonFile path) {
-            return path.getName().endsWith(CarbonCommonConstants.FACT_FILE_EXT) && !path.getName()
-                .endsWith(CarbonCommonConstants.FILE_INPROGRESS_STATUS);
-          }
-        });
-        for (int i = 0; i < listFiles.length; i++) {
-          carbonFile = listFiles[i];
-          String factFilePath = carbonFile.getCanonicalPath();
-          String changedFileName = factFilePath.replace(CarbonCommonConstants.FACT_FILE_EXT,
-              CarbonCommonConstants.FACT_UPDATE_EXTENSION);
-          carbonFile.renameTo(changedFileName);
-        }
-      }
-    } catch (IOException e) {
-      LOGGER.error("Inside renameFactFile. Problem checking file existence :: " + e.getMessage());
-    }
-  }
-
-  /**
-   * API will provide the load number inorder to record the same in metadata file.
-   */
-  public static int getLoadCount(CarbonLoadModel loadModel, int currentRestructNumber)
-      throws IOException {
-
-    String hdfsLoadedTable = getLoadFolderPath(loadModel, null, null, currentRestructNumber);
-    int loadCounter = CarbonUtil.checkAndReturnCurrentLoadFolderNumber(hdfsLoadedTable);
-
-    String hdfsStoreLoadFolder =
-        hdfsLoadedTable + File.separator + CarbonCommonConstants.LOAD_FOLDER + loadCounter;
-    hdfsStoreLoadFolder = hdfsStoreLoadFolder.replace("\\", "/");
-
-    String loadFolerCount = hdfsStoreLoadFolder
-        .substring(hdfsStoreLoadFolder.lastIndexOf('_') + 1, hdfsStoreLoadFolder.length());
-    return Integer.parseInt(loadFolerCount) + 1;
-  }
-
-  /**
-   * API will provide the load folder path for the store inorder to store the same
-   * in the metadata.
-   */
-  public static String getLoadFolderPath(CarbonLoadModel loadModel, String tableName,
-      String databaseName, int currentRestructNumber) {
-
-    //CHECKSTYLE:OFF    Approval No:Approval-V1R2C10_005
-
-    boolean copyStore =
-        Boolean.valueOf(CarbonProperties.getInstance().getProperty("dataload.hdfs.copy", "true"));
-
-    // CHECKSTYLE:ON
-    if (null == tableName && null == databaseName) {
-      databaseName = loadModel.getDatabaseName();
-      tableName = loadModel.getTableName();
-    }
-    String factTable = loadModel.getTableName();
-    String hdfsLoadedTable = null;
-    if (copyStore) {
-      String hdfsLocation =
-          CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION_HDFS);
-      String localStore = CarbonProperties.getInstance()
-          .getProperty(CarbonCommonConstants.STORE_LOCATION,
-              CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL);
-
-      if (!hdfsLocation.equals(localStore)) {
-        String hdfsStoreLocation = hdfsLocation;
-        hdfsStoreLocation =
-            hdfsStoreLocation + File.separator + databaseName + File.separator + tableName;
-
-        int rsCounter = currentRestructNumber;
-        if (rsCounter == -1) {
-          rsCounter = 0;
-        }
-
-        hdfsLoadedTable =
-            hdfsStoreLocation + File.separator + CarbonCommonConstants.RESTRUCTRE_FOLDER + rsCounter
-                + File.separator + factTable;
-
-        hdfsLoadedTable = hdfsLoadedTable.replace("\\", "/");
-      }
-
-    }
-    return hdfsLoadedTable;
-
-  }
-
-  /**
    * This API will write the load level metadata for the loadmanagement module inorder to
    * manage the load and query execution management smoothly.
    *
@@ -875,140 +491,6 @@ public final class CarbonLoaderUtil {
     return carbonTable.getMetaDataFilepath();
   }
 
-  /**
-   * This method will provide the dimension column list for a given aggregate
-   * table
-   */
-  public static Set<String> getColumnListFromAggTable(CarbonLoadModel model) {
-    Set<String> columnList = new HashSet<String>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-    CarbonTable carbonTable =
-        org.apache.carbondata.core.carbon.metadata.CarbonMetadata.getInstance()
-            .getCarbonTable(model.getDatabaseName() + '_' + model.getTableName());
-    List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(model.getAggTableName());
-    List<CarbonMeasure> measures = carbonTable.getMeasureByTableName(model.getAggTableName());
-    for (CarbonDimension carbonDimension : dimensions) {
-      columnList.add(carbonDimension.getColName());
-    }
-    for (CarbonMeasure carbonMeasure : measures) {
-      columnList.add(carbonMeasure.getColName());
-    }
-    return columnList;
-  }
-
-  public static void copyMergedLoadToHDFS(CarbonLoadModel loadModel, int currentRestructNumber,
-      String mergedLoadName) {
-    //Copy the current load folder to HDFS
-    boolean copyStore =
-        Boolean.valueOf(CarbonProperties.getInstance().getProperty("dataload.hdfs.copy", "true"));
-
-    String databaseName = loadModel.getDatabaseName();
-    String tableName = loadModel.getTableName();
-    String factTable = loadModel.getTableName();
-    String aggTableName = loadModel.getAggTableName();
-
-    if (copyStore) {
-      String hdfsLocation =
-          CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION_HDFS);
-
-      String localStore = CarbonProperties.getInstance()
-          .getProperty(CarbonCommonConstants.STORE_LOCATION,
-              CarbonCommonConstants.STORE_LOCATION_DEFAULT_VAL);
-      if (!loadModel.isAggLoadRequest()) {
-        copyMergeToHDFS(databaseName, tableName, factTable, hdfsLocation, localStore,
-            currentRestructNumber, mergedLoadName);
-      }
-      if (null != aggTableName) {
-        copyMergeToHDFS(databaseName, tableName, aggTableName, hdfsLocation, localStore,
-            currentRestructNumber, mergedLoadName);
-      }
-      try {
-        CarbonUtil.deleteFoldersAndFiles(new File[] {
-            new File(localStore + File.separator + databaseName + File.separator + tableName) });
-      } catch (CarbonUtilException e) {
-        LOGGER.error("Error while CarbonUtil.deleteFoldersAndFiles ");
-      }
-    }
-  }
-
-  public static void copyMergeToHDFS(String databaseName, String tableName, String factTable,
-      String hdfsLocation, String localStore, int currentRestructNumber, String mergedLoadName) {
-    try {
-      //If the hdfs store and the local store configured differently, then copy
-      if (hdfsLocation != null && !hdfsLocation.equals(localStore)) {
-        /**
-         * Identify the Load_X folder from the local store folder
-         */
-        String currentloadedStore = localStore;
-        currentloadedStore =
-            currentloadedStore + File.separator + databaseName + File.separator + tableName;
-
-        int rsCounter = currentRestructNumber;
-
-        if (rsCounter == -1) {
-          LOGGER.info("Unable to find the local store details (RS_-1) " + currentloadedStore);
-          return;
-        }
-        String localLoadedTable =
-            currentloadedStore + File.separator + CarbonCommonConstants.RESTRUCTRE_FOLDER
-                + rsCounter + File.separator + factTable;
-
-        localLoadedTable = localLoadedTable.replace("\\", "/");
-
-        int loadCounter = CarbonUtil.checkAndReturnCurrentLoadFolderNumber(localLoadedTable);
-
-        if (loadCounter == -1) {
-          LOGGER.info("Unable to find the local store details (Load_-1) " + currentloadedStore);
-
-          return;
-        }
-
-        String localLoadName = CarbonCommonConstants.LOAD_FOLDER + mergedLoadName;
-        String localLoadFolder =
-            localLoadedTable + File.separator + CarbonCommonConstants.LOAD_FOLDER + mergedLoadName;
-
-        LOGGER.info("Local data loaded folder ... = " + localLoadFolder);
-
-        //Identify the Load_X folder in the HDFS store
-        String hdfsStoreLocation = hdfsLocation;
-        hdfsStoreLocation =
-            hdfsStoreLocation + File.separator + databaseName + File.separator + tableName;
-
-        rsCounter = currentRestructNumber;
-        if (rsCounter == -1) {
-          rsCounter = 0;
-        }
-
-        String hdfsLoadedTable =
-            hdfsStoreLocation + File.separator + CarbonCommonConstants.RESTRUCTRE_FOLDER + rsCounter
-                + File.separator + factTable;
-
-        hdfsLoadedTable = hdfsLoadedTable.replace("\\", "/");
-
-        String hdfsStoreLoadFolder = hdfsLoadedTable + File.separator + localLoadName;
-
-        LOGGER.info("HDFS data load folder ... = " + hdfsStoreLoadFolder);
-
-        // Copy the data created through latest ETL run, to the HDFS store
-        LOGGER.info("Copying " + localLoadFolder + " --> " + hdfsStoreLoadFolder);
-
-        hdfsStoreLoadFolder = hdfsStoreLoadFolder.replace("\\", "/");
-        Path path = new Path(hdfsStoreLocation);
-
-        FileSystem fs = path.getFileSystem(FileFactory.getConfiguration());
-        fs.copyFromLocalFile(true, true, new Path(localLoadFolder), new Path(hdfsStoreLoadFolder));
-
-        LOGGER.info("Copying sliceMetaData from " + localLoadedTable + " --> " + hdfsLoadedTable);
-
-      } else {
-        LOGGER.info("Separate carbon.storelocation.hdfs is not configured for hdfs store path");
-      }
-    } catch (RuntimeException e) {
-      LOGGER.info(e.getMessage());
-    } catch (Exception e) {
-      LOGGER.info(e.getMessage());
-    }
-  }
-
   public static Dictionary getDictionary(DictionaryColumnUniqueIdentifier columnIdentifier,
       String carbonStorePath) throws CarbonUtilException {
     Cache dictCache =

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/integration/spark/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java b/integration/spark/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java
index d2af4f9..b9ac4dd 100644
--- a/integration/spark/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java
+++ b/integration/spark/src/main/java/org/apache/carbondata/spark/load/DeleteLoadFolders.java
@@ -45,7 +45,6 @@ import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
 import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
 import org.apache.carbondata.core.load.LoadMetadataDetails;
 import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.spark.util.LoadMetadataUtil;
 
 public final class DeleteLoadFolders {
 
@@ -56,67 +55,6 @@ public final class DeleteLoadFolders {
 
   }
 
-  public static boolean deleteLoadFoldersFromFileSystem(CarbonLoadModel loadModel,
-      int partitionCount, String storeLocation, boolean isForceDelete, int currentRestructNumber,
-      LoadMetadataDetails[] details) {
-    String path = null;
-    List<LoadMetadataDetails> deletedLoads =
-        new ArrayList<LoadMetadataDetails>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-
-    boolean isDeleted = false;
-
-    if (details != null && details.length != 0) {
-      for (LoadMetadataDetails oneLoad : details) {
-        if (checkIfLoadCanBeDeleted(oneLoad, isForceDelete)) {
-          boolean deletionStatus = false;
-
-          for (int partitionId = 0; partitionId < partitionCount; partitionId++) {
-            // check load folder in each restructure folder
-            for (int restructureFolderNum = 0;
-                 restructureFolderNum <= currentRestructNumber; restructureFolderNum++) {
-              CarbonFile[] aggFiles = LoadMetadataUtil
-                  .getAggregateTableList(loadModel, storeLocation, partitionId,
-                      restructureFolderNum);
-              deleteAggLoadFolders(aggFiles,
-                  CarbonCommonConstants.LOAD_FOLDER + oneLoad.getLoadName());
-              path = LoadMetadataUtil.createLoadFolderPath(loadModel, storeLocation, partitionId,
-                  restructureFolderNum);
-              String loadFolderPath = "";
-              // deleting merged load folder
-              if (oneLoad.getMergedLoadName() != null) {
-                loadFolderPath =
-                    path + CarbonCommonConstants.FILE_SEPARATOR + CarbonCommonConstants.LOAD_FOLDER
-                        + oneLoad.getMergedLoadName();
-                deletionStatus = physicalFactAndMeasureMetadataDeletion(loadFolderPath);
-              } else {
-                loadFolderPath =
-                    path + CarbonCommonConstants.FILE_SEPARATOR + CarbonCommonConstants.LOAD_FOLDER
-                        + oneLoad.getLoadName();
-                deletionStatus = physicalFactAndMeasureMetadataDeletion(loadFolderPath);
-              }
-              if (deletionStatus) {
-                cleanDeletedFactFile(loadFolderPath);
-                factFileRenaming(loadFolderPath);
-                // if deletion status is True then there is no
-                // need to traverse all the RS folders.
-                break;
-              }
-            }
-
-          }
-          if (deletionStatus) {
-            isDeleted = true;
-            oneLoad.setVisibility("false");
-            deletedLoads.add(oneLoad);
-            LOGGER.info(" Deleted the load " + oneLoad.getLoadName());
-          }
-        }
-      }
-    }
-
-    return isDeleted;
-  }
-
   /**
    * returns segment path
    *
@@ -138,40 +76,6 @@ public final class DeleteLoadFolders {
     return path;
   }
 
-  public static void deleteAggLoadFolders(CarbonFile[] aggFiles, String loadName) {
-    for (CarbonFile file : aggFiles) {
-      deleteLoadFolderFromEachAgg(file, loadName);
-    }
-
-  }
-
-  private static void deleteLoadFolderFromEachAgg(CarbonFile file, final String loadName) {
-    CarbonFile[] loadFolders = file.listFiles(new CarbonFileFilter() {
-
-      @Override public boolean accept(CarbonFile file) {
-        if (file.getName().equalsIgnoreCase(loadName)) {
-          return true;
-        }
-        return false;
-      }
-    });
-
-    for (CarbonFile loadFolder : loadFolders) {
-      CarbonFile[] files = loadFolder.listFiles();
-      // deleting individual files
-      if (files != null) {
-        for (CarbonFile eachFile : files) {
-          if (!eachFile.delete()) {
-            LOGGER.warn("Unable to delete the file as per delete command "
-                + loadFolder.getAbsolutePath());
-          }
-        }
-      }
-
-    }
-
-  }
-
   private static boolean physicalFactAndMeasureMetadataDeletion(String path) {
 
     boolean status = false;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/integration/spark/src/main/java/org/apache/carbondata/spark/util/LoadMetadataUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/java/org/apache/carbondata/spark/util/LoadMetadataUtil.java b/integration/spark/src/main/java/org/apache/carbondata/spark/util/LoadMetadataUtil.java
index 6110fb8..a897e80 100644
--- a/integration/spark/src/main/java/org/apache/carbondata/spark/util/LoadMetadataUtil.java
+++ b/integration/spark/src/main/java/org/apache/carbondata/spark/util/LoadMetadataUtil.java
@@ -28,13 +28,8 @@
  */
 package org.apache.carbondata.spark.util;
 
-import java.io.File;
-
 import org.apache.carbondata.core.carbon.metadata.schema.table.CarbonTable;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
-import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
 import org.apache.carbondata.core.load.LoadMetadataDetails;
 import org.apache.carbondata.lcm.status.SegmentStatusManager;
 import org.apache.carbondata.spark.load.CarbonLoadModel;
@@ -65,49 +60,4 @@ public final class LoadMetadataUtil {
     return false;
 
   }
-
-  public static String createLoadFolderPath(CarbonLoadModel model, String hdfsStoreLocation,
-      int partitionId, int currentRestructNumber) {
-    hdfsStoreLocation =
-        hdfsStoreLocation + File.separator + model.getDatabaseName() + '_' + partitionId
-            + File.separator + model.getTableName() + '_' + partitionId;
-    int rsCounter = currentRestructNumber;
-    if (rsCounter == -1) {
-      rsCounter = 0;
-    }
-    String hdfsLoadedTable =
-        hdfsStoreLocation + File.separator + CarbonCommonConstants.RESTRUCTRE_FOLDER + rsCounter
-            + File.separator + model.getTableName();
-    hdfsLoadedTable = hdfsLoadedTable.replace("\\", "/");
-    return hdfsLoadedTable;
-  }
-
-  public static CarbonFile[] getAggregateTableList(final CarbonLoadModel model,
-      String hdfsStoreLocation, int partitionId, int currentRestructNumber) {
-    hdfsStoreLocation =
-        hdfsStoreLocation + File.separator + model.getDatabaseName() + '_' + partitionId
-            + File.separator + model.getTableName() + '_' + partitionId;
-
-    int rsCounter = currentRestructNumber;
-    if (rsCounter == -1) {
-      rsCounter = 0;
-    }
-
-    String hdfsLoadedTable =
-        hdfsStoreLocation + File.separator + CarbonCommonConstants.RESTRUCTRE_FOLDER + rsCounter;
-
-    CarbonFile rsFile =
-        FileFactory.getCarbonFile(hdfsLoadedTable, FileFactory.getFileType(hdfsLoadedTable));
-
-    CarbonFile[] aggFiles = rsFile.listFiles(new CarbonFileFilter() {
-
-      @Override public boolean accept(CarbonFile file) {
-        return file.getName().startsWith(
-            CarbonCommonConstants.AGGREGATE_TABLE_START_TAG + CarbonCommonConstants.UNDERSCORE
-                + model.getTableName());
-      }
-    });
-
-    return aggFiles;
-  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
index abbca33..60a2d00 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataLoadRDD.scala
@@ -83,7 +83,6 @@ class CarbonNodePartition(rddId: Int, val idx: Int, host: String,
  * @param kettleHomePath        The kettle home path
  * @param partitioner           Partitioner which specify how to partition
  * @param columinar             whether it is columinar
- * @param currentRestructNumber current restruct number
  * @param loadCount             Current load count
  * @param tableCreationTime      Time of creating table
  * @param schemaLastUpdatedTime Time of last schema update
@@ -101,7 +100,6 @@ class CarbonDataLoadRDD[K, V](
     kettleHomePath: String,
     partitioner: Partitioner,
     columinar: Boolean,
-    currentRestructNumber: Integer,
     loadCount: Integer,
     tableCreationTime: Long,
     schemaLastUpdatedTime: Long,
@@ -198,8 +196,7 @@ class CarbonDataLoadRDD[K, V](
         }
         else {
           try {
-            CarbonLoaderUtil.executeGraph(model, storeLocation, hdfsStoreLocation, kettleHomePath,
-              currentRestructNumber)
+            CarbonLoaderUtil.executeGraph(model, storeLocation, hdfsStoreLocation, kettleHomePath)
           } catch {
             case e: DataLoadingException => if (e.getErrorCode ==
                                                 DataProcessorConstants.BAD_REC_FOUND) {
@@ -324,13 +321,7 @@ class CarbonDataLoadRDD[K, V](
           val copyListOfLoadFolders = listOfLoadFolders.asScala.toList
           val copyListOfUpdatedLoadFolders = listOfUpdatedLoadFolders.asScala.toList
           loadTableSlices(listOfAllLoadFolders, details)
-          var loadFolders = Array[String]()
-          val loadFolder = CarbonLoaderUtil
-            .getAggLoadFolderLocation(newSlice, model.getDatabaseName, model.getTableName,
-              model.getTableName, hdfsStoreLocation, currentRestructNumber)
-          if (null != loadFolder) {
-            loadFolders :+= loadFolder
-          }
+          val loadFolders = Array[String]()
           dataloadStatus = iterateOverAggTables(aggTables, copyListOfLoadFolders.asJava,
             copyListOfUpdatedLoadFolders.asJava, loadFolders)
           if (CarbonCommonConstants.STORE_LOADSTATUS_FAILURE.equals(dataloadStatus)) {
@@ -358,14 +349,7 @@ class CarbonDataLoadRDD[K, V](
         val listOfLoadFolders = CarbonLoaderUtil.getListOfValidSlices(details)
         val listOfUpdatedLoadFolders = CarbonLoaderUtil.getListOfUpdatedSlices(details)
         loadTableSlices(listOfAllLoadFolders, details)
-        var loadFolders = Array[String]()
-        var restructFolders = Array[String]()
-        for (number <- 0 to currentRestructNumber) {
-          restructFolders = CarbonLoaderUtil
-            .getStorelocs(model.getDatabaseName, model.getTableName, model.getTableName,
-              hdfsStoreLocation, number)
-          loadFolders = loadFolders ++ restructFolders
-        }
+        val loadFolders = Array[String]()
         val aggTable = model.getAggTableName
         dataloadStatus = loadAggregationTable(listOfLoadFolders, listOfUpdatedLoadFolders,
           loadFolders)
@@ -386,16 +370,7 @@ class CarbonDataLoadRDD[K, V](
           val listOfUpdatedLoadFolders = CarbonLoaderUtil.getListOfUpdatedSlices(details)
           val listOfAllLoadFolder = CarbonQueryUtil.getListOfSlices(details)
           loadTableSlices(listOfAllLoadFolder, details)
-          var loadFolders = Array[String]()
-          listOfUpdatedLoadFolders.asScala.foreach { sliceNum =>
-            val newSlice = CarbonCommonConstants.LOAD_FOLDER + sliceNum
-            val loadFolder = CarbonLoaderUtil
-              .getAggLoadFolderLocation(newSlice, model.getDatabaseName, model.getTableName,
-                model.getTableName, hdfsStoreLocation, currentRestructNumber)
-            if (null != loadFolder) {
-              loadFolders :+= loadFolder
-            }
-          }
+          val loadFolders = Array[String]()
           iterateOverAggTables(aggTables, listOfLoadFolders, listOfUpdatedLoadFolders, loadFolders)
         }
       }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
index bf4fa9f..0d0ac97 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDataRDDFactory.scala
@@ -42,7 +42,7 @@ import org.apache.carbondata.core.carbon.metadata.CarbonMetadata
 import org.apache.carbondata.core.carbon.metadata.schema.table.CarbonTable
 import org.apache.carbondata.core.constants.CarbonCommonConstants
 import org.apache.carbondata.core.load.{BlockDetails, LoadMetadataDetails}
-import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
+import org.apache.carbondata.core.util.CarbonProperties
 import org.apache.carbondata.integration.spark.merger.{CarbonCompactionUtil, CompactionCallable, CompactionType}
 import org.apache.carbondata.lcm.locks.{CarbonLockFactory, ICarbonLock, LockUsage}
 import org.apache.carbondata.lcm.status.SegmentStatusManager
@@ -102,11 +102,6 @@ object CarbonDataRDDFactory extends Logging {
     val table = CarbonMetadata.getInstance()
       .getCarbonTable(carbonLoadModel.getDatabaseName + "_" + carbonLoadModel.getTableName)
     val metaDataPath: String = table.getMetaDataFilepath
-    var currentRestructNumber = CarbonUtil
-      .checkAndReturnCurrentRestructFolderNumber(metaDataPath, "RS_", false)
-    if (-1 == currentRestructNumber) {
-      currentRestructNumber = 0
-    }
   }
 
   def deleteLoadByDate(
@@ -124,12 +119,6 @@ object CarbonDataRDDFactory extends Logging {
     // Delete the records based on data
     val table = org.apache.carbondata.core.carbon.metadata.CarbonMetadata.getInstance
       .getCarbonTable(databaseName + "_" + tableName)
-
-    var currentRestructNumber = CarbonUtil
-      .checkAndReturnCurrentRestructFolderNumber(table.getMetaDataFilepath, "RS_", false)
-    if (-1 == currentRestructNumber) {
-      currentRestructNumber = 0
-    }
     val segmentStatusManager = new SegmentStatusManager(table.getAbsoluteTableIdentifier)
     val loadMetadataDetailsArray =
       segmentStatusManager.readLoadMetadata(table.getMetaDataFilepath()).toList
@@ -145,8 +134,7 @@ object CarbonDataRDDFactory extends Logging {
       table.getFactTableName,
       tableName,
       hdfsStoreLocation,
-      loadMetadataDetailsArray,
-      currentRestructNumber).collect.groupBy(_._1)
+      loadMetadataDetailsArray).collect.groupBy(_._1)
 
     var updatedLoadMetadataDetailsList = new ListBuffer[LoadMetadataDetails]()
     if (resultMap.nonEmpty) {
@@ -688,7 +676,6 @@ object CarbonDataRDDFactory extends Logging {
       isAgg: Boolean,
       partitionStatus: String = CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS) {
     val carbonTable = carbonLoadModel.getCarbonDataLoadSchema.getCarbonTable
-    var currentRestructNumber = -1
 
     // for handling of the segment Merging.
     def handleSegmentMerging(tableCreationTime: Long): Unit = {
@@ -780,17 +767,9 @@ object CarbonDataRDDFactory extends Logging {
         .audit("Data load request has been received for table " + carbonLoadModel
           .getDatabaseName + "." + carbonLoadModel.getTableName
         )
-
-      currentRestructNumber = CarbonUtil
-        .checkAndReturnCurrentRestructFolderNumber(carbonTable.getMetaDataFilepath, "RS_", false)
-      if (-1 == currentRestructNumber) {
-        currentRestructNumber = 0
-      }
-
       // Check if any load need to be deleted before loading new data
       deleteLoadsAndUpdateMetadata(carbonLoadModel, carbonTable, partitioner, hdfsStoreLocation,
-        isForceDeletion = false,
-        currentRestructNumber)
+        isForceDeletion = false)
       if (null == carbonLoadModel.getLoadMetadataDetails) {
         readLoadMetadataDetails(carbonLoadModel, hdfsStoreLocation)
       }
@@ -988,7 +967,6 @@ object CarbonDataRDDFactory extends Logging {
               kettleHomePath,
               partitioner,
               columinar,
-              currentRestructNumber,
               currentLoadCount,
               tableCreationTime,
               schemaLastUpdatedTime,
@@ -1043,27 +1021,7 @@ object CarbonDataRDDFactory extends Logging {
 
       if (loadStatus == CarbonCommonConstants.STORE_LOADSTATUS_FAILURE) {
         logInfo("********starting clean up**********")
-        if (isAgg) {
-          // TODO:need to clean aggTable
-          CarbonLoaderUtil.deleteTable(partitioner.partitionCount, carbonLoadModel.getDatabaseName,
-            carbonLoadModel.getTableName, carbonLoadModel.getAggTableName, hdfsStoreLocation,
-            currentRestructNumber
-          )
-          errorMessage = "Aggregate table creation failure"
-        } else {
-          CarbonLoaderUtil.deleteSegment(carbonLoadModel, currentLoadCount)
-          val aggTables = carbonTable.getAggregateTablesName
-          if (null != aggTables && !aggTables.isEmpty) {
-            // TODO:need to clean aggTable
-            val (result, _) = status(0)
-            val newSlice = CarbonCommonConstants.LOAD_FOLDER + result
-            aggTables.asScala.foreach { aggTableName =>
-              CarbonLoaderUtil
-                .deleteSlice(partitioner.partitionCount, carbonLoadModel.getDatabaseName,
-                  carbonLoadModel.getTableName, hdfsStoreLocation, currentRestructNumber, newSlice)
-            }
-          }
-        }
+        CarbonLoaderUtil.deleteSegment(carbonLoadModel, currentLoadCount)
         logInfo("********clean up done**********")
         logger.audit(s"Data load is failed for " +
           s"${carbonLoadModel.getDatabaseName}.${carbonLoadModel.getTableName}")
@@ -1110,8 +1068,7 @@ object CarbonDataRDDFactory extends Logging {
     carbonLoadModel: CarbonLoadModel,
     table: CarbonTable, partitioner: Partitioner,
     hdfsStoreLocation: String,
-    isForceDeletion: Boolean,
-    currentRestructNumber: Integer) {
+    isForceDeletion: Boolean) {
     if (LoadMetadataUtil.isLoadDeletionRequired(carbonLoadModel)) {
       val loadMetadataFilePath = CarbonLoaderUtil
         .extractLoadMetadataFileLocation(carbonLoadModel)
@@ -1152,11 +1109,6 @@ object CarbonDataRDDFactory extends Logging {
     val table = org.apache.carbondata.core.carbon.metadata.CarbonMetadata.getInstance
       .getCarbonTable(carbonLoadModel.getDatabaseName + "_" + carbonLoadModel.getTableName)
     val metaDataPath: String = table.getMetaDataFilepath
-    var currentRestructNumber = CarbonUtil
-      .checkAndReturnCurrentRestructFolderNumber(metaDataPath, "RS_", false)
-    if (-1 == currentRestructNumber) {
-      currentRestructNumber = 0
-    }
     val carbonLock = CarbonLockFactory
       .getCarbonLockObj(table.getAbsoluteTableIdentifier.getCarbonTableIdentifier,
         LockUsage.METADATA_LOCK
@@ -1167,8 +1119,7 @@ object CarbonDataRDDFactory extends Logging {
           table,
           partitioner,
           hdfsStoreLocation,
-          isForceDeletion = true,
-          currentRestructNumber)
+          isForceDeletion = true)
       }
     }
     finally {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadByDateRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadByDateRDD.scala b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadByDateRDD.scala
index ed3be91..f3308d6 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadByDateRDD.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/spark/rdd/CarbonDeleteLoadByDateRDD.scala
@@ -41,8 +41,7 @@ class CarbonDeleteLoadByDateRDD[K, V](
     factTableName: String,
     dimTableName: String,
     hdfsStoreLocation: String,
-    loadMetadataDetails: List[LoadMetadataDetails],
-    currentRestructFolder: Integer)
+    loadMetadataDetails: List[LoadMetadataDetails])
   extends RDD[(K, V)](sc, Nil) with Logging {
 
   sc.setLocalProperty("spark.scheduler.pool", "DDL")

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/CsvInputMeta.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/CsvInputMeta.java b/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/CsvInputMeta.java
index b15862d..2e0dece 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/CsvInputMeta.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvreaderstep/CsvInputMeta.java
@@ -93,8 +93,6 @@ public class CsvInputMeta extends BaseStepMeta
 
   private boolean newlinePossibleInFields;
 
-  private int currentRestructNumber;
-
   private String blocksID;
 
   private String partitionID;
@@ -120,7 +118,6 @@ public class CsvInputMeta extends BaseStepMeta
     lazyConversionActive = true;
     isaddresult = false;
     bufferSize = "50000";
-    currentRestructNumber = -1;
     blocksID = "";
     partitionID = "";
     escapeCharacter ="\\";
@@ -157,8 +154,6 @@ public class CsvInputMeta extends BaseStepMeta
         newlinePossibleInFields = "Y".equalsIgnoreCase(nlp);
       }
       encoding = XMLHandler.getTagValue(stepnode, getXmlCode("ENCODING"));
-      currentRestructNumber =
-          Integer.parseInt(XMLHandler.getTagValue(stepnode, "currentRestructNumber"));
       blocksID = XMLHandler.getTagValue(stepnode, "blocksID");
       partitionID = XMLHandler.getTagValue(stepnode, "partitionID");
       escapeCharacter = XMLHandler.getTagValue(stepnode, "escapeCharacter");
@@ -222,8 +217,6 @@ public class CsvInputMeta extends BaseStepMeta
     retval.append("    ")
         .append(XMLHandler.addTagValue(getXmlCode("NEWLINE_POSSIBLE"), newlinePossibleInFields));
     retval.append("    ").append(XMLHandler.addTagValue(getXmlCode("ENCODING"), encoding));
-    retval.append("    ")
-        .append(XMLHandler.addTagValue("currentRestructNumber", currentRestructNumber));
     retval.append("    ").append(XMLHandler.addTagValue("blocksID", blocksID));
     retval.append("    ").append(XMLHandler.addTagValue("partitionID", partitionID));
     retval.append("    ").append(XMLHandler.addTagValue("escapeCharacter", escapeCharacter));
@@ -277,7 +270,6 @@ public class CsvInputMeta extends BaseStepMeta
           rep.getStepAttributeBoolean(idStep, 0, getRepCode("NEWLINE_POSSIBLE"),
               !runningInParallel);
       encoding = rep.getStepAttributeString(idStep, getRepCode("ENCODING"));
-      currentRestructNumber = (int) rep.getStepAttributeInteger(idStep, "currentRestructNumber");
       blocksID = rep.getStepAttributeString(idStep, getRepCode("blocksID"));
       partitionID = rep.getStepAttributeString(idStep, getRepCode("partitionID"));
       escapeCharacter = rep.getStepAttributeString(idStep, getRepCode("escapeCharacter"));
@@ -333,8 +325,6 @@ public class CsvInputMeta extends BaseStepMeta
       rep.saveStepAttribute(idTransformation, idStep, getRepCode("NEWLINE_POSSIBLE"),
           newlinePossibleInFields);
       rep.saveStepAttribute(idTransformation, idStep, getRepCode("ENCODING"), encoding);
-      rep.saveStepAttribute(idTransformation, idStep, "currentRestructNumber",
-          currentRestructNumber);
       rep.saveStepAttribute(idTransformation, idStep, getRepCode("blocksID"), blocksID);
       rep.saveStepAttribute(idTransformation, idStep, getRepCode("partitionID"), partitionID);
       rep.saveStepAttribute(idTransformation, idStep, getRepCode("escapeCharacter"),
@@ -832,8 +822,6 @@ public class CsvInputMeta extends BaseStepMeta
           isaddresult = (Boolean) entry.getValue();
         } else if ("ENCODING".equals(attributeKey)) {
           encoding = (String) entry.getValue();
-        } else if ("currentRestructNumber".equals(attributeKey)) {
-          currentRestructNumber = (Integer) entry.getValue();
         } else if ("blocksID".equals(attributeKey)) {
           blocksID = (String) entry.getValue();
         } else if ("partitionID".equals(attributeKey)) {
@@ -920,20 +908,6 @@ public class CsvInputMeta extends BaseStepMeta
     this.newlinePossibleInFields = newlinePossibleInFields;
   }
 
-  /**
-   * @return the currentRestructNumber
-   */
-  public int getCurrentRestructNumber() {
-    return currentRestructNumber;
-  }
-
-  /**
-   * @param currentRestructNum the currentRestructNumber to set
-   */
-  public void setCurrentRestructNumber(int currentRestructNum) {
-    this.currentRestructNumber = currentRestructNum;
-  }
-
   public void setPartitionID(String partitionID) {
     this.partitionID = partitionID;
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java b/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
index 10ea710..f55c247 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/GraphGenerator.java
@@ -46,7 +46,6 @@ import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonMeas
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.load.BlockDetails;
 import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.CarbonUtil;
 import org.apache.carbondata.processing.api.dataloader.DataLoadModel;
 import org.apache.carbondata.processing.api.dataloader.SchemaInfo;
 import org.apache.carbondata.processing.csvreaderstep.CsvInputMeta;
@@ -184,7 +183,6 @@ public class GraphGenerator {
   private boolean isColumnar;
   private String factTableName;
   private String factStoreLocation;
-  private int currentRestructNumber;
   private String blocksID;
   private String escapeCharacter;
   /**
@@ -205,7 +203,7 @@ public class GraphGenerator {
   private String maxColumns;
 
   public GraphGenerator(DataLoadModel dataLoadModel, boolean isHDFSReadMode, String partitionID,
-      String factStoreLocation, int currentRestructNum, int allocate,
+      String factStoreLocation, int allocate,
       CarbonDataLoadSchema carbonDataLoadSchema, String segmentId) {
     CarbonMetadata.getInstance().addCarbonTable(carbonDataLoadSchema.getCarbonTable());
     this.schemaInfo = dataLoadModel.getSchemaInfo();
@@ -219,7 +217,6 @@ public class GraphGenerator {
     this.partitionID = partitionID;
     this.factStoreLocation = factStoreLocation;
     this.isColumnar = Boolean.parseBoolean(CarbonCommonConstants.IS_COLUMNAR_STORAGE_DEFAULTVALUE);
-    this.currentRestructNumber = currentRestructNum;
     this.blocksID = dataLoadModel.getBlocksID();
     this.taskNo = dataLoadModel.getTaskNo();
     this.factTimeStamp = dataLoadModel.getFactTimeStamp();
@@ -231,10 +228,10 @@ public class GraphGenerator {
   }
 
   public GraphGenerator(DataLoadModel dataLoadModel, boolean isHDFSReadMode, String partitionID,
-      String factStoreLocation, int currentRestructNum, int allocate,
-      CarbonDataLoadSchema carbonDataLoadSchema, String segmentId, String outputLocation) {
-    this(dataLoadModel, isHDFSReadMode, partitionID, factStoreLocation, currentRestructNum,
-        allocate, carbonDataLoadSchema, segmentId);
+      String factStoreLocation, int allocate, CarbonDataLoadSchema carbonDataLoadSchema,
+      String segmentId, String outputLocation) {
+    this(dataLoadModel, isHDFSReadMode, partitionID, factStoreLocation, allocate,
+        carbonDataLoadSchema, segmentId);
     this.outputLocation = outputLocation;
   }
 
@@ -442,7 +439,6 @@ public class GraphGenerator {
     csvInputMeta.setEnclosure("\"");
     csvInputMeta.setHeaderPresent(true);
     csvInputMeta.setMaxColumns(maxColumns);
-    csvInputMeta.setCurrentRestructNumber(graphConfiguration.getCurrentRestructNumber());
     StepMeta csvDataStep =
         new StepMeta(GraphGeneratorConstants.CSV_INPUT, (StepMetaInterface) csvInputMeta);
     csvDataStep.setLocation(100, 100);
@@ -473,12 +469,6 @@ public class GraphGenerator {
     sliceMerger.setTabelName(configurationInfo.getTableName());
     sliceMerger.setTableName(schemaInfo.getTableName());
     sliceMerger.setDatabaseName(schemaInfo.getDatabaseName());
-    if (null != this.factStoreLocation) {
-      sliceMerger.setCurrentRestructNumber(
-          CarbonUtil.getRestructureNumber(this.factStoreLocation, this.factTableName));
-    } else {
-      sliceMerger.setCurrentRestructNumber(configurationInfo.getCurrentRestructNumber());
-    }
     sliceMerger.setGroupByEnabled(isAutoAggRequest + "");
     if (isAutoAggRequest) {
       String[] aggType = configurationInfo.getAggType();
@@ -559,7 +549,6 @@ public class GraphGenerator {
     seqMeta.setDatabaseName(schemaInfo.getDatabaseName());
     seqMeta.setComplexDelimiterLevel1(schemaInfo.getComplexDelimiterLevel1());
     seqMeta.setComplexDelimiterLevel2(schemaInfo.getComplexDelimiterLevel2());
-    seqMeta.setCurrentRestructNumber(graphConfiguration.getCurrentRestructNumber());
     seqMeta.setCarbonMetaHier(graphConfiguration.getMetaHeirString());
     seqMeta.setCarbonmsr(graphConfiguration.getMeasuresString());
     seqMeta.setCarbonProps(graphConfiguration.getPropertiesString());
@@ -620,7 +609,6 @@ public class GraphGenerator {
     carbonMdKey.setDatabaseName(schemaInfo.getDatabaseName());
     carbonMdKey.setTableName(schemaInfo.getTableName());
     carbonMdKey.setComplexTypeString(graphConfiguration.getComplexTypeString());
-    carbonMdKey.setCurrentRestructNumber(graphConfiguration.getCurrentRestructNumber());
     carbonMdKey.setAggregateLevels(CarbonDataProcessorUtil
         .getLevelCardinalitiesString(graphConfiguration.getDimCardinalities(),
             graphConfiguration.getDimensions()));
@@ -765,7 +753,6 @@ public class GraphGenerator {
     sortRowsMeta.setTableName(schemaInfo.getTableName());
     sortRowsMeta.setDatabaseName(schemaInfo.getDatabaseName());
     sortRowsMeta.setOutputRowSize(actualMeasures.length + 1 + "");
-    sortRowsMeta.setCurrentRestructNumber(graphConfiguration.getCurrentRestructNumber());
     sortRowsMeta.setDimensionCount(graphConfiguration.getDimensions().length + "");
     sortRowsMeta.setComplexDimensionCount(graphConfiguration.getComplexTypeString().isEmpty() ?
         "0" :
@@ -797,7 +784,6 @@ public class GraphGenerator {
       CarbonDataLoadSchema carbonDataLoadSchema) throws GraphGeneratorException {
     //
     GraphConfigurationInfo graphConfiguration = new GraphConfigurationInfo();
-    graphConfiguration.setCurrentRestructNumber(currentRestructNumber);
     List<CarbonDimension> dimensions = carbonDataLoadSchema.getCarbonTable()
         .getDimensionByTableName(carbonDataLoadSchema.getCarbonTable().getFactTableName());
     prepareIsUseInvertedIndex(dimensions, graphConfiguration);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/configuration/GraphConfigurationInfo.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/configuration/GraphConfigurationInfo.java b/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/configuration/GraphConfigurationInfo.java
index 6ff1eb1..7c428a7 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/configuration/GraphConfigurationInfo.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/graphgenerator/configuration/GraphConfigurationInfo.java
@@ -186,7 +186,6 @@ public class GraphConfigurationInfo {
    * type
    */
   private char[] type;
-  private int currentRestructNumber;
   private String levelAnddataType;
 
   private Boolean[] isNoDictionaryDimMapping;
@@ -950,17 +949,6 @@ public class GraphConfigurationInfo {
     this.type = type;
   }
 
-  /**
-   * @return currentRestructNumber the current Restruct Number
-   */
-  public int getCurrentRestructNumber() {
-    return currentRestructNumber;
-  }
-
-  public void setCurrentRestructNumber(int currentRestructNum) {
-    this.currentRestructNumber = currentRestructNum;
-  }
-
   public String getLevelAnddataType() {
     return levelAnddataType;
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java b/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java
index bf1ecb3..f28950b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/mdkeygen/MDKeyGenStepMeta.java
@@ -97,8 +97,6 @@ public class MDKeyGenStepMeta extends BaseStepMeta implements StepMetaInterface
 
   private Map<String, GenericDataType> complexTypes;
 
-  private int currentRestructNumber;
-
   /**
    * It is column groups in below format
    * 0,1~2~3,4,5,6~7~8,9
@@ -155,7 +153,6 @@ public class MDKeyGenStepMeta extends BaseStepMeta implements StepMetaInterface
     databaseName = "";
     columnGroupsString = "";
     noDictionaryDims = "";
-    currentRestructNumber = -1;
     measureDataType = "";
     taskNo = "";
     factTimeStamp = "";
@@ -178,8 +175,6 @@ public class MDKeyGenStepMeta extends BaseStepMeta implements StepMetaInterface
     retval.append("    ").append(XMLHandler.addTagValue("dimensionCount", dimensionCount));
     retval.append("    ").append(XMLHandler.addTagValue("complexDimsCount", complexDimsCount));
     retval.append("    ").append(XMLHandler.addTagValue("complexTypeString", complexTypeString));
-    retval.append("    ")
-        .append(XMLHandler.addTagValue("currentRestructNumber", currentRestructNumber));
     retval.append("    ").append(XMLHandler.addTagValue("measureDataType", measureDataType));
     retval.append("    ").append(XMLHandler.addTagValue("taskNo", taskNo));
     retval.append("    ").append(XMLHandler.addTagValue("factTimeStamp", factTimeStamp));
@@ -207,8 +202,6 @@ public class MDKeyGenStepMeta extends BaseStepMeta implements StepMetaInterface
       dimensionCount = XMLHandler.getTagValue(stepnode, "dimensionCount");
       complexDimsCount = XMLHandler.getTagValue(stepnode, "complexDimsCount");
       complexTypeString = XMLHandler.getTagValue(stepnode, "complexTypeString");
-      currentRestructNumber =
-          Integer.parseInt(XMLHandler.getTagValue(stepnode, "currentRestructNumber"));
       measureDataType = XMLHandler.getTagValue(stepnode, "measureDataType");
       taskNo = XMLHandler.getTagValue(stepnode, "taskNo");
       factTimeStamp = XMLHandler.getTagValue(stepnode, "factTimeStamp");
@@ -235,8 +228,6 @@ public class MDKeyGenStepMeta extends BaseStepMeta implements StepMetaInterface
       rep.saveStepAttribute(idTransformation, idStep, "dimensionCount", dimensionCount);
       rep.saveStepAttribute(idTransformation, idStep, "complexDimsCount", complexDimsCount);
       rep.saveStepAttribute(idTransformation, idStep, "complexTypeString", complexTypeString);
-      rep.saveStepAttribute(idTransformation, idStep, "currentRestructNumber",
-          currentRestructNumber);
       rep.saveStepAttribute(idTransformation, idStep, "measureDataType", measureDataType);
       rep.saveStepAttribute(idTransformation, idStep, "taskNo", taskNo);
       rep.saveStepAttribute(idTransformation, idStep, "factTimeStamp", factTimeStamp);
@@ -267,7 +258,6 @@ public class MDKeyGenStepMeta extends BaseStepMeta implements StepMetaInterface
       dimensionCount = rep.getStepAttributeString(idStep, "dimensionCount");
       complexDimsCount = rep.getStepAttributeString(idStep, "complexDimsCount");
       complexTypeString = rep.getStepAttributeString(idStep, "complexTypeString");
-      currentRestructNumber = (int) rep.getStepAttributeInteger(idStep, "currentRestructNumber");
       measureDataType = rep.getStepAttributeString(idStep, "measureDataType");
       taskNo = rep.getStepAttributeString(idStep, "taskNo");
       factTimeStamp = rep.getStepAttributeString(idStep, "factTimeStamp");
@@ -400,20 +390,6 @@ public class MDKeyGenStepMeta extends BaseStepMeta implements StepMetaInterface
   }
 
   /**
-   * @return the currentRestructNumber
-   */
-  public int getCurrentRestructNumber() {
-    return currentRestructNumber;
-  }
-
-  /**
-   * @param currentRestructNum the currentRestructNumber to set
-   */
-  public void setCurrentRestructNumber(int currentRestructNum) {
-    this.currentRestructNumber = currentRestructNum;
-  }
-
-  /**
    * @return
    */
   public String getNoDictionaryDims() {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/processing/src/main/java/org/apache/carbondata/processing/merger/step/CarbonSliceMergerStepMeta.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/step/CarbonSliceMergerStepMeta.java b/processing/src/main/java/org/apache/carbondata/processing/merger/step/CarbonSliceMergerStepMeta.java
index c42ed0a..1d712ec 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/merger/step/CarbonSliceMergerStepMeta.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/merger/step/CarbonSliceMergerStepMeta.java
@@ -102,8 +102,6 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta
    */
   private String factDimLensString;
 
-  private int currentRestructNumber = -1;
-
   private String levelAnddataTypeString;
   /**
    * partitionID
@@ -139,7 +137,6 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta
     aggregatorClassString = "";
     aggregatorString = "";
     factDimLensString = "";
-    currentRestructNumber = -1;
     levelAnddataTypeString = "";
     partitionID = "";
     segmentId = "";
@@ -166,8 +163,6 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta
     retval.append("    ").append(XMLHandler.addTagValue("aggregatorString", aggregatorString));
     retval.append("    ").append(XMLHandler.addTagValue("factDimLensString", factDimLensString));
     retval.append("    ")
-        .append(XMLHandler.addTagValue("currentRestructNumber", currentRestructNumber));
-    retval.append("    ")
         .append(XMLHandler.addTagValue("levelAnddataTypeString", levelAnddataTypeString));
     retval.append("    ").append(XMLHandler.addTagValue("partitionID", partitionID));
     retval.append("    ").append(XMLHandler.addTagValue("segmentId", segmentId));
@@ -197,8 +192,6 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta
       aggregatorString = XMLHandler.getTagValue(stepnode, "aggregatorString");
       factDimLensString = XMLHandler.getTagValue(stepnode, "factDimLensString");
       levelAnddataTypeString = XMLHandler.getTagValue(stepnode, "levelAnddataTypeString");
-      currentRestructNumber =
-          Integer.parseInt(XMLHandler.getTagValue(stepnode, "currentRestructNumber"));
       partitionID = XMLHandler.getTagValue(stepnode, "partitionID");
       segmentId = XMLHandler.getTagValue(stepnode, "segmentId");
       taskNo = XMLHandler.getTagValue(stepnode, "taskNo");
@@ -232,8 +225,6 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta
       rep.saveStepAttribute(idTransformation, idStep, "factDimLensString", factDimLensString);
       rep.saveStepAttribute(idTransformation, idStep, "levelAnddataTypeString",
           levelAnddataTypeString);
-      rep.saveStepAttribute(idTransformation, idStep, "currentRestructNumber",
-          currentRestructNumber);
       rep.saveStepAttribute(idTransformation, idStep, "partitionID", partitionID);
       rep.saveStepAttribute(idTransformation, idStep, "segmentId", segmentId);
       rep.saveStepAttribute(idTransformation, idStep, "taskNo", taskNo);
@@ -278,7 +269,6 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta
       aggregatorString = rep.getStepAttributeString(idStep, "aggregatorString");
       factDimLensString = rep.getStepAttributeString(idStep, "factDimLensString");
       levelAnddataTypeString = rep.getStepAttributeString(idStep, "levelAnddataTypeString");
-      currentRestructNumber = (int) rep.getStepAttributeInteger(idStep, "currentRestructNumber");
       partitionID = rep.getStepAttributeString(idStep, "partitionID");
       segmentId = rep.getStepAttributeString(idStep, "segmentId");
       taskNo = rep.getStepAttributeString(idStep, "taskNo");
@@ -518,20 +508,6 @@ public class CarbonSliceMergerStepMeta extends BaseStepMeta
     this.factDimLensString = factDimLensString1;
   }
 
-  /**
-   * @return the currentRestructNumber
-   */
-  public int getCurrentRestructNumber() {
-    return currentRestructNumber;
-  }
-
-  /**
-   * @param currentRestructNum the currentRestructNumber to set
-   */
-  public void setCurrentRestructNumber(int currentRestructNum) {
-    this.currentRestructNumber = currentRestructNum;
-  }
-
   public String getLevelAnddataTypeString() {
     return levelAnddataTypeString;
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/5aedf544/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStepMeta.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStepMeta.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStepMeta.java
index a53fae4..9416134 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStepMeta.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStepMeta.java
@@ -97,8 +97,6 @@ public class SortKeyStepMeta extends BaseStepMeta implements StepMetaInterface {
    */
   private String updateMemberRequest;
 
-  private int currentRestructNumber;
-
   private String measureDataType;
 
   private String noDictionaryDims;
@@ -134,7 +132,6 @@ public class SortKeyStepMeta extends BaseStepMeta implements StepMetaInterface {
     complexDimensionCount = "";
     measureCount = "";
     updateMemberRequest = "";
-    currentRestructNumber = -1;
     measureDataType = "";
     partitionID = "";
     segmentId = "";
@@ -163,8 +160,6 @@ public class SortKeyStepMeta extends BaseStepMeta implements StepMetaInterface {
     retval.append("    ").append(XMLHandler.addTagValue("measureCount", this.measureCount));
     retval.append("    ")
         .append(XMLHandler.addTagValue("isUpdateMemberRequest", this.updateMemberRequest));
-    retval.append("    ")
-        .append(XMLHandler.addTagValue("currentRestructNumber", currentRestructNumber));
     retval.append("    ").append(XMLHandler.addTagValue("measureDataType", measureDataType));
     retval.append("    ").append(XMLHandler.addTagValue("partitionID", partitionID));
     retval.append("    ").append(XMLHandler.addTagValue("segmentId", segmentId));
@@ -195,8 +190,6 @@ public class SortKeyStepMeta extends BaseStepMeta implements StepMetaInterface {
       this.measureCount = XMLHandler.getTagValue(stepnode, "measureCount");
       this.updateMemberRequest = XMLHandler.getTagValue(stepnode, "isUpdateMemberRequest");
       this.measureDataType = XMLHandler.getTagValue(stepnode, "measureDataType");
-      currentRestructNumber =
-          Integer.parseInt(XMLHandler.getTagValue(stepnode, "currentRestructNumber"));
       this.partitionID = XMLHandler.getTagValue(stepnode, "partitionID");
       this.segmentId = XMLHandler.getTagValue(stepnode, "segmentId");
       this.taskNo = XMLHandler.getTagValue(stepnode, "taskNo");
@@ -231,8 +224,6 @@ public class SortKeyStepMeta extends BaseStepMeta implements StepMetaInterface {
       rep.saveStepAttribute(idTransformation, idStep, "measureCount", this.measureCount);
       rep.saveStepAttribute(idTransformation, idStep, "isUpdateMemberRequest",
           this.updateMemberRequest);
-      rep.saveStepAttribute(idTransformation, idStep, "currentRestructNumber",
-          currentRestructNumber);
       rep.saveStepAttribute(idTransformation, idStep, "measureDataType", measureDataType);
       rep.saveStepAttribute(idTransformation, idStep, "partitionID", partitionID);
       rep.saveStepAttribute(idTransformation, idStep, "segmentId", segmentId);
@@ -267,8 +258,6 @@ public class SortKeyStepMeta extends BaseStepMeta implements StepMetaInterface {
       this.measureCount = rep.getStepAttributeString(idStep, "measureCount");
       this.updateMemberRequest = rep.getStepAttributeString(idStep, "isUpdateMemberRequest");
       this.measureDataType = rep.getStepAttributeString(idStep, "measureDataType");
-      this.currentRestructNumber =
-          (int) rep.getStepAttributeInteger(idStep, "currentRestructNumber");
       this.partitionID = rep.getStepAttributeString(idStep, "partitionID");
       this.segmentId = rep.getStepAttributeString(idStep, "segmentId");
       this.taskNo = rep.getStepAttributeString(idStep, "taskNo");
@@ -450,20 +439,6 @@ public class SortKeyStepMeta extends BaseStepMeta implements StepMetaInterface {
     this.updateMemberRequest = isUpdateMemberRequest;
   }
 
-  /**
-   * @return the currentRestructNumber
-   */
-  public int getCurrentRestructNumber() {
-    return currentRestructNumber;
-  }
-
-  /**
-   * @param currentRestructNum the currentRestructNumber to set
-   */
-  public void setCurrentRestructNumber(int currentRestructNum) {
-    this.currentRestructNumber = currentRestructNum;
-  }
-
   public String getMeasureDataType() {
     return measureDataType;
   }


Mime
View raw message