carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [1/9] incubator-carbondata git commit: clean up core
Date Wed, 04 Jan 2017 13:34:55 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 82072ee19 -> 2ad623dcb


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
index 214e574..63d68f6 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/UnsafeSortDataRows.java
@@ -177,9 +177,9 @@ public class UnsafeSortDataRows {
    * record holder heap and then it will read first record from each file and
    * initialize the heap
    *
-   * @throws CarbonSortKeyAndGroupByException
+   * @throws InterruptedException
    */
-  public void startSorting() throws CarbonSortKeyAndGroupByException {
+  public void startSorting() throws InterruptedException {
     LOGGER.info("Unsafe based sorting will be used");
     if (this.rowPage.getUsedSize() > 0) {
       TimSort<UnsafeCarbonRow, IntPointerBuffer> timSort = new TimSort<>(
@@ -223,25 +223,19 @@ public class UnsafeSortDataRows {
 
   /**
    * This method will be used to delete sort temp location is it is exites
-   *
-   * @throws CarbonSortKeyAndGroupByException
    */
-  public void deleteSortLocationIfExists() throws CarbonSortKeyAndGroupByException {
+  public void deleteSortLocationIfExists() {
     CarbonDataProcessorUtil.deleteSortLocationIfExists(parameters.getTempFileLocation());
   }
 
   /**
    * Below method will be used to start file based merge
    *
-   * @throws CarbonSortKeyAndGroupByException
+   * @throws InterruptedException
    */
-  private void startFileBasedMerge() throws CarbonSortKeyAndGroupByException {
-    try {
-      dataSorterAndWriterExecutorService.shutdown();
-      dataSorterAndWriterExecutorService.awaitTermination(2, TimeUnit.DAYS);
-    } catch (InterruptedException e) {
-      throw new CarbonSortKeyAndGroupByException("Problem while shutdown the server ", e);
-    }
+  private void startFileBasedMerge() throws InterruptedException {
+    dataSorterAndWriterExecutorService.shutdown();
+    dataSorterAndWriterExecutorService.awaitTermination(2, TimeUnit.DAYS);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
index 735243e..a893ba8 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/sort/unsafe/merger/UnsafeIntermediateFileMerger.java
@@ -36,7 +36,6 @@ import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.CarbonUtilException;
 import org.apache.carbondata.core.util.DataTypeUtil;
 import org.apache.carbondata.processing.newflow.sort.unsafe.UnsafeCarbonRowPage;
 import org.apache.carbondata.processing.newflow.sort.unsafe.holder.SortTempChunkHolder;
@@ -357,7 +356,7 @@ public class UnsafeIntermediateFileMerger implements Callable<Void>
{
     try {
       CarbonUtil.deleteFiles(intermediateFiles);
       rowData.clear();
-    } catch (CarbonUtilException e) {
+    } catch (IOException e) {
       throw new CarbonSortKeyAndGroupByException("Problem while deleting the intermediate
files");
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
index 1c59d29..41f7cd2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
@@ -20,6 +20,7 @@
 package org.apache.carbondata.processing.newflow.steps;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.Iterator;
 
 import org.apache.carbondata.common.CarbonIterator;
@@ -33,7 +34,6 @@ import org.apache.carbondata.processing.newflow.DataField;
 import org.apache.carbondata.processing.newflow.constants.DataLoadProcessorConstants;
 import org.apache.carbondata.processing.newflow.converter.RowConverter;
 import org.apache.carbondata.processing.newflow.converter.impl.RowConverterImpl;
-import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException;
 import org.apache.carbondata.processing.newflow.row.CarbonRow;
 import org.apache.carbondata.processing.newflow.row.CarbonRowBatch;
 import org.apache.carbondata.processing.surrogatekeysgenerator.csvbased.BadRecordsLogger;
@@ -57,7 +57,7 @@ public class DataConverterProcessorStepImpl extends AbstractDataLoadProcessorSte
   }
 
   @Override
-  public void initialize() throws CarbonDataLoadingException {
+  public void initialize() throws IOException {
     child.initialize();
     BadRecordsLogger badRecordLogger = createBadRecordLogger();
     converter = new RowConverterImpl(child.getOutput(), configuration, badRecordLogger);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
index 16b203a..23d7af1 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
@@ -20,6 +20,7 @@
 package org.apache.carbondata.processing.newflow.steps;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
@@ -39,7 +40,6 @@ import org.apache.carbondata.processing.newflow.DataField;
 import org.apache.carbondata.processing.newflow.constants.DataLoadProcessorConstants;
 import org.apache.carbondata.processing.newflow.converter.RowConverter;
 import org.apache.carbondata.processing.newflow.converter.impl.RowConverterImpl;
-import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException;
 import org.apache.carbondata.processing.newflow.row.CarbonRow;
 import org.apache.carbondata.processing.newflow.row.CarbonRowBatch;
 import org.apache.carbondata.processing.surrogatekeysgenerator.csvbased.BadRecordsLogger;
@@ -65,7 +65,7 @@ public class DataConverterProcessorWithBucketingStepImpl extends AbstractDataLoa
   }
 
   @Override
-  public void initialize() throws CarbonDataLoadingException {
+  public void initialize() throws IOException {
     child.initialize();
     BadRecordsLogger badRecordLogger = createBadRecordLogger();
     converter = new RowConverterImpl(child.getOutput(), configuration, badRecordLogger);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
index 8318530..6f1c035 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
@@ -19,6 +19,7 @@
 package org.apache.carbondata.processing.newflow.steps;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.Iterator;
 
 import org.apache.carbondata.common.logging.LogService;
@@ -78,7 +79,7 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep
{
     return child.getOutput();
   }
 
-  @Override public void initialize() throws CarbonDataLoadingException {
+  @Override public void initialize() throws IOException {
     child.initialize();
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
index ef43751..b8f08e9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
@@ -18,6 +18,7 @@
  */
 package org.apache.carbondata.processing.newflow.steps;
 
+import java.io.IOException;
 import java.util.Iterator;
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
@@ -53,7 +54,7 @@ public class SortProcessorStepImpl extends AbstractDataLoadProcessorStep
{
   }
 
   @Override
-  public void initialize() throws CarbonDataLoadingException {
+  public void initialize() throws IOException {
     child.initialize();
     SortParameters sortParameters = SortParameters.createSortParameters(configuration);
     boolean offheapsort = Boolean.parseBoolean(CarbonProperties.getInstance()

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java
b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java
index f1b062e..db9d379 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/IntermediateFileMerger.java
@@ -33,7 +33,6 @@ import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.CarbonUtilException;
 import org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException;
 import org.apache.carbondata.processing.util.RemoveDictionaryUtil;
 
@@ -453,7 +452,7 @@ public class IntermediateFileMerger implements Callable<Void> {
     }
     try {
       CarbonUtil.deleteFiles(intermediateFiles);
-    } catch (CarbonUtilException e) {
+    } catch (IOException e) {
       throw new CarbonSortKeyAndGroupByException("Problem while deleting the intermediate
files");
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
index 9b3bec7..c92b11b 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
@@ -20,6 +20,7 @@
 package org.apache.carbondata.processing.surrogatekeysgenerator.csvbased;
 
 import java.io.File;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -27,8 +28,6 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
 
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.cache.Cache;
 import org.apache.carbondata.core.cache.CacheProvider;
 import org.apache.carbondata.core.cache.CacheType;
@@ -39,12 +38,10 @@ import org.apache.carbondata.core.carbon.ColumnIdentifier;
 import org.apache.carbondata.core.carbon.metadata.CarbonMetadata;
 import org.apache.carbondata.core.carbon.metadata.schema.table.CarbonTable;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
 import org.apache.carbondata.core.keygenerator.KeyGenException;
 import org.apache.carbondata.core.keygenerator.KeyGenerator;
 import org.apache.carbondata.core.util.CarbonProperties;
 import org.apache.carbondata.core.util.CarbonTimeStatisticsFactory;
-import org.apache.carbondata.core.util.CarbonUtilException;
 import org.apache.carbondata.core.writer.ByteArrayHolder;
 import org.apache.carbondata.core.writer.HierarchyValueWriterForCSV;
 import org.apache.carbondata.processing.datatypes.GenericDataType;
@@ -61,12 +58,6 @@ import org.pentaho.di.core.exception.KettleException;
 public class FileStoreSurrogateKeyGenForCSV extends CarbonCSVBasedDimSurrogateKeyGen {
 
   /**
-   * LOGGER
-   */
-  private static final LogService LOGGER =
-      LogServiceFactory.getLogService(FileStoreSurrogateKeyGenForCSV.class.getName());
-
-  /**
    * hierValueWriter
    */
   private Map<String, HierarchyValueWriterForCSV> hierValueWriter;
@@ -77,21 +68,11 @@ public class FileStoreSurrogateKeyGenForCSV extends CarbonCSVBasedDimSurrogateKe
   private Map<String, KeyGenerator> keyGenerator;
 
   /**
-   * baseStorePath
-   */
-  private String baseStorePath;
-
-  /**
    * LOAD_FOLDER
    */
   private String loadFolderName;
 
   /**
-   * folderList
-   */
-  private List<CarbonFile> folderList = new ArrayList<CarbonFile>(5);
-
-  /**
    * primaryKeyStringArray
    */
   private String[] primaryKeyStringArray;
@@ -110,10 +91,10 @@ public class FileStoreSurrogateKeyGenForCSV extends CarbonCSVBasedDimSurrogateKe
 
   /**
    * @param columnsInfo
-   * @throws KettleException
+   * @throws IOException
    */
   public FileStoreSurrogateKeyGenForCSV(ColumnsInfo columnsInfo, String partitionID, int
segmentId,
-      String taskNo) throws KettleException {
+      String taskNo) throws IOException {
     super(columnsInfo);
     populatePrimaryKeyarray(dimInsertFileNames, columnsInfo.getPrimaryKeyMap());
     this.partitionID = partitionID;
@@ -121,7 +102,6 @@ public class FileStoreSurrogateKeyGenForCSV extends CarbonCSVBasedDimSurrogateKe
     this.taskNo = taskNo;
     keyGenerator = new HashMap<String, KeyGenerator>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
 
-    baseStorePath = columnsInfo.getBaseStoreLocation();
     setStoreFolderWithLoadNumber(
         checkAndCreateLoadFolderNumber(columnsInfo.getDatabaseName(),
             columnsInfo.getTableName()));
@@ -191,13 +171,13 @@ public class FileStoreSurrogateKeyGenForCSV extends CarbonCSVBasedDimSurrogateKe
   }
 
   private String checkAndCreateLoadFolderNumber(String databaseName,
-      String tableName) throws KettleException {
+      String tableName) throws IOException {
     String carbonDataDirectoryPath = CarbonDataProcessorUtil
         .getLocalDataFolderLocation(databaseName, tableName, taskNo, partitionID, segmentId
+ "",
             false);
     boolean isDirCreated = new File(carbonDataDirectoryPath).mkdirs();
     if (!isDirCreated) {
-      throw new KettleException("Unable to create data load directory" + carbonDataDirectoryPath);
+      throw new IOException("Unable to create data load directory" + carbonDataDirectoryPath);
     }
     return carbonDataDirectoryPath;
   }
@@ -214,7 +194,7 @@ public class FileStoreSurrogateKeyGenForCSV extends CarbonCSVBasedDimSurrogateKe
   /**
    * This method will generate cache for all the global dictionaries during data loading.
    */
-  private void populateCache() throws KettleException {
+  private void populateCache() throws IOException {
     String carbonStorePath =
         CarbonProperties.getInstance().getProperty(CarbonCommonConstants.STORE_LOCATION_HDFS);
     String[] dimColumnNames = columnsInfo.getDimColNames();
@@ -268,7 +248,7 @@ public class FileStoreSurrogateKeyGenForCSV extends CarbonCSVBasedDimSurrogateKe
       }
     }
     initDictionaryCacheInfo(dictionaryKeys, dictionaryColumnUniqueIdentifiers,
-        reverseDictionaryCache, carbonStorePath);
+        reverseDictionaryCache);
   }
 
   /**
@@ -277,25 +257,20 @@ public class FileStoreSurrogateKeyGenForCSV extends CarbonCSVBasedDimSurrogateKe
    * @param dictionaryKeys
    * @param dictionaryColumnUniqueIdentifiers
    * @param reverseDictionaryCache
-   * @param carbonStorePath
    * @throws KettleException
    */
   private void initDictionaryCacheInfo(List<String> dictionaryKeys,
       List<DictionaryColumnUniqueIdentifier> dictionaryColumnUniqueIdentifiers,
-      Cache reverseDictionaryCache, String carbonStorePath) throws KettleException {
+      Cache reverseDictionaryCache) throws IOException {
     long lruCacheStartTime = System.currentTimeMillis();
-    try {
-      List reverseDictionaries = reverseDictionaryCache.getAll(dictionaryColumnUniqueIdentifiers);
-      for (int i = 0; i < reverseDictionaries.size(); i++) {
-        Dictionary reverseDictionary = (Dictionary) reverseDictionaries.get(i);
-        getDictionaryCaches().put(dictionaryKeys.get(i), reverseDictionary);
-        updateMaxKeyInfo(dictionaryKeys.get(i), reverseDictionary.getDictionaryChunks().getSize());
-      }
-      CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordLruCacheLoadTime(
-          (System.currentTimeMillis() - lruCacheStartTime)/1000.0);
-    } catch (CarbonUtilException e) {
-      throw new KettleException(e.getMessage());
+    List reverseDictionaries = reverseDictionaryCache.getAll(dictionaryColumnUniqueIdentifiers);
+    for (int i = 0; i < reverseDictionaries.size(); i++) {
+      Dictionary reverseDictionary = (Dictionary) reverseDictionaries.get(i);
+      getDictionaryCaches().put(dictionaryKeys.get(i), reverseDictionary);
+      updateMaxKeyInfo(dictionaryKeys.get(i), reverseDictionary.getDictionaryChunks().getSize());
     }
+    CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordLruCacheLoadTime(
+        (System.currentTimeMillis() - lruCacheStartTime)/1000.0);
   }
 
   @Override protected int getSurrogateFromStore(String value, int index, Object[] properties)

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 3cb984d..cc142e7 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -52,10 +52,8 @@ import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile;
 import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFileFilter;
 import org.apache.carbondata.core.datastorage.store.impl.FileFactory;
 import org.apache.carbondata.core.datastorage.store.impl.FileFactory.FileType;
-import org.apache.carbondata.core.load.LoadMetadataDetails;
 import org.apache.carbondata.core.util.CarbonProperties;
 import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.CarbonUtilException;
 import org.apache.carbondata.core.util.DataTypeUtil;
 import org.apache.carbondata.processing.datatypes.ArrayDataType;
 import org.apache.carbondata.processing.datatypes.GenericDataType;
@@ -63,7 +61,6 @@ import org.apache.carbondata.processing.datatypes.PrimitiveDataType;
 import org.apache.carbondata.processing.datatypes.StructDataType;
 import org.apache.carbondata.processing.etl.DataLoadingException;
 import org.apache.carbondata.processing.newflow.DataField;
-import org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException;
 
 import org.apache.commons.lang3.ArrayUtils;
 import org.pentaho.di.core.CheckResult;
@@ -225,11 +222,8 @@ public final class CarbonDataProcessorUtil {
 
   /**
    * This method will be used to delete sort temp location is it is exites
-   *
-   * @throws CarbonSortKeyAndGroupByException
    */
-  public static void deleteSortLocationIfExists(String tempFileLocation)
-      throws CarbonSortKeyAndGroupByException {
+  public static void deleteSortLocationIfExists(String tempFileLocation) {
     // create new temp file location where this class
     //will write all the temp files
     File file = new File(tempFileLocation);
@@ -237,43 +231,13 @@ public final class CarbonDataProcessorUtil {
     if (file.exists()) {
       try {
         CarbonUtil.deleteFoldersAndFiles(file);
-      } catch (CarbonUtilException e) {
+      } catch (IOException | InterruptedException e ) {
         LOGGER.error(e);
       }
     }
   }
 
   /**
-   * return the modification TimeStamp Separated by HASH_SPC_CHARACTER
-   */
-  public static String getLoadNameFromLoadMetaDataDetails(
-      List<LoadMetadataDetails> loadMetadataDetails) {
-    StringBuilder builder = new StringBuilder();
-    for (LoadMetadataDetails loadMetadataDetail : loadMetadataDetails) {
-      builder.append(CarbonCommonConstants.LOAD_FOLDER).append(loadMetadataDetail.getLoadName())
-          .append(CarbonCommonConstants.HASH_SPC_CHARACTER);
-    }
-    String loadNames =
-        builder.substring(0, builder.lastIndexOf(CarbonCommonConstants.HASH_SPC_CHARACTER));
-    return loadNames;
-  }
-
-  /**
-   * return the modOrDelTimesStamp TimeStamp Separated by HASH_SPC_CHARACTER
-   */
-  public static String getModificationOrDeletionTimesFromLoadMetadataDetails(
-      List<LoadMetadataDetails> loadMetadataDetails) {
-    StringBuilder builder = new StringBuilder();
-    for (LoadMetadataDetails loadMetadataDetail : loadMetadataDetails) {
-      builder.append(loadMetadataDetail.getModificationOrdeletionTimesStamp())
-          .append(CarbonCommonConstants.HASH_SPC_CHARACTER);
-    }
-    String modOrDelTimesStamp =
-        builder.substring(0, builder.indexOf(CarbonCommonConstants.HASH_SPC_CHARACTER));
-    return modOrDelTimesStamp;
-  }
-
-  /**
    * This method will form the local data folder store location
    *
    * @param databaseName

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
b/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
index 136a8f3..fa5aff3 100644
--- a/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
+++ b/processing/src/test/java/org/apache/carbondata/carbon/datastore/BlockIndexStoreTest.java
@@ -40,13 +40,9 @@ import org.apache.carbondata.core.carbon.datastore.block.TableBlockInfo;
 import org.apache.carbondata.core.carbon.datastore.block.TableBlockUniqueIdentifier;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.util.CarbonProperties;
-import org.apache.carbondata.core.util.CarbonUtilException;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.util.CarbonProperties;
 import org.apache.carbondata.test.util.StoreCreator;
 
 import junit.framework.TestCase;
-
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -77,7 +73,7 @@ public class BlockIndexStoreTest extends TestCase {
 	  }
 
   @Test public void testLoadAndGetTaskIdToSegmentsMapForSingleSegment()
-      throws IOException, CarbonUtilException {
+      throws IOException {
     String canonicalPath =
         new File(this.getClass().getResource("/").getPath() + "/../../").getCanonicalPath();
     File file = getPartFile();
@@ -94,7 +90,7 @@ public class BlockIndexStoreTest extends TestCase {
           getTableBlockUniqueIdentifierList(Arrays.asList(new TableBlockInfo[] { info }),
absoluteTableIdentifier);
       List<AbstractIndex> loadAndGetBlocks = cache.getAll(tableBlockInfoList);
       assertTrue(loadAndGetBlocks.size() == 1);
-    } catch (CarbonUtilException e) {
+    } catch (Exception e) {
       assertTrue(false);
     }
     List<String> segmentIds = new ArrayList<>();
@@ -161,7 +157,7 @@ public class BlockIndexStoreTest extends TestCase {
           getTableBlockUniqueIdentifierList(tableBlockInfos, absoluteTableIdentifier);
       List<AbstractIndex> loadAndGetBlocks = cache.getAll(tableBlockUniqueIdentifiers);
       assertTrue(loadAndGetBlocks.size() == 5);
-    } catch (CarbonUtilException e) {
+    } catch (Exception e) {
       assertTrue(false);
     }
     List<String> segmentIds = new ArrayList<>();
@@ -233,7 +229,7 @@ public class BlockIndexStoreTest extends TestCase {
           getTableBlockUniqueIdentifierList(tableBlockInfos, absoluteTableIdentifier);
       List<AbstractIndex> loadAndGetBlocks = cache.getAll(blockUniqueIdentifierList);
       assertTrue(loadAndGetBlocks.size() == 8);
-    } catch (CarbonUtilException e) {
+    } catch (Exception e) {
       assertTrue(false);
     }
     List<String> segmentIds = new ArrayList<>();

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/eaadc88a/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java b/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
index 95f3d35..7e9feb6 100644
--- a/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
+++ b/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
@@ -82,7 +82,6 @@ import org.apache.carbondata.processing.dataprocessor.DataProcessTaskStatus;
 import org.apache.carbondata.processing.dataprocessor.IDataProcessStatus;
 import org.apache.carbondata.processing.graphgenerator.GraphGenerator;
 import org.apache.carbondata.processing.graphgenerator.GraphGeneratorException;
-import org.apache.carbondata.processing.util.CarbonDataProcessorUtil;
 
 import com.google.gson.Gson;
 import org.apache.hadoop.fs.Path;
@@ -377,8 +376,7 @@ public class StoreCreator {
 
     DataGraphExecuter graphExecuter = new DataGraphExecuter(dataProcessTaskStatus);
     graphExecuter
-        .executeGraph(graphPath, new ArrayList<String>(CarbonCommonConstants.CONSTANT_SIZE_TEN),
-            info, "0", loadModel.getSchema());
+        .executeGraph(graphPath, info, loadModel.getSchema());
     //    LoadMetadataDetails[] loadDetails =
     //        CarbonUtil.readLoadMetadata(loadModel.schema.getCarbonTable().getMetaDataFilepath());
     writeLoadMetadata(loadModel.schema, loadModel.getTableName(), loadModel.getTableName(),


Mime
View raw message