carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject carbondata git commit: [CARBONDATA-1326] Fixed high priority findbug issues
Date Sat, 22 Jul 2017 09:05:33 GMT
Repository: carbondata
Updated Branches:
  refs/heads/metadata 67b59634f -> 2c9866e85


[CARBONDATA-1326] Fixed high priority findbug issues

Fixed high priority findbug issues in the code

This closes #1191


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2c9866e8
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2c9866e8
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2c9866e8

Branch: refs/heads/metadata
Commit: 2c9866e85451170d9539c353e70d38fc35567e46
Parents: 67b5963
Author: manishgupta88 <tomanishgupta18@gmail.com>
Authored: Fri Jul 21 21:48:20 2017 +0800
Committer: Jacky Li <jacky.likun@qq.com>
Committed: Sat Jul 22 17:05:13 2017 +0800

----------------------------------------------------------------------
 .../cache/dictionary/ColumnDictionaryInfo.java  | 10 +++++---
 .../dictionary/DoubleArrayTrieDictionary.java   |  7 ++++--
 .../core/constants/CarbonCommonConstants.java   | 13 +++++-----
 .../core/datastore/block/BlockInfo.java         |  4 ++-
 .../block/SegmentTaskIndexWrapper.java          | 17 -------------
 .../core/datastore/block/TableBlockInfo.java    |  7 ++++--
 .../core/datastore/page/ColumnPage.java         |  2 +-
 .../generator/key/DictionaryMessage.java        | 16 +++++++-----
 .../core/indexstore/DataMapStoreManager.java    |  2 +-
 .../datatype/DecimalConverterFactory.java       |  2 +-
 .../core/mutate/DeleteDeltaBlockletDetails.java |  6 +++--
 .../scan/filter/FilterExpressionProcessor.java  | 26 +++++++++++++++++---
 .../scan/filter/intf/FilterOptimizerImpl.java   |  2 +-
 .../RowLevelRangeFilterResolverImpl.java        |  2 +-
 .../core/scan/result/AbstractScannedResult.java |  7 ++++--
 .../statusmanager/SegmentStatusManager.java     |  5 +++-
 .../SegmentUpdateStatusManager.java             |  3 ---
 .../CarbonDictionarySortInfoPreparator.java     |  4 +--
 .../carbondata/hadoop/CarbonInputSplit.java     |  7 ++++--
 .../internal/index/impl/InMemoryBTreeIndex.java |  2 +-
 .../carbondata/spark/util/CarbonScalaUtil.scala |  4 ++-
 .../org/apache/spark/util/TableAPIUtil.scala    |  2 +-
 .../processing/csvload/CSVInputFormat.java      |  5 +++-
 .../newflow/AbstractDataLoadProcessorStep.java  | 26 ++++++++++----------
 .../CarbonRowDataWriterProcessorStepImpl.java   |  1 +
 .../steps/DataConverterProcessorStepImpl.java   |  1 +
 ...ConverterProcessorWithBucketingStepImpl.java |  1 +
 .../steps/DataWriterBatchProcessorStepImpl.java |  1 +
 .../steps/DataWriterProcessorStepImpl.java      |  1 +
 .../newflow/steps/InputProcessorStepImpl.java   |  5 ++--
 .../newflow/steps/SortProcessorStepImpl.java    |  1 +
 .../writer/v2/CarbonFactDataWriterImplV2.java   |  4 ++-
 32 files changed, 117 insertions(+), 79 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
index d3599da..260ba90 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/ColumnDictionaryInfo.java
@@ -194,8 +194,9 @@ public class ColumnDictionaryInfo extends AbstractColumnDictionaryInfo
{
       byte[] dictionaryValue = getDictionaryBytesFromSurrogate(surrogateKey);
       int cmp = -1;
       if (this.getDataType() != DataType.STRING) {
-        cmp = compareFilterKeyWithDictionaryKey(new String(dictionaryValue), filterKey,
-            this.getDataType());
+        cmp = compareFilterKeyWithDictionaryKey(
+            new String(dictionaryValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)),
+            filterKey, this.getDataType());
 
       } else {
         cmp = ByteUtil.UnsafeComparer.INSTANCE.compareTo(dictionaryValue, key);
@@ -240,8 +241,9 @@ public class ColumnDictionaryInfo extends AbstractColumnDictionaryInfo
{
         if (null == dictionaryValue) {
           cmp = -1;
         } else if (this.getDataType() != DataType.STRING) {
-          cmp = compareFilterKeyWithDictionaryKey(new String(dictionaryValue), filterKey,
-              this.getDataType());
+          cmp = compareFilterKeyWithDictionaryKey(
+              new String(dictionaryValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)),
+              filterKey, this.getDataType());
 
         } else {
           cmp =

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
index f738008..cab1926 100644
--- a/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
+++ b/core/src/main/java/org/apache/carbondata/core/cache/dictionary/DoubleArrayTrieDictionary.java
@@ -21,9 +21,12 @@ import java.io.DataInputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.nio.charset.Charset;
 import java.util.SortedSet;
 import java.util.TreeSet;
 
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+
 /**
  * A dictionary based on DoubleArrayTrie data structure that maps enumerations
  * of byte[] to int IDs. With DoubleArrayTrie the memory footprint of the mapping
@@ -116,7 +119,7 @@ public class DoubleArrayTrieDictionary {
    */
   public int getValue(String key) {
     String k = key + '\0';
-    byte[] bKeys = k.getBytes();
+    byte[] bKeys = k.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
     return getValue(bKeys);
   }
 
@@ -315,7 +318,7 @@ public class DoubleArrayTrieDictionary {
    */
   public boolean insert(String key) {
     String k = key + '\0';
-    byte[] bKeys = k.getBytes();
+    byte[] bKeys = k.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
     if (!insert(bKeys)) {
       return false;
     }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index b0f9d32..91be1bd 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -171,9 +171,15 @@ public final class CarbonCommonConstants {
   public static final String MEMBER_DEFAULT_VAL = "@NU#LL$!";
 
   /**
+   * default charset to be used for reading and writing
+   */
+  public static final String DEFAULT_CHARSET = "UTF-8";
+
+  /**
    * MEMBER_DEFAULT_VAL_ARRAY
    */
-  public static final byte[] MEMBER_DEFAULT_VAL_ARRAY = MEMBER_DEFAULT_VAL.getBytes();
+  public static final byte[] MEMBER_DEFAULT_VAL_ARRAY =
+      MEMBER_DEFAULT_VAL.getBytes(Charset.forName(DEFAULT_CHARSET));
 
   /**
    * Bytes for string 0, it is used in codegen in case of null values.
@@ -758,11 +764,6 @@ public final class CarbonCommonConstants {
   public static final String ENABLE_XXHASH_DEFAULT = "true";
 
   /**
-   * default charset to be used for reading and writing
-   */
-  public static final String DEFAULT_CHARSET = "UTF-8";
-
-  /**
    * default charset class to be used for reading and writing
    */
   public static final Charset DEFAULT_CHARSET_CLASS = Charset.forName(DEFAULT_CHARSET);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/core/src/main/java/org/apache/carbondata/core/datastore/block/BlockInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/BlockInfo.java
b/core/src/main/java/org/apache/carbondata/core/datastore/block/BlockInfo.java
index 2833316..d2eae8b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/BlockInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/BlockInfo.java
@@ -16,6 +16,8 @@
  */
 package org.apache.carbondata.core.datastore.block;
 
+import java.io.Serializable;
+
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.util.path.CarbonTablePath;
 
@@ -26,7 +28,7 @@ import org.apache.carbondata.core.util.path.CarbonTablePath;
  * block will be loaded together. This class will be used to store table block info
  * and equals and hash code method is used to identify blocklet belongs to same block
  */
-public class BlockInfo {
+public class BlockInfo implements Serializable {
 
   /**
    * table block info, stores all the details

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java
b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java
index 516cd6a..4fe6d1b 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/SegmentTaskIndexWrapper.java
@@ -23,7 +23,6 @@ import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.carbondata.core.cache.Cacheable;
 import org.apache.carbondata.core.datastore.SegmentTaskIndexStore;
-import org.apache.carbondata.core.mutate.UpdateVO;
 
 /**
  * SegmentTaskIndexWrapper class holds the  taskIdToTableSegmentMap
@@ -45,7 +44,6 @@ public class SegmentTaskIndexWrapper implements Cacheable {
   protected AtomicLong memorySize = new AtomicLong();
 
   private Long refreshedTimeStamp;
-  private UpdateVO invalidTaskKey;
   public SegmentTaskIndexWrapper(
       Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> taskIdToTableSegmentMap)
{
     this.taskIdToTableSegmentMap = taskIdToTableSegmentMap;
@@ -128,19 +126,4 @@ public class SegmentTaskIndexWrapper implements Cacheable {
     this.refreshedTimeStamp = refreshedTimeStamp;
   }
 
-  public void removeEntryFromCacheAndRefresh(String taskId) {
-    AbstractIndex blockEntry = this.getTaskIdToTableSegmentMap().remove(taskId);
-    if (null != blockEntry) {
-      memorySize.set(memorySize.get() - blockEntry.getMemorySize());
-    }
-  }
-
-  public void setLastUpdateVO(UpdateVO invalidTaskKey) {
-    this.invalidTaskKey = invalidTaskKey;
-  }
-
-  public UpdateVO getInvalidTaskKey() {
-    return invalidTaskKey;
-  }
-
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
index 44347cf..1da6699 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/block/TableBlockInfo.java
@@ -17,6 +17,7 @@
 package org.apache.carbondata.core.datastore.block;
 
 import java.io.Serializable;
+import java.nio.charset.Charset;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -221,8 +222,10 @@ public class TableBlockInfo implements Distributable, Serializable {
     // offset of
     // the file
     if (CarbonTablePath.isCarbonDataFile(filePath)) {
-      int compare = ByteUtil.compare(DataFileUtil.getTaskNo(filePath).getBytes(),
-          DataFileUtil.getTaskNo(((TableBlockInfo) other).filePath).getBytes());
+      int compare = ByteUtil.compare(DataFileUtil.getTaskNo(filePath)
+              .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)),
+          DataFileUtil.getTaskNo(((TableBlockInfo) other).filePath)
+              .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
       if (compare != 0) {
         return compare;
       }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
index 2c43165..ef2c9ec 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/ColumnPage.java
@@ -109,7 +109,7 @@ public abstract class ColumnPage {
   }
 
   private static ColumnPage createPage(DataType dataType, int pageSize, int scale, int precision)
{
-    if (dataType.equals(BYTE_ARRAY) | dataType.equals(DECIMAL)) {
+    if (dataType.equals(BYTE_ARRAY) || dataType.equals(DECIMAL)) {
       return createVarLengthPage(dataType, pageSize, scale, precision);
     } else {
       return createFixLengthPage(dataType, pageSize, scale, precision);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
index 2ae0521..0bb82f9 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/key/DictionaryMessage.java
@@ -16,6 +16,8 @@
  */
 package org.apache.carbondata.core.dictionary.generator.key;
 
+import java.nio.charset.Charset;
+
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 
 import io.netty.buffer.ByteBuf;
@@ -53,11 +55,12 @@ public class DictionaryMessage {
   public void readData(ByteBuf byteBuf) {
     byte[] tableBytes = new byte[byteBuf.readInt()];
     byteBuf.readBytes(tableBytes);
-    tableUniqueName = new String(tableBytes);
+    tableUniqueName =
+        new String(tableBytes, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
 
     byte[] colBytes = new byte[byteBuf.readInt()];
     byteBuf.readBytes(colBytes);
-    columnName = new String(colBytes);
+    columnName = new String(colBytes, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
 
     byte typeByte = byteBuf.readByte();
     type = getKeyType(typeByte);
@@ -68,7 +71,7 @@ public class DictionaryMessage {
     } else {
       byte[] dataBytes = new byte[byteBuf.readInt()];
       byteBuf.readBytes(dataBytes);
-      data = new String(dataBytes);
+      data = new String(dataBytes, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
     }
   }
 
@@ -77,11 +80,12 @@ public class DictionaryMessage {
     // Just reserve the bytes to add length of header at last.
     byteBuf.writeShort(Short.MAX_VALUE);
 
-    byte[] tableBytes = tableUniqueName.getBytes();
+    byte[] tableBytes =
+        tableUniqueName.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
     byteBuf.writeInt(tableBytes.length);
     byteBuf.writeBytes(tableBytes);
 
-    byte[] colBytes = columnName.getBytes();
+    byte[] colBytes = columnName.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
     byteBuf.writeInt(colBytes.length);
     byteBuf.writeBytes(colBytes);
 
@@ -92,7 +96,7 @@ public class DictionaryMessage {
       byteBuf.writeInt(dictionaryValue);
     } else {
       byteBuf.writeByte(1);
-      byte[] dataBytes = data.getBytes();
+      byte[] dataBytes = data.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
       byteBuf.writeInt(dataBytes.length);
       byteBuf.writeBytes(dataBytes);
     }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java
b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java
index 06638ad..64c6e20 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/DataMapStoreManager.java
@@ -86,7 +86,7 @@ public class DataMapStoreManager {
     } catch (Exception e) {
       LOGGER.error(e);
     }
-    dataMap.init(identifier, dataMapName);
+    // TODO: Initialize a data map by calling init method on the data map
     map.put(dataMapName, dataMap);
     return dataMap;
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
index 555df1c..459eb24 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DecimalConverterFactory.java
@@ -29,7 +29,7 @@ import org.apache.carbondata.core.util.DataTypeUtil;
  */
 public final class DecimalConverterFactory {
 
-  public static DecimalConverterFactory INSTANCE = new DecimalConverterFactory();
+  public static final DecimalConverterFactory INSTANCE = new DecimalConverterFactory();
 
   private int[] minBytesForPrecision = minBytesForPrecision();
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java
b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java
index 0f54f3a..22785fa 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/DeleteDeltaBlockletDetails.java
@@ -69,9 +69,11 @@ public class DeleteDeltaBlockletDetails implements Serializable {
     if (obj == null || !(obj instanceof DeleteDeltaBlockletDetails)) {
       return false;
     }
-
     DeleteDeltaBlockletDetails that = (DeleteDeltaBlockletDetails) obj;
-    return id.equals(that.id) && pageId == that.pageId;
+    if (null == pageId || null == that.pageId) {
+      return false;
+    }
+    return id.equals(that.id) && (pageId.intValue() == that.pageId.intValue());
   }
 
   @Override public int hashCode() {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
index ce31283..36e1fb3 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterExpressionProcessor.java
@@ -130,11 +130,14 @@ public class FilterExpressionProcessor implements FilterProcessor {
       }
     }
     if (LOGGER.isDebugEnabled()) {
+      char delimiter = ',';
       LOGGER.debug(
-          "Successfully retrieved the start and end key" + "Dictionary Start Key: " + searchStartKey
-              .getDictionaryKeys() + "No Dictionary Start Key " + searchStartKey
-              .getNoDictionaryKeys() + "Dictionary End Key: " + searchEndKey.getDictionaryKeys()
-              + "No Dictionary End Key " + searchEndKey.getNoDictionaryKeys());
+          "Successfully retrieved the start and end key" + "Dictionary Start Key: " + joinByteArray(
+              searchStartKey.getDictionaryKeys(), delimiter) + "No Dictionary Start Key "
+              + joinByteArray(searchStartKey.getNoDictionaryKeys(), delimiter)
+              + "Dictionary End Key: " + joinByteArray(searchEndKey.getDictionaryKeys(),
delimiter)
+              + "No Dictionary End Key " + joinByteArray(searchEndKey.getNoDictionaryKeys(),
+              delimiter));
     }
     long startTimeInMillis = System.currentTimeMillis();
     DataRefNodeFinder blockFinder = new BTreeDataRefNodeFinder(
@@ -158,6 +161,21 @@ public class FilterExpressionProcessor implements FilterProcessor {
     return listOfDataBlocksToScan;
   }
 
+  private String joinByteArray(byte[] bytes, char delimiter) {
+    String byteArrayAsString = "";
+    if (null != bytes) {
+      for (int i = 0; i < bytes.length; i++) {
+        byteArrayAsString = byteArrayAsString + delimiter + bytes[i];
+      }
+      if (byteArrayAsString.length() > 0) {
+        byteArrayAsString = byteArrayAsString.substring(1);
+      }
+    } else {
+      byteArrayAsString = null;
+    }
+    return byteArrayAsString;
+  }
+
   /**
    * Get the map of required partitions
    * The value of "1" in BitSet represent the required partition

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java
index 145fc09..cb9495b 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/intf/FilterOptimizerImpl.java
@@ -30,6 +30,6 @@ public class FilterOptimizerImpl implements FilterOptimizer {
   }
 
   @Override public Expression optimizeFilter() {
-    return this.optimizeFilter();
+    return filterExpression;
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
index 6d4e528..47ca7e1 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/resolver/RowLevelRangeFilterResolverImpl.java
@@ -145,7 +145,7 @@ public class RowLevelRangeFilterResolverImpl extends ConditionalFilterResolverIm
     for (ExpressionResult result : listOfExpressionResults) {
       try {
         if (result.getString() == null) {
-          filterValuesList.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL.getBytes());
+          filterValuesList.add(CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY);
           continue;
         }
         filterValuesList.add(DataTypeUtil

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
b/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
index e78383d..21cf6c8 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/result/AbstractScannedResult.java
@@ -21,6 +21,7 @@ import java.io.DataOutputStream;
 import java.io.IOException;
 import java.math.BigDecimal;
 import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
 import java.util.Map;
 
 import org.apache.carbondata.common.logging.LogService;
@@ -304,7 +305,8 @@ public abstract class AbstractScannedResult {
               j :
               rowMapping[pageCounter][j]);
         }
-        vector.putBytes(vectorOffset++, data.getBytes());
+        vector.putBytes(vectorOffset++,
+            data.getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
       }
     }
   }
@@ -394,7 +396,8 @@ public abstract class AbstractScannedResult {
     int position = 0;
     for (int i = 0; i < this.noDictionaryColumnBlockIndexes.length; i++) {
       noDictionaryColumnsKeys[position++] = new String(
-          dimensionDataChunks[noDictionaryColumnBlockIndexes[i]][pageCounter].getChunkData(rowId));
+          dimensionDataChunks[noDictionaryColumnBlockIndexes[i]][pageCounter].getChunkData(rowId),
+          Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
     }
     return noDictionaryColumnsKeys;
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
index beef0c6..28d3f18 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentStatusManager.java
@@ -34,6 +34,7 @@ import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.datastore.impl.FileFactory;
+import org.apache.carbondata.core.datastore.row.LoadStatusType;
 import org.apache.carbondata.core.fileoperations.AtomicFileOperations;
 import org.apache.carbondata.core.fileoperations.AtomicFileOperationsImpl;
 import org.apache.carbondata.core.fileoperations.FileWriteOperation;
@@ -526,7 +527,9 @@ public class SegmentStatusManager {
               + "as the segment has been compacted.");
           continue;
         }
-        if (!CarbonCommonConstants.MARKED_FOR_DELETE.equals(loadMetadata.getLoadStatus()))
{
+        if (!CarbonCommonConstants.MARKED_FOR_DELETE.equals(loadMetadata.getLoadStatus())
+            && !LoadStatusType.IN_PROGRESS.getMessage().equals(loadMetadata.getLoadStatus())
+            && !LoadStatusType.INSERT_OVERWRITE.getMessage().equals(loadMetadata.getLoadStatus()))
{
           loadFound = true;
           updateSegmentMetadataDetails(loadMetadata);
           LOG.info("Info: " +

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
index 3e0a2cb..9abc58f 100644
--- a/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/statusmanager/SegmentUpdateStatusManager.java
@@ -788,9 +788,6 @@ public class SegmentUpdateStatusManager {
         CarbonUpdateUtil.getRequiredFieldFromTID(completeBlockName, TupleIdEnum.BLOCK_ID)));
     String segmentId =
         CarbonUpdateUtil.getRequiredFieldFromTID(completeBlockName, TupleIdEnum.SEGMENT_ID);
-    CarbonTablePath carbonTablePath = CarbonStorePath
-        .getCarbonTablePath(absoluteTableIdentifier.getStorePath(),
-            absoluteTableIdentifier.getCarbonTableIdentifier());
     SegmentUpdateDetails[] listOfSegmentUpdateDetailsArray =
         readLoadMetadata();
     for (SegmentUpdateDetails block : listOfSegmentUpdateDetailsArray) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java
b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java
index 61ae6c2..8c7d922 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/sortindex/CarbonDictionarySortInfoPreparator.java
@@ -123,8 +123,8 @@ public class CarbonDictionarySortInfoPreparator {
     // for new distinct values
     Iterator<String> distinctValue = distinctValues.iterator();
     while (distinctValue.hasNext()) {
-      dictionarySortModels[surrogate - 1] =
-          createDictionarySortModel(surrogate, dataType, distinctValue.next().getBytes());
+      dictionarySortModels[surrogate - 1] = createDictionarySortModel(surrogate, dataType,
+          distinctValue.next().getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
       surrogate++;
     }
     return dictionarySortModels;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
index 631bc2c..1cdbb26 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputSplit.java
@@ -20,6 +20,7 @@ import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 import java.io.Serializable;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -261,8 +262,10 @@ public class CarbonInputSplit extends FileSplit
     String filePath1 = this.getPath().getName();
     String filePath2 = other.getPath().getName();
     if (CarbonTablePath.isCarbonDataFile(filePath1)) {
-      byte[] firstTaskId = CarbonTablePath.DataFileUtil.getTaskNo(filePath1).getBytes();
-      byte[] otherTaskId = CarbonTablePath.DataFileUtil.getTaskNo(filePath2).getBytes();
+      byte[] firstTaskId = CarbonTablePath.DataFileUtil.getTaskNo(filePath1)
+          .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+      byte[] otherTaskId = CarbonTablePath.DataFileUtil.getTaskNo(filePath2)
+          .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
       int compare = ByteUtil.compare(firstTaskId, otherTaskId);
       if (compare != 0) {
         return compare;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
index ed98871..4268ee2 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/internal/index/impl/InMemoryBTreeIndex.java
@@ -79,7 +79,7 @@ class InMemoryBTreeIndex implements Index {
 
     FilterExpressionProcessor filterExpressionProcessor = new FilterExpressionProcessor();
 
-    AbsoluteTableIdentifier identifier = null;
+    AbsoluteTableIdentifier identifier = AbsoluteTableIdentifier.from(segment.getPath(),
"", "");
 
     //for this segment fetch blocks matching filter in BTree
     List<DataRefNode> dataRefNodes =

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
index fb01b27..65f2ba2 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CarbonScalaUtil.scala
@@ -17,6 +17,7 @@
 
 package org.apache.carbondata.spark.util
 
+import java.nio.charset.Charset
 import java.text.SimpleDateFormat
 
 import org.apache.spark.sql._
@@ -109,7 +110,8 @@ object CarbonScalaUtil {
         case b: java.lang.Boolean => b.toString
         case s: java.lang.Short => s.toString
         case f: java.lang.Float => f.toString
-        case bs: Array[Byte] => new String(bs)
+        case bs: Array[Byte] => new String(bs,
+          Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET))
         case s: scala.collection.Seq[Any] =>
           val delimiter = if (level == 1) {
             delimiterLevel1

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala
index a57ab10..bc62902 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/util/TableAPIUtil.scala
@@ -33,7 +33,7 @@ object TableAPIUtil {
 
   def parseSchemaName(tableName: String): (String, String) = {
     if (tableName.contains(".")) {
-      val parts = tableName.split(".")
+      val parts = tableName.split("\\.")
       (parts(0), parts(1))
     } else {
       ("default", tableName)

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVInputFormat.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVInputFormat.java
b/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVInputFormat.java
index e252e7f..7ff8d5f 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVInputFormat.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/csvload/CSVInputFormat.java
@@ -20,9 +20,11 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.Reader;
+import java.nio.charset.Charset;
 
 import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
 
 import com.univocity.parsers.csv.CsvParser;
 import com.univocity.parsers.csv.CsvParserSettings;
@@ -225,7 +227,8 @@ public class CSVInputFormat extends FileInputFormat<NullWritable, StringArrayWri
         filePosition = fileIn;
         inputStream = boundedInputStream;
       }
-      reader = new InputStreamReader(inputStream);
+      reader = new InputStreamReader(inputStream,
+          Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
       csvParser = new CsvParser(extractCsvParserSettings(job));
       csvParser.beginParsing(reader);
     }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/processing/src/main/java/org/apache/carbondata/processing/newflow/AbstractDataLoadProcessorStep.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/AbstractDataLoadProcessorStep.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/AbstractDataLoadProcessorStep.java
index a5887e6..9e0aa02 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/AbstractDataLoadProcessorStep.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/AbstractDataLoadProcessorStep.java
@@ -56,7 +56,20 @@ public abstract class AbstractDataLoadProcessorStep {
     this.child = child;
     this.rowCounter = new AtomicLong();
     this.closed = false;
+  }
+
+  /**
+   * The output meta for this step. The data returns from this step is as per this meta.
+   *
+   */
+  public abstract DataField[] getOutput();
 
+  /**
+   * Initialization process for this step.
+   *
+   * @throws IOException
+   */
+  public void initialize() throws IOException {
     if (LOGGER.isInfoEnabled()) {
       // This thread prints the rows processed in each step for every 10 seconds.
       new Thread() {
@@ -76,19 +89,6 @@ public abstract class AbstractDataLoadProcessorStep {
   }
 
   /**
-   * The output meta for this step. The data returns from this step is as per this meta.
-   *
-   */
-  public abstract DataField[] getOutput();
-
-  /**
-   * Initialization process for this step.
-   *
-   * @throws IOException
-   */
-  public abstract void initialize() throws IOException;
-
-  /**
    * Tranform the data as per the implementation.
    *
    * @return Array of Iterator with data. It can be processed parallel if implementation
class wants

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
index 71e5727..4ed588d 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/CarbonRowDataWriterProcessorStepImpl.java
@@ -84,6 +84,7 @@ public class CarbonRowDataWriterProcessorStepImpl extends AbstractDataLoadProces
   }
 
   @Override public void initialize() throws IOException {
+    super.initialize();
     child.initialize();
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
index 62d6c94..c46ea25 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorStepImpl.java
@@ -61,6 +61,7 @@ public class DataConverterProcessorStepImpl extends AbstractDataLoadProcessorSte
 
   @Override
   public void initialize() throws IOException {
+    super.initialize();
     child.initialize();
     converters = new ArrayList<>();
     badRecordLogger = createBadRecordLogger(configuration);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
index c6f83ed..fe36f82 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataConverterProcessorWithBucketingStepImpl.java
@@ -68,6 +68,7 @@ public class DataConverterProcessorWithBucketingStepImpl extends AbstractDataLoa
 
   @Override
   public void initialize() throws IOException {
+    super.initialize();
     child.initialize();
     converters = new ArrayList<>();
     badRecordLogger = createBadRecordLogger();

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterBatchProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterBatchProcessorStepImpl.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterBatchProcessorStepImpl.java
index 46c1020..00cb302 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterBatchProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterBatchProcessorStepImpl.java
@@ -55,6 +55,7 @@ public class DataWriterBatchProcessorStepImpl extends AbstractDataLoadProcessorS
   }
 
   @Override public void initialize() throws IOException {
+    super.initialize();
     child.initialize();
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
index 087b0c7..afb3062 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/DataWriterProcessorStepImpl.java
@@ -62,6 +62,7 @@ public class DataWriterProcessorStepImpl extends AbstractDataLoadProcessorStep
{
   }
 
   @Override public void initialize() throws IOException {
+    super.initialize();
     child.initialize();
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/InputProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/InputProcessorStepImpl.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/InputProcessorStepImpl.java
index 4a78e8f..cbeb20a 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/InputProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/InputProcessorStepImpl.java
@@ -16,6 +16,7 @@
  */
 package org.apache.carbondata.processing.newflow.steps;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
@@ -32,7 +33,6 @@ import org.apache.carbondata.core.util.CarbonProperties;
 import org.apache.carbondata.processing.newflow.AbstractDataLoadProcessorStep;
 import org.apache.carbondata.processing.newflow.CarbonDataLoadConfiguration;
 import org.apache.carbondata.processing.newflow.DataField;
-import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException;
 import org.apache.carbondata.processing.newflow.parser.RowParser;
 import org.apache.carbondata.processing.newflow.parser.impl.RowParserImpl;
 import org.apache.carbondata.processing.newflow.row.CarbonRowBatch;
@@ -61,7 +61,8 @@ public class InputProcessorStepImpl extends AbstractDataLoadProcessorStep
{
     return configuration.getDataFields();
   }
 
-  @Override public void initialize() throws CarbonDataLoadingException {
+  @Override public void initialize() throws IOException {
+    super.initialize();
     rowParser = new RowParserImpl(getOutput(), configuration);
     executorService = Executors.newCachedThreadPool();
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
index 987cb15..0b93b7a 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/newflow/steps/SortProcessorStepImpl.java
@@ -49,6 +49,7 @@ public class SortProcessorStepImpl extends AbstractDataLoadProcessorStep
{
 
   @Override
   public void initialize() throws IOException {
+    super.initialize();
     child.initialize();
     SortParameters sortParameters = SortParameters.createSortParameters(configuration);
     sorter = SorterFactory.createSorter(configuration, rowCounter);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c9866e8/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
index 82e83d5..c835332 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v2/CarbonFactDataWriterImplV2.java
@@ -19,6 +19,7 @@ package org.apache.carbondata.processing.store.writer.v2;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.channels.FileChannel;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -107,7 +108,8 @@ public class CarbonFactDataWriterImplV2 extends CarbonFactDataWriterImplV1
{
     try {
       if (fileChannel.size() == 0) {
         ColumnarFormatVersion version = CarbonProperties.getInstance().getFormatVersion();
-        byte[] header = (CarbonCommonConstants.CARBON_DATA_VERSION_HEADER + version).getBytes();
+        byte[] header = (CarbonCommonConstants.CARBON_DATA_VERSION_HEADER + version).getBytes(
+            Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
         ByteBuffer buffer = ByteBuffer.allocate(header.length);
         buffer.put(header);
         buffer.rewind();


Mime
View raw message