carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject carbondata git commit: [CARBONDATA-2113]compatibility fix for v2
Date Thu, 01 Feb 2018 16:46:51 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 1248bd4b7 -> 1202e209e


[CARBONDATA-2113]compatibility fix for v2

Fixes related to backword compatibility:

Count() issue:
when count() is run on old store where file format version is V2, it was unable to get the
files, because when forming the filepath while creating the table block info object from index
file info, the path was formed with double slash(//), beacuse in v2, local path was stored.
so when trying to get this path with key, it was failing. So form the correct file path.

Select * issue:
when select * is run, then only the datachunck was considered as whole data and while uncompressing
the measure data, it was failing. So, read proper data and datachunck before uncompressing.

Read Metadata file:
when readMetadataFile is called , the while reading the schema, it was explicitly returning
null, so columns will be null and hence some nullpointerxception was coming. so do not return
null, return proper schema, by reading the footer.

Vesion compatibility:
calculation the number of pages to be filled based on row count was not handled for V2 version,
handled same, as no. of rows per page is different in V2 and V3

This closes #1901


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/1202e209
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/1202e209
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/1202e209

Branch: refs/heads/master
Commit: 1202e209eccda172f9671fa0cc2deeebeb4af456
Parents: 1248bd4
Author: akashrn5 <akashnilugal@gmail.com>
Authored: Thu Feb 1 12:42:49 2018 +0530
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Thu Feb 1 22:16:39 2018 +0530

----------------------------------------------------------------------
 .../core/constants/CarbonVersionConstants.java       |  5 +++++
 .../v2/CompressedMeasureChunkFileBasedReaderV2.java  | 15 ++++++++++-----
 .../blockletindex/BlockletDataRefNodeWrapper.java    | 13 +++++++++++--
 .../carbondata/core/mutate/CarbonUpdateUtil.java     | 10 ++++------
 .../core/util/AbstractDataFileFooterConverter.java   |  3 ++-
 .../core/util/DataFileFooterConverter2.java          |  2 +-
 6 files changed, 33 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/1202e209/core/src/main/java/org/apache/carbondata/core/constants/CarbonVersionConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonVersionConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonVersionConstants.java
index 2d58b0b..22fbaf2 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonVersionConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonVersionConstants.java
@@ -49,6 +49,11 @@ public final class CarbonVersionConstants {
    */
   public static final String CARBONDATA_BUILD_DATE;
 
+  /**
+   * number of rows per blocklet column page default value for V2 version
+   */
+  public static final int NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT_V2 = 120000;
+
   static {
     // create input stream for CARBONDATA_VERSION_INFO_FILE
     InputStream resourceStream = Thread.currentThread().getContextClassLoader()

http://git-wip-us.apache.org/repos/asf/carbondata/blob/1202e209/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
index 2ddc202..d61f98a 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/chunk/reader/measure/v2/CompressedMeasureChunkFileBasedReaderV2.java
@@ -52,7 +52,14 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
       throws IOException {
     int dataLength = 0;
     if (measureColumnChunkOffsets.size() - 1 == columnIndex) {
-      dataLength = measureColumnChunkLength.get(columnIndex);
+      DataChunk2 metadataChunk = null;
+      synchronized (fileReader) {
+        metadataChunk = CarbonUtil.readDataChunk(ByteBuffer.wrap(fileReader
+                .readByteArray(filePath, measureColumnChunkOffsets.get(columnIndex),
+                    measureColumnChunkLength.get(columnIndex))), 0,
+            measureColumnChunkLength.get(columnIndex));
+      }
+      dataLength = measureColumnChunkLength.get(columnIndex) + metadataChunk.data_page_length;
     } else {
       long currentMeasureOffset = measureColumnChunkOffsets.get(columnIndex);
       dataLength = (int) (measureColumnChunkOffsets.get(columnIndex + 1) - currentMeasureOffset);
@@ -115,9 +122,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
     ByteBuffer rawData = measureRawColumnChunk.getRawData();
     DataChunk2 measureColumnChunk = CarbonUtil.readDataChunk(rawData, copyPoint,
         measureColumnChunkLength.get(blockIndex));
-    if (measureColumnChunkOffsets.size() - 1 != blockIndex) {
-      copyPoint += measureColumnChunkLength.get(blockIndex);
-    }
+    copyPoint += measureColumnChunkLength.get(blockIndex);
 
     ColumnPage page = decodeMeasure(measureRawColumnChunk, measureColumnChunk, copyPoint);
     page.setNullBits(getNullBitSet(measureColumnChunk.presence));
@@ -130,7 +135,7 @@ public class CompressedMeasureChunkFileBasedReaderV2 extends AbstractMeasureChun
     List<ByteBuffer> encoder_meta = measureColumnChunk.getEncoder_meta();
     byte[] encodedMeta = encoder_meta.get(0).array();
 
-    ValueEncoderMeta meta = CarbonUtil.deserializeEncoderMetaV3(encodedMeta);
+    ValueEncoderMeta meta = CarbonUtil.deserializeEncoderMetaV2(encodedMeta);
     ColumnPageDecoder codec = encodingFactory.createDecoderLegacy(meta);
     byte[] rawData = measureRawColumnChunk.getRawData().array();
     return codec.decode(rawData, copyPoint, measureColumnChunk.data_page_length);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/1202e209/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
index b672c58..4e49ede 100644
--- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
+++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataRefNodeWrapper.java
@@ -21,6 +21,7 @@ import java.util.List;
 
 import org.apache.carbondata.core.cache.update.BlockletLevelDeleteDeltaDataCache;
 import org.apache.carbondata.core.constants.CarbonV3DataFormatConstants;
+import org.apache.carbondata.core.constants.CarbonVersionConstants;
 import org.apache.carbondata.core.datastore.DataRefNode;
 import org.apache.carbondata.core.datastore.FileHolder;
 import org.apache.carbondata.core.datastore.block.TableBlockInfo;
@@ -56,8 +57,16 @@ public class BlockletDataRefNodeWrapper implements DataRefNode {
       detailInfo.getBlockletInfo().setNumberOfPages(detailInfo.getPagesCount());
       detailInfo.setBlockletId(blockInfo.getDetailInfo().getBlockletId());
       int[] pageRowCount = new int[detailInfo.getPagesCount()];
-      int numberOfPagesCompletelyFilled = detailInfo.getRowCount()
-          / CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
+      int numberOfPagesCompletelyFilled = detailInfo.getRowCount();
+      // no. of rows to a page is 120000 in V2 and 32000 in V3, same is handled to get the
number
+      // of pages filled
+      if (blockInfo.getVersion() == ColumnarFormatVersion.V2) {
+        numberOfPagesCompletelyFilled /=
+            CarbonVersionConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT_V2;
+      } else {
+        numberOfPagesCompletelyFilled /=
+            CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
+      }
       int lastPageRowCount = detailInfo.getRowCount()
           % CarbonV3DataFormatConstants.NUMBER_OF_ROWS_PER_BLOCKLET_COLUMN_PAGE_DEFAULT;
       for (int i = 0; i < numberOfPagesCompletelyFilled; i++) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/1202e209/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
index 0e4eec7..c5f61c2 100644
--- a/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/mutate/CarbonUpdateUtil.java
@@ -587,26 +587,24 @@ public class CarbonUpdateUtil {
    */
   private static void deleteStaleCarbonDataFiles(LoadMetadataDetails segment,
       CarbonFile[] allSegmentFiles, SegmentUpdateStatusManager updateStatusManager) {
-    boolean doForceDelete = true;
-    boolean isAbortedFile = true;
     CarbonFile[] invalidUpdateDeltaFiles = updateStatusManager
         .getUpdateDeltaFilesList(segment.getLoadName(), false,
             CarbonCommonConstants.UPDATE_DELTA_FILE_EXT, true, allSegmentFiles,
-            isAbortedFile);
+            true);
     // now for each invalid delta file need to check the query execution time out
     // and then delete.
     for (CarbonFile invalidFile : invalidUpdateDeltaFiles) {
-      compareTimestampsAndDelete(invalidFile, doForceDelete, false);
+      compareTimestampsAndDelete(invalidFile, true, false);
     }
     // do the same for the index files.
     CarbonFile[] invalidIndexFiles = updateStatusManager
         .getUpdateDeltaFilesList(segment.getLoadName(), false,
             CarbonCommonConstants.UPDATE_INDEX_FILE_EXT, true, allSegmentFiles,
-            isAbortedFile);
+            true);
     // now for each invalid index file need to check the query execution time out
     // and then delete.
     for (CarbonFile invalidFile : invalidIndexFiles) {
-      compareTimestampsAndDelete(invalidFile, doForceDelete, false);
+      compareTimestampsAndDelete(invalidFile, true, false);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/1202e209/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
index c7bc6aa..94a041a 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/AbstractDataFileFooterConverter.java
@@ -208,7 +208,8 @@ public abstract class AbstractDataFileFooterConverter {
     if (fileName.lastIndexOf("/") > 0) {
       fileName = fileName.substring(fileName.lastIndexOf("/"));
     }
-    tableBlockInfo.setFilePath(parentPath + "/" + fileName);
+    fileName = (CarbonCommonConstants.FILE_SEPARATOR + fileName).replaceAll("//", "/");
+    tableBlockInfo.setFilePath(parentPath + fileName);
     return tableBlockInfo;
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/1202e209/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
index 8cd437f..863e1df 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataFileFooterConverter2.java
@@ -141,6 +141,6 @@ public class DataFileFooterConverter2 extends AbstractDataFileFooterConverter
{
   }
 
   @Override public List<ColumnSchema> getSchema(TableBlockInfo tableBlockInfo) throws
IOException {
-    return null;
+    return new DataFileFooterConverter().getSchema(tableBlockInfo);
   }
 }


Mime
View raw message