carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From akash...@apache.org
Subject [carbondata] branch master updated: [CARBONDATA-3766] Fixed desc formatted and show segment data size issues
Date Tue, 07 Apr 2020 08:22:14 GMT
This is an automated email from the ASF dual-hosted git repository.

akashrn5 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/carbondata.git


The following commit(s) were added to refs/heads/master by this push:
     new e643747  [CARBONDATA-3766] Fixed desc formatted and show segment data size issues
e643747 is described below

commit e6437479fa11eef9a8fc03c2572905bee458c5ad
Author: kunal642 <kunalkapoor642@gmail.com>
AuthorDate: Thu Mar 26 14:55:34 2020 +0530

    [CARBONDATA-3766] Fixed desc formatted and show segment data size issues
    
    Why is this PR needed?
    Show segments shows the datasize as index size for non carbon segments in show segments
command.
    
    What changes were proposed in this PR?
    Take data size from indexsize variable and show in show segments as non carbon segmnets
do not have
    index files. Index Size wil be shown as 0.
    
    This closes #3680
---
 .../carbondata/core/metadata/SegmentFileStore.java       |  3 ++-
 .../java/org/apache/carbondata/core/util/CarbonUtil.java | 16 ++++++++++++----
 .../scala/org/apache/carbondata/api/CarbonStore.scala    |  2 +-
 .../spark/testsuite/addsegment/AddSegmentTestCase.scala  |  2 ++
 4 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
index afb76bd..7aa2843 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java
@@ -527,10 +527,11 @@ public class SegmentFileStore {
         for (LoadMetadataDetails detail : listOfLoadFolderDetailsArray) {
           // if the segments is in the list of marked for delete then update the status.
           if (segmentId.equals(detail.getLoadName())) {
+            detail.setLoadEndTime(System.currentTimeMillis());
             detail.setSegmentFile(segmentFile);
             if (segmentStatus != null) {
               HashMap<String, Long> dataSizeAndIndexSize =
-                  CarbonUtil.getDataSizeAndIndexSize(segmentFileStore);
+                  CarbonUtil.getDataSizeAndIndexSize(segmentFileStore, detail.isCarbonFormat());
               detail.setDataSize(
                   dataSizeAndIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE)
                       .toString());
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index d00da15..7917ddd 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -2514,7 +2514,8 @@ public final class CarbonUtil {
   }
 
   // Get the total size of carbon data and the total size of carbon index
-  public static HashMap<String, Long> getDataSizeAndIndexSize(SegmentFileStore fileStore)
+  public static HashMap<String, Long> getDataSizeAndIndexSize(SegmentFileStore fileStore,
+      boolean isCarbonSegment)
       throws IOException {
     long carbonDataSize = 0L;
     long carbonIndexSize = 0L;
@@ -2532,8 +2533,15 @@ public final class CarbonUtil {
         }
       }
     }
-    dataAndIndexSize.put(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE, carbonDataSize);
-    dataAndIndexSize.put(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE, carbonIndexSize);
+    if (isCarbonSegment) {
+      dataAndIndexSize.put(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE, carbonDataSize);
+      dataAndIndexSize.put(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE, carbonIndexSize);
+    } else {
+      // In case of non-carbon segments all the dataSize is collected in carbonIndexSize
because of
+      // listing behaviour of the SegmentFileStore.
+      dataAndIndexSize.put(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE, carbonIndexSize);
+      dataAndIndexSize.put(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE, 0L);
+    }
     return dataAndIndexSize;
   }
 
@@ -2582,7 +2590,7 @@ public final class CarbonUtil {
       Segment segment) throws IOException {
     if (segment.getSegmentFileName() != null) {
       SegmentFileStore fileStore = new SegmentFileStore(tablePath, segment.getSegmentFileName());
-      return getDataSizeAndIndexSize(fileStore);
+      return getDataSizeAndIndexSize(fileStore, segment.isCarbonSegment());
     } else {
       return getDataSizeAndIndexSize(tablePath, segment.getSegmentNo());
     }
diff --git a/integration/spark/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
b/integration/spark/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
index 5571656..275d260 100644
--- a/integration/spark/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
+++ b/integration/spark/src/main/scala/org/apache/carbondata/api/CarbonStore.scala
@@ -123,7 +123,7 @@ object CarbonStore {
             // If the added segment is other than carbon segment then we can only display
the data
             // size and not index size, we can get the data size from table status file directly
             if (!load.getFileFormat.isCarbonFormat) {
-              (if (load.getIndexSize == null) -1L else load.getIndexSize.toLong, -1L)
+              (if (load.getDataSize == null) -1L else load.getDataSize.toLong, -1L)
             } else {
               (if (load.getDataSize == null) -1L else load.getDataSize.toLong,
                 if (load.getIndexSize == null) -1L else load.getIndexSize.toLong)
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
index 13472cc..6c41abd 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/addsegment/AddSegmentTestCase.scala
@@ -230,7 +230,9 @@ class AddSegmentTestCase extends QueryTest with BeforeAndAfterAll {
     checkAnswer(sql("select count(*) from addsegment1"), Seq(Row(20)))
     sql("show segments for table addsegment1").show(100, false)
     val showSeg = sql("show segments for table addsegment1").collectAsList()
+    val descFormattedSize = sql("desc formatted addsegment1").collect().filter(_.get(0).toString.startsWith("Table
Data Size")).head.get(1).toString
     val size = getDataSize(newPath)
+    assert(descFormattedSize.split("KB")(0).toDouble > 0.0d)
     assert(showSeg.get(0).get(6).toString.equalsIgnoreCase(size))
     assert(showSeg.get(0).get(7).toString.equalsIgnoreCase("NA"))
     FileFactory.deleteAllFilesOfDir(new File(newPath))


Mime
View raw message