carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [1/2] incubator-carbondata git commit: fix load bug when table name has '_'
Date Wed, 21 Dec 2016 08:45:47 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master a561d869c -> ddeb00425


fix load bug when table name has '_'


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/fc1d620e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/fc1d620e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/fc1d620e

Branch: refs/heads/master
Commit: fc1d620e729746271c70159636e0cc05ac46d99e
Parents: a561d86
Author: Jay357089 <liujunjie9@huawei.com>
Authored: Wed Nov 16 10:49:35 2016 +0800
Committer: jackylk <jacky.likun@huawei.com>
Committed: Wed Dec 21 16:43:20 2016 +0800

----------------------------------------------------------------------
 .../core/carbon/path/CarbonTablePath.java       |  8 ++--
 .../core/constants/CarbonCommonConstants.java   |  4 ++
 .../dataload/TestLoadDataGeneral.scala          | 42 +++++++++++++++++++-
 3 files changed, 50 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fc1d620e/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
b/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
index 99531e4..f90073e 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
@@ -397,10 +397,12 @@ public class CarbonTablePath extends Path {
      */
     public static String getSegmentId(String dataFileAbsolutePath) {
       // find segment id from last of data file path
-      int endIndex = dataFileAbsolutePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR);
+      String tempdataFileAbsolutePath = dataFileAbsolutePath.replace(
+              CarbonCommonConstants.WINDOWS_FILE_SEPARATOR, CarbonCommonConstants.FILE_SEPARATOR);
+      int endIndex = tempdataFileAbsolutePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR);
       // + 1 for size of "/"
-      int startIndex =
-          dataFileAbsolutePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR, endIndex
- 1) + 1;
+      int startIndex = tempdataFileAbsolutePath.lastIndexOf(
+              CarbonCommonConstants.FILE_SEPARATOR, endIndex - 1) + 1;
       String segmentDirStr = dataFileAbsolutePath.substring(startIndex, endIndex);
       //identify id in segment_<id>
       String[] segmentDirSplits = segmentDirStr.split("_");

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fc1d620e/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index c32956c..2d1ba9f 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -461,6 +461,10 @@ public final class CarbonCommonConstants {
    */
   public static final String POINT = ".";
   /**
+   * Windows File separator
+   */
+  public static final String WINDOWS_FILE_SEPARATOR = "\\";
+  /**
    * File separator
    */
   public static final String FILE_SEPARATOR = "/";

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/fc1d620e/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
index 9904c93..6a274b6 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataGeneral.scala
@@ -22,11 +22,16 @@ package org.apache.carbondata.integration.spark.testsuite.dataload
 import java.io.File
 import java.math.BigDecimal
 
+import org.apache.carbondata.core.carbon.path.{CarbonStorePath, CarbonTablePath}
+import org.apache.carbondata.core.datastorage.store.filesystem.CarbonFile
+import org.apache.carbondata.core.datastorage.store.impl.FileFactory
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.common.util.CarbonHiveContext._
-import org.apache.spark.sql.common.util.QueryTest
+import org.apache.spark.sql.common.util.{CarbonHiveContext, QueryTest}
 import org.scalatest.BeforeAndAfterAll
 
+import scala.collection.mutable.ArrayBuffer
+
 class TestLoadDataGeneral extends QueryTest with BeforeAndAfterAll {
 
   var currentDirectory: String = _
@@ -43,6 +48,24 @@ class TestLoadDataGeneral extends QueryTest with BeforeAndAfterAll {
         .getCanonicalPath
   }
 
+  private def checkSegmentExists(
+      segmentId: String,
+      datbaseName: String,
+      tableName: String): Boolean = {
+    val carbonTable = org.apache.carbondata.core.carbon.metadata.CarbonMetadata.getInstance()
+      .getCarbonTable(datbaseName + "_" + tableName)
+    val partitionPath = CarbonStorePath.getCarbonTablePath(
+      CarbonHiveContext.hdfsCarbonBasePath,
+      carbonTable.getCarbonTableIdentifier).getPartitionDir("0")
+    val fileType: FileFactory.FileType = FileFactory.getFileType(partitionPath)
+    val carbonFile = FileFactory.getCarbonFile(partitionPath, fileType)
+    val segments: ArrayBuffer[String] = ArrayBuffer()
+    carbonFile.listFiles.foreach { file =>
+      segments += CarbonTablePath.DataPathUtil.getSegmentId(file.getAbsolutePath + "/dummy")
+    }
+    segments.contains(segmentId)
+  }
+
   test("test data loading CSV file") {
     val testData = currentDirectory + "/src/test/resources/sample.csv"
     sql(s"LOAD DATA LOCAL INPATH '$testData' into table loadtest")
@@ -99,6 +122,23 @@ class TestLoadDataGeneral extends QueryTest with BeforeAndAfterAll {
     )
   }
 
+  test("test data loading into table whose name has '_'") {
+    sql("DROP TABLE IF EXISTS load_test")
+    sql(""" CREATE TABLE load_test(id int, name string, city string, age int)
+        STORED BY 'org.apache.carbondata.format' """)
+    val testData = currentDirectory + "/src/test/resources/sample.csv"
+    try {
+      sql(s"LOAD DATA LOCAL INPATH '$testData' into table load_test")
+      sql(s"LOAD DATA LOCAL INPATH '$testData' into table load_test")
+    } catch {
+      case ex: Exception =>
+        assert(false)
+    }
+    assert(checkSegmentExists("0", "default", "load_test"))
+    assert(checkSegmentExists("1", "default", "load_test"))
+    sql("DROP TABLE load_test")
+  }
+
   override def afterAll {
     sql("DROP TABLE loadtest")
   }


Mime
View raw message