carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject carbondata git commit: [CARBONDATA-2455]Fix _System Folder creation and lucene AND, OR, NOT Filter fix
Date Fri, 11 May 2018 01:29:39 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 6b70b7e47 -> 61afa42da


[CARBONDATA-2455]Fix _System Folder creation and lucene AND,OR,NOT Filter fix

This closes #2281


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/61afa42d
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/61afa42d
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/61afa42d

Branch: refs/heads/master
Commit: 61afa42daafa03a7e8e0eb980cb6d667d6304785
Parents: 6b70b7e
Author: Indhumathi27 <indhumathim27@gmail.com>
Authored: Tue May 8 15:36:47 2018 +0530
Committer: Jacky Li <jacky.likun@qq.com>
Committed: Fri May 11 09:29:19 2018 +0800

----------------------------------------------------------------------
 .../schema/table/DiskBasedDMSchemaStorageProvider.java   |  2 +-
 .../carbondata/datamap/lucene/LuceneDataMapWriter.java   |  1 -
 .../datamap/lucene/LuceneFineGrainDataMap.java           |  3 +--
 .../cluster/sdv/generated/LuceneTestCase.scala           |  7 +++----
 .../datamap/lucene/LuceneFineGrainDataMapSuite.scala     | 11 +++++------
 5 files changed, 10 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/61afa42d/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStorageProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStorageProvider.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStorageProvider.java
index e1a929c..4ebbcd0 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStorageProvider.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/DiskBasedDMSchemaStorageProvider.java
@@ -55,7 +55,7 @@ public class DiskBasedDMSchemaStorageProvider implements DataMapSchemaStoragePro
   private Set<DataMapSchema> dataMapSchemas = new HashSet<>();
 
   public DiskBasedDMSchemaStorageProvider(String storePath) {
-    this.storePath = storePath;
+    this.storePath = CarbonUtil.checkAndAppendHDFSUrl(storePath);
     this.mdtFilePath = storePath + CarbonCommonConstants.FILE_SEPARATOR + "datamap.mdtfile";
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/61afa42d/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapWriter.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapWriter.java
b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapWriter.java
index 12fa1ef..3615936 100644
--- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapWriter.java
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneDataMapWriter.java
@@ -274,7 +274,6 @@ public class LuceneDataMapWriter extends DataMapWriter {
       }
     } else if (type == DataTypes.STRING) {
       byte[] value = (byte[]) data;
-      // TODO: how to get string value
       String strValue = null;
       try {
         strValue = new String(value, 2, value.length - 2, "UTF-8");

http://git-wip-us.apache.org/repos/asf/carbondata/blob/61afa42d/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java
----------------------------------------------------------------------
diff --git a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java
b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java
index 8e457b7..f8d1b12 100644
--- a/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java
+++ b/datamap/lucene/src/main/java/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMap.java
@@ -205,8 +205,7 @@ public class LuceneFineGrainDataMap extends FineGrainDataMap {
     queryParser.setAllowLeadingWildcard(true);
     Query query;
     try {
-      // always send lowercase string to lucene as it is case sensitive
-      query = queryParser.parse(strQuery.toLowerCase());
+      query = queryParser.parse(strQuery);
     } catch (ParseException e) {
       String errorMessage = String.format(
           "failed to filter block with query %s, detail is %s", strQuery, e.getMessage());

http://git-wip-us.apache.org/repos/asf/carbondata/blob/61afa42d/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/LuceneTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/LuceneTestCase.scala
b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/LuceneTestCase.scala
index d111594..54d0d11 100644
--- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/LuceneTestCase.scala
+++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/LuceneTestCase.scala
@@ -110,14 +110,13 @@ class LuceneTestCase extends QueryTest with BeforeAndAfterAll {
     checkAnswer(sql("SELECT * FROM datamap_main WHERE TEXT_MATCH('country:ch*')"),
       sql("select * from datamap_main where country like 'ch%'"))
     checkAnswer(sql(
-      "SELECT * FROM datamap_main WHERE TEXT_MATCH('country:ch*') AND TEXT_MATCH('name:aa*')"),
+      "SELECT * FROM datamap_main WHERE TEXT_MATCH('country:ch* AND name:aa*')"),
       sql("select * from datamap_main where country like 'ch%' and name like 'aa%'"))
     checkAnswer(sql(
-      "SELECT * FROM datamap_main WHERE TEXT_MATCH('country:u* or name:aa*')"),
+      "SELECT * FROM datamap_main WHERE TEXT_MATCH('country:u* OR name:aa*')"),
       sql("select * from datamap_main where country like 'u%'or name like 'aa%'"))
     checkAnswer(sql(
-      "SELECT * FROM datamap_main WHERE TEXT_MATCH('country:u*') OR TEXT_MATCH('name:aaa1*')
AND " +
-      "TEXT_MATCH('name:aaa2*')"),
+      "SELECT * FROM datamap_main WHERE TEXT_MATCH('country:u* OR (name:aaa1* AND name:aaa2*)')"),
       sql(
         "select * from datamap_main where country like 'u%' OR name like 'aaa1%' AND name
like " +
         "'aaa2%'"))

http://git-wip-us.apache.org/repos/asf/carbondata/blob/61afa42d/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
index 9981ff5..6d2eb3f 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/datamap/lucene/LuceneFineGrainDataMapSuite.scala
@@ -290,7 +290,7 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll
{
       """.stripMargin)
     sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test_table OPTIONS('header'='false')")
     checkAnswer(sql(
-      "SELECT * FROM datamap_test_table WHERE TEXT_MATCH('name:n0*') AND TEXT_MATCH(' city:c0*')"),
+      "SELECT * FROM datamap_test_table WHERE TEXT_MATCH('name:n0* AND city:c0*')"),
       sql("select * from datamap_test_table where name like 'n0%' and city like 'c0%'"))
     sql("drop datamap if exists dm on table datamap_test_table")
   }
@@ -311,7 +311,7 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll
{
       """.stripMargin)
     sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test_table OPTIONS('header'='false')")
     checkAnswer(sql(
-      "SELECT * FROM datamap_test_table WHERE TEXT_MATCH('name:n1*') or TEXT_MATCH('city:c01*')"),
+      "SELECT * FROM datamap_test_table WHERE TEXT_MATCH('name:n1* OR city:c01*')"),
       sql("select * from datamap_test_table where name like 'n1%' or city like 'c01%'"))
     sql("drop datamap if exists dm on table datamap_test_table")
   }
@@ -332,8 +332,7 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll
{
       """.stripMargin)
     sql(s"LOAD DATA LOCAL INPATH '$file2' INTO TABLE datamap_test_table OPTIONS('header'='false')")
     checkAnswer(sql(
-      "SELECT * FROM datamap_test_table WHERE TEXT_MATCH('name:n1*') OR TEXT_MATCH ('city:c01*')
" +
-      "AND TEXT_MATCH('city:C02*')"),
+      "SELECT * FROM datamap_test_table WHERE TEXT_MATCH('name:n1* OR (city:c01* AND city:c02*)')"),
       sql(
         "select * from datamap_test_table where name like 'n1%' OR city like 'c01%' and city
like" +
         " 'c02%'"))
@@ -488,7 +487,7 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll
{
       sql("select *from datamap_test_table where name like'n1%' AND not name like 'n2%'"))
     //check NOT filter with TEXTMATCH wildcard-search using AND on different columns
     checkAnswer(sql(
-      "select *from datamap_test_table where TEXT_MATCH('name:n1*')AND TEXT_MATCH('city:c01*
NOT " +
+      "select *from datamap_test_table where TEXT_MATCH('name:n1* AND city:c01* NOT " +
       "c02*')"),
       sql("select *from datamap_test_table where name like'n1%' AND not city='c02%'"))
     sql("drop datamap if exists dm on table datamap_test_table")
@@ -716,7 +715,7 @@ class LuceneFineGrainDataMapSuite extends QueryTest with BeforeAndAfterAll
{
       sql(s"SELECT * FROM datamap_test5 WHERE city='c020'"))
     sql("DROP TABLE IF EXISTS datamap_test5")
   }
-
+  
   test("test text_match on normal table") {
     sql("DROP TABLE IF EXISTS table1")
     sql(


Mime
View raw message