carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [18/47] incubator-carbondata git commit: [CARBONDATA-105][Bug] In case of concurrent data loading, pre calculation of existence of dictionary file will not have proper result. (#866)
Date Mon, 01 Aug 2016 10:05:16 GMT
[CARBONDATA-105][Bug] In case of concurrent data loading, pre calculation of existence of dictionary
file will not have proper result. (#866)

Check added after acquiring lock

Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/8289525d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/8289525d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/8289525d

Branch: refs/heads/master
Commit: 8289525d286d2d4e135c4405097a45d105b48a1a
Parents: f57a95c
Author: ashokblend <ashok.blend@gmail.com>
Authored: Mon Jul 25 14:42:10 2016 +0530
Committer: Venkata Ramana G <g.ramana.v1@gmail.com>
Committed: Mon Jul 25 14:42:10 2016 +0530

----------------------------------------------------------------------
 .../org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala      | 3 +++
 .../spark/util/GlobalDictionaryUtilConcurrentTestCase.scala       | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8289525d/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
index 7067fd3..9a94c5b 100644
--- a/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
+++ b/integration/spark/src/main/scala/org/carbondata/spark/rdd/CarbonGlobalDictionaryRDD.scala
@@ -342,6 +342,9 @@ class CarbonGlobalDictionaryGenerateRDD(
               } is locked for updation. Please try after some time")
           }
           val t2 = System.currentTimeMillis
+          val fileType = FileFactory.getFileType(model.dictFilePaths(split.index))
+          model.dictFileExists(split.index) = FileFactory
+            .isFileExist(model.dictFilePaths(split.index), fileType)
           dictionaryForDistinctValueLookUp = if (model.dictFileExists(split.index)) {
             CarbonLoaderUtil.getDictionary(model.table,
               model.columnIdentifier(split.index),

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/8289525d/integration/spark/src/test/scala/org/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
b/integration/spark/src/test/scala/org/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
index 60e9281..9230c6d 100644
--- a/integration/spark/src/test/scala/org/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
+++ b/integration/spark/src/test/scala/org/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala
@@ -143,7 +143,7 @@ class GlobalDictionaryUtilConcurrentTestCase extends QueryTest with BeforeAndAft
     val sortIndexFile = FileFactory.getCarbonFile(sortIndexPath, FileFactory.getFileType(sortIndexPath))
     assert(sortIndexFile.exists())
     val sortIndexFiles = carbonTablePath.getSortIndexFiles(sortIndexFile.getParentFile, columnIdentifier.getColumnId)
-    assert(sortIndexFiles.length == 2)
+    assert(sortIndexFiles.length >= 1)
     deleteFiles(files)
   }
 


Mime
View raw message