carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From qiang...@apache.org
Subject carbondata git commit: [CARBONDATA-2220] Reduce unnecessary audit log
Date Fri, 23 Mar 2018 14:03:49 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master de52da40f -> 873c3ded0


[CARBONDATA-2220] Reduce unnecessary audit log

This closes #2020


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/873c3ded
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/873c3ded
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/873c3ded

Branch: refs/heads/master
Commit: 873c3ded04738fa9cbef9a5f0d37902fc758a88b
Parents: de52da4
Author: Jacky Li <jacky.likun@qq.com>
Authored: Fri Mar 2 15:40:27 2018 +0800
Committer: QiangCai <qiangcai@qq.com>
Committed: Fri Mar 23 22:02:42 2018 +0800

----------------------------------------------------------------------
 .../generator/TableDictionaryGenerator.java     |  2 +-
 .../server/NonSecureDictionaryServer.java       |  2 +-
 .../NonSecureDictionaryServerHandler.java       |  2 +-
 .../carbondata/core/scan/filter/FilterUtil.java |  4 +-
 .../core/util/CarbonLoadStatisticsImpl.java     | 50 ++++++++++----------
 .../carbondata/core/util/CarbonProperties.java  |  6 ++-
 .../carbondata/examples/CompareTest.scala       |  4 +-
 .../management/CarbonInsertIntoCommand.scala    |  2 +-
 .../CarbonAlterTableDropPartitionCommand.scala  |  1 -
 .../CarbonAlterTableSplitPartitionCommand.scala |  1 -
 .../spark/sql/hive/CarbonFileMetastore.scala    |  2 +-
 .../spark/sql/hive/CarbonHiveMetadataUtil.scala |  2 +-
 .../apache/spark/sql/hive/CarbonMetaStore.scala |  4 +-
 .../holder/UnsafeFinalMergePageHolder.java      |  2 +-
 .../unsafe/holder/UnsafeInmemoryHolder.java     |  2 +-
 .../holder/UnsafeInmemoryMergeHolder.java       |  2 +-
 .../holder/UnsafeSortTempFileChunkHolder.java   |  2 +-
 .../util/CarbonDataProcessorUtil.java           | 15 ++++--
 .../carbondata/streaming/StreamHandoffRDD.scala |  2 +-
 19 files changed, 55 insertions(+), 52 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
index 905d2fa..5db13b6 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/generator/TableDictionaryGenerator.java
@@ -84,7 +84,7 @@ public class TableDictionaryGenerator
     } catch (InterruptedException e) {
       LOGGER.error("Error loading the dictionary: " + e.getMessage());
     }
-    LOGGER.audit("Total time taken to write dictionary file is: " +
+    LOGGER.info("Total time taken to write dictionary file is: " +
             (System.currentTimeMillis() - start));
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServer.java
b/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServer.java
index daba470..95f3d69 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServer.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServer.java
@@ -113,7 +113,7 @@ public class NonSecureDictionaryServer extends AbstractDictionaryServer
             new InetSocketAddress(newPort) :
             new InetSocketAddress(hostToBind, newPort);
         bootstrap.bind(address).sync();
-        LOGGER.audit("Dictionary Server started, Time spent " + (System.currentTimeMillis()
- start)
+        LOGGER.info("Dictionary Server started, Time spent " + (System.currentTimeMillis()
- start)
             + " Listening on port " + newPort);
         this.port = newPort;
         this.host = hostToBind;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServerHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServerHandler.java
b/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServerHandler.java
index dc3d078..82efe80 100644
--- a/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServerHandler.java
+++ b/core/src/main/java/org/apache/carbondata/core/dictionary/server/NonSecureDictionaryServerHandler.java
@@ -48,7 +48,7 @@ import io.netty.channel.ChannelInboundHandlerAdapter;
    * @throws Exception
    */
   public void channelActive(ChannelHandlerContext ctx) throws Exception {
-    LOGGER.audit("Connected " + ctx);
+    LOGGER.info("Connected " + ctx);
     super.channelActive(ctx);
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index 0b72e77..a397355 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -777,7 +777,7 @@ public final class FilterUtil {
         columnFilterInfo.setFilterList(filterValuesList);
       }
     } catch (FilterIllegalMemberException e) {
-      LOGGER.audit(e.getMessage());
+      LOGGER.error(e.getMessage());
     }
     return columnFilterInfo;
   }
@@ -817,7 +817,7 @@ public final class FilterUtil {
         }
       }
     } catch (FilterIllegalMemberException e) {
-      LOGGER.audit(e.getMessage());
+      LOGGER.error(e.getMessage());
     }
 
     if (null == defaultValues) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/core/src/main/java/org/apache/carbondata/core/util/CarbonLoadStatisticsImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonLoadStatisticsImpl.java
b/core/src/main/java/org/apache/carbondata/core/util/CarbonLoadStatisticsImpl.java
index c9fc8ba..9c6ab95 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonLoadStatisticsImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonLoadStatisticsImpl.java
@@ -311,52 +311,52 @@ public class CarbonLoadStatisticsImpl implements LoadStatistics {
   //Print the statistics information
   private void printDicGenStatisticsInfo() {
     double loadCsvfilesToDfTime = getLoadCsvfilesToDfTime();
-    LOGGER.audit("STAGE 1 ->Load csv to DataFrame and generate" +
+    LOGGER.info("STAGE 1 ->Load csv to DataFrame and generate" +
             " block distinct values: " + loadCsvfilesToDfTime + "(s)");
     double dicShuffleAndWriteFileTotalTime = getDicShuffleAndWriteFileTotalTime();
-    LOGGER.audit("STAGE 2 ->Global dict shuffle and write dict file: " +
+    LOGGER.info("STAGE 2 ->Global dict shuffle and write dict file: " +
             + dicShuffleAndWriteFileTotalTime + "(s)");
   }
 
   private void printLruCacheLoadTimeInfo() {
-    LOGGER.audit("STAGE 3 ->LRU cache load: " + getLruCacheLoadTime() + "(s)");
+    LOGGER.info("STAGE 3 ->LRU cache load: " + getLruCacheLoadTime() + "(s)");
   }
 
   private void printDictionaryValuesGenStatisticsInfo(String partitionID) {
     double dictionaryValuesTotalTime = getDictionaryValuesTotalTime(partitionID);
-    LOGGER.audit("STAGE 4 ->Total cost of gen dictionary values, sort and write to temp
files: "
+    LOGGER.info("STAGE 4 ->Total cost of gen dictionary values, sort and write to temp
files: "
             + dictionaryValuesTotalTime + "(s)");
     double csvInputStepTime = getCsvInputStepTime(partitionID);
     double generatingDictionaryValuesTime = getGeneratingDictionaryValuesTime(partitionID);
-    LOGGER.audit("STAGE 4.1 ->  |_read csv file: " + csvInputStepTime + "(s)");
-    LOGGER.audit("STAGE 4.2 ->  |_transform to surrogate key: "
+    LOGGER.info("STAGE 4.1 ->  |_read csv file: " + csvInputStepTime + "(s)");
+    LOGGER.info("STAGE 4.2 ->  |_transform to surrogate key: "
             + generatingDictionaryValuesTime + "(s)");
   }
 
   private void printSortRowsStepStatisticsInfo(String partitionID) {
     double sortRowsStepTotalTime = getSortRowsStepTotalTime(partitionID);
-    LOGGER.audit("STAGE 4.3 ->  |_sort rows and write to temp file: "
+    LOGGER.info("STAGE 4.3 ->  |_sort rows and write to temp file: "
             + sortRowsStepTotalTime + "(s)");
   }
 
   private void printGenMdkStatisticsInfo(String partitionID) {
     double dictionaryValue2MdkAdd2FileTime = getDictionaryValue2MdkAdd2FileTime(partitionID);
-    LOGGER.audit("STAGE 5 ->Transform to MDK, compress and write fact files: "
+    LOGGER.info("STAGE 5 ->Transform to MDK, compress and write fact files: "
             + dictionaryValue2MdkAdd2FileTime + "(s)");
   }
 
   //Print the node blocks information
   private void printHostBlockMapInfo() {
-    LOGGER.audit("========== BLOCK_INFO ==========");
+    LOGGER.info("========== BLOCK_INFO ==========");
     if (getHostBlockMap().size() > 0) {
       for (String host: getHostBlockMap().keySet()) {
-        LOGGER.audit("BLOCK_INFO ->Node host: " + host);
-        LOGGER.audit("BLOCK_INFO ->The block count in this node: " + getHostBlockMap().get(host));
+        LOGGER.info("BLOCK_INFO ->Node host: " + host);
+        LOGGER.info("BLOCK_INFO ->The block count in this node: " + getHostBlockMap().get(host));
       }
     } else if (getPartitionBlockMap().size() > 0) {
       for (String parID: getPartitionBlockMap().keySet()) {
-        LOGGER.audit("BLOCK_INFO ->Partition ID: " + parID);
-        LOGGER.audit("BLOCK_INFO ->The block count in this partition: " +
+        LOGGER.info("BLOCK_INFO ->Partition ID: " + parID);
+        LOGGER.info("BLOCK_INFO ->The block count in this partition: " +
                 getPartitionBlockMap().get(parID));
       }
     }
@@ -364,21 +364,21 @@ public class CarbonLoadStatisticsImpl implements LoadStatistics {
 
   //Print the speed information
   private void printLoadSpeedInfo(String partitionID) {
-    LOGGER.audit("===============Load_Speed_Info===============");
-    LOGGER.audit("Total Num of Records Processed: " + getTotalRecords());
-    LOGGER.audit("Total Time Cost: " + getTotalTime(partitionID) + "(s)");
-    LOGGER.audit("Total Load Speed: " + getLoadSpeed() + "records/s");
-    LOGGER.audit("Generate Dictionaries Speed: " + getGenDicSpeed() + "records/s");
-    LOGGER.audit("Read CSV Speed: " + getReadCSVSpeed(partitionID) + " records/s");
-    LOGGER.audit("Generate Surrogate Key Speed: " + getGenSurKeySpeed(partitionID) + " records/s");
-    LOGGER.audit("Sort Key/Write Temp Files Speed: " + getSortKeySpeed(partitionID) + " records/s");
-    LOGGER.audit("MDK Step Speed: " + getMDKSpeed(partitionID) + " records/s");
-    LOGGER.audit("=============================================");
+    LOGGER.info("===============Load_Speed_Info===============");
+    LOGGER.info("Total Num of Records Processed: " + getTotalRecords());
+    LOGGER.info("Total Time Cost: " + getTotalTime(partitionID) + "(s)");
+    LOGGER.info("Total Load Speed: " + getLoadSpeed() + "records/s");
+    LOGGER.info("Generate Dictionaries Speed: " + getGenDicSpeed() + "records/s");
+    LOGGER.info("Read CSV Speed: " + getReadCSVSpeed(partitionID) + " records/s");
+    LOGGER.info("Generate Surrogate Key Speed: " + getGenSurKeySpeed(partitionID) + " records/s");
+    LOGGER.info("Sort Key/Write Temp Files Speed: " + getSortKeySpeed(partitionID) + " records/s");
+    LOGGER.info("MDK Step Speed: " + getMDKSpeed(partitionID) + " records/s");
+    LOGGER.info("=============================================");
   }
 
   public void printStatisticsInfo(String partitionID) {
     try {
-      LOGGER.audit("========== TIME_STATISTICS PartitionID: " + partitionID + "==========");
+      LOGGER.info("========== TIME_STATISTICS PartitionID: " + partitionID + "==========");
       printDicGenStatisticsInfo();
       printLruCacheLoadTimeInfo();
       printDictionaryValuesGenStatisticsInfo(partitionID);
@@ -387,7 +387,7 @@ public class CarbonLoadStatisticsImpl implements LoadStatistics {
       printHostBlockMapInfo();
       printLoadSpeedInfo(partitionID);
     } catch (Exception e) {
-      LOGGER.audit("Can't print Statistics Information");
+      LOGGER.error("Can't print Statistics Information");
     } finally {
       resetLoadStatistics();
     }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index acc266c..5ac359b 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -947,8 +947,10 @@ public final class CarbonProperties {
   public int getNumberOfCores() {
     int numberOfCores;
     try {
-      numberOfCores = Integer.parseInt(CarbonProperties.getInstance()
-          .getProperty(CarbonCommonConstants.NUM_CORES_LOADING));
+      numberOfCores = Integer.parseInt(
+          CarbonProperties.getInstance().getProperty(
+              CarbonCommonConstants.NUM_CORES_LOADING,
+              CarbonCommonConstants.NUM_CORES_DEFAULT_VAL));
     } catch (NumberFormatException exc) {
       LOGGER.error("Configured value for property " + CarbonCommonConstants.NUM_CORES_LOADING
           + " is wrong. Falling back to the default value "

http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
index 64ccca9..d27b1c4 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CompareTest.scala
@@ -268,16 +268,14 @@ object CompareTest {
   private def loadCarbonTable(spark: SparkSession, input: DataFrame, tableName: String):
Double = {
     CarbonProperties.getInstance().addProperty(
       CarbonCommonConstants.CARBON_DATA_FILE_VERSION,
-      "3"
+      "V3"
     )
     spark.sql(s"drop table if exists $tableName")
     time {
       input.write
           .format("carbondata")
           .option("tableName", tableName)
-          .option("tempCSV", "false")
           .option("single_pass", "true")
-          .option("dictionary_exclude", "id") // id is high cardinality column
           .option("table_blocksize", "32")
           .mode(SaveMode.Overwrite)
           .save()

http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
index 86d6759..702f954 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/management/CarbonInsertIntoCommand.scala
@@ -52,7 +52,7 @@ case class CarbonInsertIntoCommand(
       isPersistEnabledUserValue.equalsIgnoreCase("true") || containsLimit(child)
     val df =
       if (isPersistRequired) {
-        LOGGER.audit("Persist enabled for Insert operation")
+        LOGGER.info("Persist enabled for Insert operation")
         Dataset.ofRows(sparkSession, child)
           .persist(StorageLevel.MEMORY_AND_DISK)
       } else {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
index 705455f..9c2835e 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableDropPartitionCommand.scala
@@ -158,7 +158,6 @@ case class CarbonAlterTableDropPartitionCommand(
       CacheProvider.getInstance().dropAllCache()
       AlterTableUtil.releaseLocks(locks)
       LOGGER.info("Locks released after alter table drop partition action.")
-      LOGGER.audit("Locks released after alter table drop partition action.")
     }
     LOGGER.info(s"Alter table drop partition is successful for table $dbName.$tableName")
     LOGGER.audit(s"Alter table drop partition is successful for table $dbName.$tableName")

http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
index 99691d2..4b89296 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/partition/CarbonAlterTableSplitPartitionCommand.scala
@@ -167,7 +167,6 @@ case class CarbonAlterTableSplitPartitionCommand(
       CacheProvider.getInstance().dropAllCache()
       val LOGGER: LogService = LogServiceFactory.getLogService(this.getClass.getName)
       LOGGER.info("Locks released after alter table add/split partition action.")
-      LOGGER.audit("Locks released after alter table add/split partition action.")
       if (success) {
         LOGGER.info(s"Alter table add/split partition is successful for table $dbName.$tableName")
         LOGGER.audit(s"Alter table add/split partition is successful for table $dbName.$tableName")

http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
index b3438a4..52bae1a 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala
@@ -487,7 +487,7 @@ class CarbonFileMetastore extends CarbonMetaStore {
   private def touchSchemaFileSystemTime(): Long = {
     val (timestampFile, timestampFileType) = getTimestampFileAndType()
     if (!FileFactory.isFileExist(timestampFile, timestampFileType)) {
-      LOGGER.audit(s"Creating timestamp file for $timestampFile")
+      LOGGER.info(s"Creating timestamp file for $timestampFile")
       FileFactory
         .createNewFile(timestampFile,
           timestampFileType,

http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala
index d67ae6b..dfa9904 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonHiveMetadataUtil.scala
@@ -46,7 +46,7 @@ object CarbonHiveMetadataUtil {
       sparkSession.sessionState.catalog.dropTable(tabelIdentifier, true, false)
     } catch {
       case e: Exception =>
-        LOGGER.audit(
+        LOGGER.error(
           s"Error While deleting the table $databaseName.$tableName during drop carbon table"
+
           e.getMessage)
     }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
index 3f1ebbc..9127f14 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetaStore.scala
@@ -184,10 +184,10 @@ object CarbonMetaStoreFactory {
   def createCarbonMetaStore(conf: RuntimeConfig): CarbonMetaStore = {
     val readSchemaFromHiveMetaStore = readSchemaFromHive(conf)
     if (readSchemaFromHiveMetaStore) {
-      LOGGER.audit("Hive based carbon metastore is enabled")
+      LOGGER.info("Hive based carbon metastore is enabled")
       new CarbonHiveMetaStore()
     } else {
-      LOGGER.audit("File based carbon metastore is enabled")
+      LOGGER.info("File based carbon metastore is enabled")
       new CarbonFileMetastore()
     }
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
index a776db1..c966ff2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeFinalMergePageHolder.java
@@ -49,7 +49,7 @@ public class UnsafeFinalMergePageHolder implements SortTempChunkHolder {
     this.mergedAddresses = merger.getMergedAddresses();
     this.rowPageIndexes = merger.getRowPageIndexes();
     this.rowPages = merger.getUnsafeCarbonRowPages();
-    LOGGER.audit("Processing unsafe inmemory rows page with size : " + actualSize);
+    LOGGER.info("Processing unsafe inmemory rows page with size : " + actualSize);
     this.comparator = new IntermediateSortTempRowComparator(noDictSortColumnMapping);
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
index cbcbbae..8b4b550 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryHolder.java
@@ -43,7 +43,7 @@ public class UnsafeInmemoryHolder implements SortTempChunkHolder {
   public UnsafeInmemoryHolder(UnsafeCarbonRowPage rowPage) {
     this.actualSize = rowPage.getBuffer().getActualSize();
     this.rowPage = rowPage;
-    LOGGER.audit("Processing unsafe inmemory rows page with size : " + actualSize);
+    LOGGER.info("Processing unsafe inmemory rows page with size : " + actualSize);
     this.comparator = new IntermediateSortTempRowComparator(
         rowPage.getTableFieldStat().getIsSortColNoDictFlags());
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryMergeHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryMergeHolder.java
b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryMergeHolder.java
index 3b9d8d7..f8689d9 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryMergeHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeInmemoryMergeHolder.java
@@ -48,7 +48,7 @@ public class UnsafeInmemoryMergeHolder implements Comparable<UnsafeInmemoryMerge
   public UnsafeInmemoryMergeHolder(UnsafeCarbonRowPage rowPage, byte index) {
     this.actualSize = rowPage.getBuffer().getActualSize();
     this.rowPage = rowPage;
-    LOGGER.audit("Processing unsafe inmemory rows page with size : " + actualSize);
+    LOGGER.info("Processing unsafe inmemory rows page with size : " + actualSize);
     this.comparator = new UnsafeRowComparator(rowPage);
     this.baseObject = rowPage.getDataBlock().getBaseObject();
     currentRow = new UnsafeCarbonRowForMerge();

http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
index 527452a..86b7ac8 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/holder/UnsafeSortTempFileChunkHolder.java
@@ -131,7 +131,7 @@ public class UnsafeSortTempFileChunkHolder implements SortTempChunkHolder
{
       stream = FileFactory.getDataInputStream(tempFile.getPath(), FileFactory.FileType.LOCAL,
           readBufferSize, compressorName);
       this.entryCount = stream.readInt();
-      LOGGER.audit("Processing unsafe mode file rows with size : " + entryCount);
+      LOGGER.info("Processing unsafe mode file rows with size : " + entryCount);
       if (prefetch) {
         new DataFetcher(false).call();
         totalRecordFetch += currentBuffer.length;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index ba2b0c2..392ad59 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -110,8 +110,13 @@ public final class CarbonDataProcessorUtil {
    */
   public static void createLocations(String[] locations) {
     for (String loc : locations) {
-      if (!new File(loc).mkdirs()) {
-        LOGGER.warn("Error occurs while creating dirs: " + loc);
+      File dir = new File(loc);
+      if (dir.exists()) {
+        LOGGER.warn("dir already exists, skip dir creation: " + loc);
+      } else {
+        if (!dir.mkdirs()) {
+          LOGGER.error("Error occurs while creating dir: " + loc);
+        }
       }
     }
   }
@@ -433,7 +438,7 @@ public final class CarbonDataProcessorUtil {
             configuration.getDataLoadProperty(CarbonCommonConstants.LOAD_SORT_SCOPE)
                 .toString());
       }
-      LOGGER.warn("sort scope is set to " + sortScope);
+      LOGGER.info("sort scope is set to " + sortScope);
     } catch (Exception e) {
       sortScope = SortScopeOptions.getSortScope(CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT);
       LOGGER.warn("Exception occured while resolving sort scope. " +
@@ -453,7 +458,7 @@ public final class CarbonDataProcessorUtil {
       } else {
         sortScope = SortScopeOptions.getSortScope(sortScopeString);
       }
-      LOGGER.warn("sort scope is set to " + sortScope);
+      LOGGER.info("sort scope is set to " + sortScope);
     } catch (Exception e) {
       sortScope = SortScopeOptions.getSortScope(CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT);
       LOGGER.warn("Exception occured while resolving sort scope. " +
@@ -481,7 +486,7 @@ public final class CarbonDataProcessorUtil {
             configuration.getDataLoadProperty(CarbonCommonConstants.LOAD_BATCH_SORT_SIZE_INMB)
                 .toString());
       }
-      LOGGER.warn("batch sort size is set to " + batchSortSizeInMb);
+      LOGGER.info("batch sort size is set to " + batchSortSizeInMb);
     } catch (Exception e) {
       batchSortSizeInMb = 0;
       LOGGER.warn("Exception occured while resolving batch sort size. " +

http://git-wip-us.apache.org/repos/asf/carbondata/blob/873c3ded/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
----------------------------------------------------------------------
diff --git a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
index 47ef5f2..0a95c24 100644
--- a/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
+++ b/streaming/src/main/scala/org/apache/carbondata/streaming/StreamHandoffRDD.scala
@@ -351,7 +351,7 @@ object StreamHandoffRDD {
       LOGGER.info("********clean up done**********")
       LOGGER.audit(s"Handoff is failed for " +
                    s"${ carbonLoadModel.getDatabaseName }.${ carbonLoadModel.getTableName
}")
-      LOGGER.warn("Cannot write load metadata file as handoff failed")
+      LOGGER.error("Cannot write load metadata file as handoff failed")
       throw new Exception(errorMessage)
     }
 


Mime
View raw message