carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject carbondata git commit: [CARBONDATA-1831] BAD_RECORDS: Data Loading with Action as Redirect & logger enable is not logging the logs in the defined path.
Date Tue, 05 Dec 2017 16:29:25 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 3d8f09bac -> 4e3151087


[CARBONDATA-1831] BAD_RECORDS: Data Loading with Action as Redirect & logger enable is
not logging the logs in the defined path.

Current system expects the carbon.badRecords.location to be absolute path with filescheme.
But, it should support absolute path without file scheme(eg.: hdfs://, viewfs://)

This closes #1609


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/4e315108
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/4e315108
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/4e315108

Branch: refs/heads/master
Commit: 4e315108763858781d0ef2c340e3343e9860b14a
Parents: 3d8f09b
Author: dhatchayani <dhatcha.official@gmail.com>
Authored: Mon Dec 4 20:01:06 2017 +0530
Committer: Jacky Li <jacky.likun@qq.com>
Committed: Wed Dec 6 00:29:10 2017 +0800

----------------------------------------------------------------------
 .../carbondata/core/datastore/filesystem/LocalCarbonFile.java     | 2 ++
 .../scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala  | 3 ++-
 2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/4e315108/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
index 5b2cc68..30eb7fe 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/filesystem/LocalCarbonFile.java
@@ -123,6 +123,7 @@ public class LocalCarbonFile implements CarbonFile {
   }
 
   public boolean renameTo(String changetoName) {
+    changetoName = getUpdatedFilePath(changetoName);
     return file.renameTo(new File(changetoName));
   }
 
@@ -318,6 +319,7 @@ public class LocalCarbonFile implements CarbonFile {
   @Override public DataOutputStream getDataOutputStream(String path, FileFactory.FileType
fileType)
       throws IOException {
     path = path.replace("\\", "/");
+    path = getUpdatedFilePath(path);
     return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(path)));
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/4e315108/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
index 77f75f2..74ed6a6 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/DataLoadingUtil.scala
@@ -209,7 +209,7 @@ object DataLoadingUtil {
     val single_pass = optionsFinal("single_pass")
     val bad_records_logger_enable = optionsFinal("bad_records_logger_enable")
     val bad_records_action = optionsFinal("bad_records_action")
-    val bad_record_path = optionsFinal("bad_record_path")
+    var bad_record_path = optionsFinal("bad_record_path")
     val global_sort_partitions = optionsFinal("global_sort_partitions")
     val timestampformat = optionsFinal("timestampformat")
     val dateFormat = optionsFinal("dateformat")
@@ -224,6 +224,7 @@ object DataLoadingUtil {
 
     if (bad_records_logger_enable.toBoolean ||
         LoggerAction.REDIRECT.name().equalsIgnoreCase(bad_records_action)) {
+      bad_record_path = CarbonUtil.checkAndAppendHDFSUrl(bad_record_path)
       if (!CarbonUtil.isValidBadStorePath(bad_record_path)) {
         CarbonException.analysisException("Invalid bad records location.")
       }


Mime
View raw message