carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From manishgupt...@apache.org
Subject carbondata git commit: [CARBONDATA-2410] Error message correction when column value length exceeds 320000 character
Date Thu, 03 May 2018 08:53:18 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 167b6c004 -> 2c095542b


[CARBONDATA-2410] Error message correction when column value length exceeds 320000 character

The raw data is displayed in the client, to avoid the same the exception message is intercepted
and sent back to driver

This closes #2243


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/2c095542
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/2c095542
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/2c095542

Branch: refs/heads/master
Commit: 2c095542bd3969c28fe13866ec5bad6a94cf6b3d
Parents: 167b6c0
Author: mohammadshahidkhan <mohdshahidkhan1987@gmail.com>
Authored: Fri Apr 27 19:08:03 2018 +0530
Committer: manishgupta88 <tomanishgupta18@gmail.com>
Committed: Thu May 3 14:26:29 2018 +0530

----------------------------------------------------------------------
 .../carbondata/core/constants/CarbonCommonConstants.java      | 7 +++++++
 .../apache/carbondata/spark/util/GlobalDictionaryUtil.scala   | 4 +++-
 .../processing/loading/csvinput/CSVInputFormat.java           | 3 +++
 .../carbondata/processing/util/CarbonDataProcessorUtil.java   | 4 +++-
 4 files changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c095542/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
index b902359..553698a 100644
--- a/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
+++ b/core/src/main/java/org/apache/carbondata/core/constants/CarbonCommonConstants.java
@@ -1477,6 +1477,13 @@ public final class CarbonCommonConstants {
   @CarbonProperty public static final String CARBON_SKIP_EMPTY_LINE = "carbon.skip.empty.line";
 
   public static final String CARBON_SKIP_EMPTY_LINE_DEFAULT = "false";
+  /**
+   * Configures the parser/writer to limit the length of displayed contents being parsed/written
+   * in the exception message when an error occurs.
+   * Here {@code 0} means no exceptions will include the content being manipulated in their
+   * attributes.
+   */
+  public static final int CARBON_ERROR_CONTENT_LENGTH = 0;
 
   /**
    * if the byte size of streaming segment reach this value,

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c095542/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
index fb9ecac..5129e59 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtil.scala
@@ -824,9 +824,11 @@ object GlobalDictionaryUtil {
   def trimErrorMessage(input: String): String = {
     var errorMessage: String = null
     if (input != null && input.contains("TextParsingException:")) {
-      if (input.split("Hint").length > 0 &&
+      if (input.split("Hint").length > 1 &&
           input.split("Hint")(0).split("TextParsingException: ").length > 1) {
         errorMessage = input.split("Hint")(0).split("TextParsingException: ")(1)
+      } else if (input.split("Parser Configuration:").length > 1) {
+        errorMessage = input.split("Parser Configuration:")(0)
       }
     } else if (input != null && input.contains("Exception:")) {
       errorMessage = input.split("Exception: ")(1).split("\n")(0)

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c095542/processing/src/main/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormat.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormat.java
b/processing/src/main/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormat.java
index aebaf3b..2e3479c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormat.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/loading/csvinput/CSVInputFormat.java
@@ -210,6 +210,9 @@ public class CSVInputFormat extends FileInputFormat<NullWritable, StringArrayWri
     parserSettings.setMaxColumns(Integer.parseInt(maxColumns));
     parserSettings.getFormat().setQuote(job.get(QUOTE, QUOTE_DEFAULT).charAt(0));
     parserSettings.getFormat().setQuoteEscape(job.get(ESCAPE, ESCAPE_DEFAULT).charAt(0));
+    // setting the content length to to limit the length of displayed contents being parsed/written
+    // in the exception message when an error occurs.
+    parserSettings.setErrorContentLength(CarbonCommonConstants.CARBON_ERROR_CONTENT_LENGTH);
     return parserSettings;
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/2c095542/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
index 6ba05a2..f47853e 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonDataProcessorUtil.java
@@ -592,8 +592,10 @@ public final class CarbonDataProcessorUtil {
   public static String trimErrorMessage(String input) {
     String errorMessage = input;
     if (input != null) {
-      if (input.split("Hint").length > 0) {
+      if (input.split("Hint").length > 1) {
         errorMessage = input.split("Hint")[0];
+      } else if (input.split("Parser Configuration:").length > 1) {
+        errorMessage = input.split("Parser Configuration:")[0];
       }
     }
     return errorMessage;


Mime
View raw message