carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject carbondata git commit: [CARBONDATA-1935]fix the backword compatibility issue for tableInfo deserialization
Date Tue, 02 Jan 2018 08:40:53 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master be5134e1f -> 03ddcc85d


[CARBONDATA-1935]fix the backword compatibility issue for tableInfo deserialization

This closes #1720


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/03ddcc85
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/03ddcc85
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/03ddcc85

Branch: refs/heads/master
Commit: 03ddcc85db37cba37aedd3feb2f99f43d14b0c8a
Parents: be5134e
Author: akashrn5 <akashnilugal@gmail.com>
Authored: Fri Dec 22 18:37:20 2017 +0530
Committer: Jacky Li <jacky.likun@qq.com>
Committed: Tue Jan 2 16:40:39 2018 +0800

----------------------------------------------------------------------
 .../core/metadata/datatype/DataTypeAdapter.java | 51 ++++++++++++++++++++
 .../core/metadata/datatype/DataTypes.java       | 16 +++---
 .../apache/carbondata/core/util/CarbonUtil.java | 11 ++++-
 .../carbondata/core/util/DataTypeUtil.java      | 40 +++++++++++++++
 .../table/CarbonDescribeFormattedCommand.scala  |  2 +-
 5 files changed, 110 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/03ddcc85/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypeAdapter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypeAdapter.java
b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypeAdapter.java
new file mode 100644
index 0000000..08fbcca
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypeAdapter.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.metadata.datatype;
+
+import java.io.IOException;
+
+import org.apache.carbondata.core.util.DataTypeUtil;
+
+import com.google.gson.Gson;
+import com.google.gson.TypeAdapter;
+import com.google.gson.stream.JsonReader;
+import com.google.gson.stream.JsonToken;
+import com.google.gson.stream.JsonWriter;
+
+/**
+ * This class is added to support backward compatibility with table info object, where DATATYPE
+ * is string in old version(1.2) and OBJECT in new version(1.3)
+ */
+public class DataTypeAdapter extends TypeAdapter<Object> {
+
+  private Gson fallBack_original = new Gson();
+
+  @Override public void write(JsonWriter jsonWriter, Object o) throws IOException {
+  }
+
+  @Override public Object read(JsonReader jsonReader) throws IOException {
+    JsonToken token = jsonReader.peek();
+    if (token == JsonToken.STRING) {
+      return DataTypeUtil.valueOf(jsonReader.nextString());
+    } else {
+      // use original deserializer logic
+      return fallBack_original.fromJson(jsonReader, DataType.class);
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/03ddcc85/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
index e96917f..ad21eaa 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataTypes.java
@@ -58,14 +58,14 @@ public class DataTypes {
   static final int LONG_TYPE_ID = 7;
   static final int DOUBLE_TYPE_ID = 8;
   static final int NULL_TYPE_ID = 9;
-  static final int BYTE_TYPE_ID = 10;
-  static final int BYTE_ARRAY_TYPE_ID = 11;
-  static final int SHORT_INT_TYPE_ID = 12;
-  static final int LEGACY_LONG_TYPE_ID = 13;
-  static final int DECIMAL_TYPE_ID = 20;
-  static final int ARRAY_TYPE_ID = 21;
-  static final int STRUCT_TYPE_ID = 22;
-  static final int MAP_TYPE_ID = 23;
+  static final int BYTE_TYPE_ID = 14;
+  static final int BYTE_ARRAY_TYPE_ID = 15;
+  static final int SHORT_INT_TYPE_ID = 16;
+  static final int LEGACY_LONG_TYPE_ID = 17;
+  static final int DECIMAL_TYPE_ID = 10;
+  static final int ARRAY_TYPE_ID = 11;
+  static final int STRUCT_TYPE_ID = 12;
+  static final int MAP_TYPE_ID = 13;
 
   /**
    * create a DataType instance from uniqueId of the DataType

http://git-wip-us.apache.org/repos/asf/carbondata/blob/03ddcc85/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index bd01772..ffe4654 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -65,6 +65,7 @@ import org.apache.carbondata.core.metadata.ValueEncoderMeta;
 import org.apache.carbondata.core.metadata.blocklet.DataFileFooter;
 import org.apache.carbondata.core.metadata.blocklet.SegmentInfo;
 import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataTypeAdapter;
 import org.apache.carbondata.core.metadata.datatype.DataTypes;
 import org.apache.carbondata.core.metadata.encoder.Encoding;
 import org.apache.carbondata.core.metadata.schema.table.AggregationDataMapSchema;
@@ -90,6 +91,8 @@ import org.apache.carbondata.format.DataChunk2;
 import org.apache.carbondata.format.DataChunk3;
 
 import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.hadoop.conf.Configuration;
@@ -1924,7 +1927,6 @@ public final class CarbonUtil {
 
   // TODO: move this to carbon store API as it is related to TableInfo creation
   public static TableInfo convertGsonToTableInfo(Map<String, String> properties) {
-    Gson gson = new Gson();
     String partsNo = properties.get("carbonSchemaPartsNo");
     if (partsNo == null) {
       return null;
@@ -1938,6 +1940,13 @@ public final class CarbonUtil {
       }
       builder.append(part);
     }
+
+    // Datatype GSON adapter is added to support backward compatibility for tableInfo
+    // deserialization
+    GsonBuilder gsonBuilder = new GsonBuilder();
+    gsonBuilder.registerTypeAdapter(DataType.class, new DataTypeAdapter());
+
+    Gson gson = gsonBuilder.create();
     TableInfo tableInfo = gson.fromJson(builder.toString(), TableInfo.class);
 
     // The tableInfo is deserialized from GSON string, need to update the scale and

http://git-wip-us.apache.org/repos/asf/carbondata/blob/03ddcc85/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 0e819c4..65ea63e 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -735,4 +735,44 @@ public final class DataTypeUtil {
     return converter;
   }
 
+  /**
+   * @param name datatype string extracted from the json data
+   * @return returns the datatype based on the input string from json to deserialize the
tableInfo
+   */
+  public static DataType valueOf(String name) {
+    if (DataTypes.STRING.getName().equalsIgnoreCase(name)) {
+      return DataTypes.STRING;
+    } else if (DataTypes.DATE.getName().equalsIgnoreCase(name)) {
+      return DataTypes.DATE;
+    } else if (DataTypes.TIMESTAMP.getName().equalsIgnoreCase(name)) {
+      return DataTypes.TIMESTAMP;
+    } else if (DataTypes.BOOLEAN.getName().equalsIgnoreCase(name)) {
+      return DataTypes.BOOLEAN;
+    } else if (DataTypes.BYTE.getName().equalsIgnoreCase(name)) {
+      return DataTypes.BYTE;
+    } else if (DataTypes.SHORT.getName().equalsIgnoreCase(name)) {
+      return DataTypes.SHORT;
+    } else if (DataTypes.SHORT_INT.getName().equalsIgnoreCase(name)) {
+      return DataTypes.SHORT_INT;
+    } else if (DataTypes.INT.getName().equalsIgnoreCase(name)) {
+      return DataTypes.INT;
+    } else if (DataTypes.LONG.getName().equalsIgnoreCase(name)) {
+      return DataTypes.LONG;
+    } else if (DataTypes.LEGACY_LONG.getName().equalsIgnoreCase(name)) {
+      return DataTypes.LEGACY_LONG;
+    } else if (DataTypes.FLOAT.getName().equalsIgnoreCase(name)) {
+      return DataTypes.FLOAT;
+    } else if (DataTypes.DOUBLE.getName().equalsIgnoreCase(name)) {
+      return DataTypes.DOUBLE;
+    } else if (DataTypes.NULL.getName().equalsIgnoreCase(name)) {
+      return DataTypes.NULL;
+    } else if (DataTypes.BYTE_ARRAY.getName().equalsIgnoreCase(name)) {
+      return DataTypes.BYTE_ARRAY;
+    } else if (DataTypes.BYTE_ARRAY.getName().equalsIgnoreCase(name)) {
+      return DataTypes.BYTE_ARRAY;
+    } else {
+      throw new RuntimeException("create DataType with invalid name: " + name);
+    }
+  }
+
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/03ddcc85/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
index 5d19e0a..41656e5 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala
@@ -109,7 +109,7 @@ private[sql] case class CarbonDescribeFormattedCommand(
     results ++= Seq(("Streaming", isStreaming, ""))
 
     val tblProps = carbonTable.getTableInfo.getFactTable.getTableProperties
-    results ++= Seq(("SORT_SCOPE", tblProps.getOrDefault("sort_scope", CarbonCommonConstants
+    results ++= Seq(("SORT_SCOPE", tblProps.asScala.getOrElse("sort_scope", CarbonCommonConstants
       .LOAD_SORT_SCOPE_DEFAULT), CarbonCommonConstants.LOAD_SORT_SCOPE_DEFAULT))
 
     // show table level compaction options


Mime
View raw message