carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [47/50] carbondata git commit: [CARBONDATA-2775] Adaptive encoding fails for Unsafe OnHeap. if, target datatype is SHORT_INT
Date Mon, 30 Jul 2018 18:43:13 GMT
[CARBONDATA-2775] Adaptive encoding fails for Unsafe OnHeap. if, target datatype is SHORT_INT

problem:
[CARBONDATA-2775] Adaptive encoding fails for Unsafe OnHeap if, target data type is SHORT_INT

solution: If ENABLE_OFFHEAP_SORT = false, in carbon property. UnsafeFixLengthColumnPage.java
will use different compress logic. Not the raw compression. In that case, for SHORT_INT data
type , conversion need to handle.

This closes #2546


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9767a586
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9767a586
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9767a586

Branch: refs/heads/branch-1.4
Commit: 9767a586f99c8ae20582ea29fa5dd58b362e0303
Parents: d720372
Author: ajantha-bhat <ajanthabhat@gmail.com>
Authored: Tue Jul 24 12:33:47 2018 +0530
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Tue Jul 31 00:11:26 2018 +0530

----------------------------------------------------------------------
 .../page/UnsafeFixLengthColumnPage.java         |  2 +
 ...UnsafeHeapColumnPageForComplexDataType.scala | 61 ++++++++++++++++++++
 2 files changed, 63 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/9767a586/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
index bcb74c0..f75deb6 100644
--- a/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/UnsafeFixLengthColumnPage.java
@@ -495,6 +495,8 @@ public class UnsafeFixLengthColumnPage extends ColumnPage {
       return totalLength / ByteUtil.SIZEOF_BYTE;
     } else if (dataType == DataTypes.SHORT) {
       return totalLength / ByteUtil.SIZEOF_SHORT;
+    } else if (dataType == DataTypes.SHORT_INT) {
+      return totalLength / ByteUtil.SIZEOF_SHORT_INT;
     } else if (dataType == DataTypes.INT) {
       return totalLength / ByteUtil.SIZEOF_INT;
     } else if (dataType == DataTypes.LONG) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/9767a586/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingUnsafeHeapColumnPageForComplexDataType.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingUnsafeHeapColumnPageForComplexDataType.scala
b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingUnsafeHeapColumnPageForComplexDataType.scala
new file mode 100644
index 0000000..acf75c1
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestAdaptiveEncodingUnsafeHeapColumnPageForComplexDataType.scala
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.integration.spark.testsuite.complexType
+
+import java.io.File
+
+import org.apache.spark.sql.test.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+/**
+ * Test class of Adaptive Encoding UnSafe Column Page with Complex Data type
+ *
+ */
+
+class TestAdaptiveEncodingUnsafeHeapColumnPageForComplexDataType
+  extends QueryTest with BeforeAndAfterAll with TestAdaptiveComplexType {
+
+  override def beforeAll(): Unit = {
+
+    new File(CarbonProperties.getInstance().getSystemFolderLocation).delete()
+    sql("DROP TABLE IF EXISTS adaptive")
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE,
+        "true")
+
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT,
+        "false")
+  }
+
+  override def afterAll(): Unit = {
+    sql("DROP TABLE IF EXISTS adaptive")
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.ENABLE_UNSAFE_COLUMN_PAGE,
+        "true")
+
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.ENABLE_OFFHEAP_SORT,
+        "true")
+  }
+
+
+}


Mime
View raw message