carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject carbondata git commit: Problem: Preaggregate select queries require table path of parent table to access dictionary files. Therefore in executor CarbonMetadata class was used to get parent table object. As CarbonMetadata class is only meant to be used in
Date Wed, 03 Oct 2018 14:32:59 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 682160fa1 -> 9ca985f0d


Problem:
Preaggregate select queries require table path of parent table to access dictionary files.
Therefore in executor CarbonMetadata class was used to get parent table object. As CarbonMetadata
class is only meant to be used in driver and is not filled with carbontable objects for select
queries therefore the query was throwing NPE.

Solution:
Pass parent table path from driver to executor by adding a new variable in RelationIdentifier.
This will not be written to thrift, instead will be used to carry tablePath property from
driver to executor.

This closes #2786


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/9ca985f0
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/9ca985f0
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/9ca985f0

Branch: refs/heads/master
Commit: 9ca985f0d96380d96dae6fdab2d4ee014d5ac345
Parents: 682160f
Author: kunal642 <kunalkapoor642@gmail.com>
Authored: Fri Sep 28 17:41:32 2018 +0530
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Wed Oct 3 20:02:49 2018 +0530

----------------------------------------------------------------------
 .../core/metadata/schema/table/RelationIdentifier.java  | 12 ++++++++++++
 .../carbondata/core/scan/executor/util/QueryUtil.java   | 10 ++--------
 .../spark/sql/CarbonDatasourceHadoopRelation.scala      | 12 ++++++++++++
 3 files changed, 26 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/9ca985f0/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java
index 9a1dad1..0e8042d 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/RelationIdentifier.java
@@ -32,6 +32,8 @@ public class RelationIdentifier implements Serializable, Writable {
 
   private String tableId;
 
+  private String tablePath = "";
+
   public RelationIdentifier(String databaseName, String tableName, String tableId) {
     this.databaseName = databaseName;
     this.tableName = tableName;
@@ -50,16 +52,26 @@ public class RelationIdentifier implements Serializable, Writable {
     return tableId;
   }
 
+  public String getTablePath() {
+    return tablePath;
+  }
+
+  public void setTablePath(String tablePath) {
+    this.tablePath = tablePath;
+  }
+
   @Override public void write(DataOutput out) throws IOException {
     out.writeUTF(databaseName);
     out.writeUTF(tableName);
     out.writeUTF(tableId);
+    out.writeUTF(tablePath);
   }
 
   @Override public void readFields(DataInput in) throws IOException {
     this.databaseName = in.readUTF();
     this.tableName = in.readUTF();
     this.tableId = in.readUTF();
+    this.tablePath = in.readUTF();
   }
 
   @Override public boolean equals(Object o) {

http://git-wip-us.apache.org/repos/asf/carbondata/blob/9ca985f0/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
index 9fb0857..7849d10 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
@@ -393,12 +393,6 @@ public class QueryUtil {
 
   public static AbsoluteTableIdentifier getTableIdentifierForColumn(
       CarbonDimension carbonDimension) {
-    RelationIdentifier parentRelationIdentifier =
-        carbonDimension.getColumnSchema().getParentColumnTableRelations().get(0)
-            .getRelationIdentifier();
-    String parentTablePath = CarbonMetadata.getInstance()
-        .getCarbonTable(parentRelationIdentifier.getDatabaseName(),
-            parentRelationIdentifier.getTableName()).getTablePath();
     RelationIdentifier relation = carbonDimension.getColumnSchema()
         .getParentColumnTableRelations()
         .get(0)
@@ -406,8 +400,8 @@ public class QueryUtil {
     String parentTableName = relation.getTableName();
     String parentDatabaseName = relation.getDatabaseName();
     String parentTableId = relation.getTableId();
-    return AbsoluteTableIdentifier.from(parentTablePath, parentDatabaseName, parentTableName,
-        parentTableId);
+    return AbsoluteTableIdentifier.from(relation.getTablePath(), parentDatabaseName,
+        parentTableName, parentTableId);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/carbondata/blob/9ca985f0/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
index 8a0404c..04ec75d 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql
 
+import scala.collection.JavaConverters._
 import scala.collection.mutable.{ArrayBuffer, ListBuffer}
 import scala.util.control.Breaks._
 
@@ -35,6 +36,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants
 import org.apache.carbondata.core.indexstore.PartitionSpec
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier
 import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension
 import org.apache.carbondata.core.scan.expression.Expression
 import org.apache.carbondata.core.scan.expression.logical.AndExpression
 import org.apache.carbondata.hadoop.CarbonProjection
@@ -80,6 +82,16 @@ case class CarbonDatasourceHadoopRelation(
 
     val projection = new CarbonProjection
 
+    if (carbonTable.isChildDataMap) {
+      val parentTableIdentifier = carbonTable.getTableInfo.getParentRelationIdentifiers.get(0)
+      val path = CarbonEnv.getCarbonTable(Some(parentTableIdentifier.getDatabaseName),
+        parentTableIdentifier.getTableName)(sparkSession).getTablePath
+      for (carbonDimension: CarbonDimension <- carbonTable.getAllDimensions.asScala) {
+        carbonDimension.getColumnSchema.getParentColumnTableRelations.get(0)
+          .getRelationIdentifier.setTablePath(path)
+      }
+    }
+
     // As Filter pushdown for Complex datatype is not supported, if filter is applied on
complex
     // column, then Projection pushdown on Complex Columns will not take effect. Hence, check
if
     // filter contains Struct Complex Column.


Mime
View raw message