drill-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jacq...@apache.org
Subject [5/5] git commit: DRILL-680: INFORMATION_SCHEMA.COLUMNS does not display HBase column families
Date Wed, 21 May 2014 20:28:43 GMT
DRILL-680: INFORMATION_SCHEMA.COLUMNS does not display HBase column families

+ Enhanced result layout with option to set output width on per column basis.
+ Pretty print plan fragments.


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ae570aad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ae570aad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ae570aad

Branch: refs/heads/master
Commit: ae570aad8b572a5c1fe612f79fe2524cb5e4e879
Parents: ea1ab62
Author: Aditya Kishore <aditya@maprtech.com>
Authored: Tue May 20 06:34:48 2014 -0700
Committer: Jacques Nadeau <jacques@apache.org>
Committed: Wed May 21 08:41:57 2014 -0700

----------------------------------------------------------------------
 .../drill/exec/store/hbase/DrillHBaseTable.java | 62 ++++++++++++++++++++
 .../exec/store/hbase/HBaseSchemaFactory.java    |  6 +-
 .../org/apache/drill/hbase/BaseHBaseTest.java   | 12 ++--
 .../drill/hbase/TestHBaseFilterPushDown.java    |  6 ++
 .../drill/hbase/TestHBaseProjectPushDown.java   |  8 ++-
 .../planner/fragment/SimpleParallelizer.java    |  8 +--
 .../org/apache/drill/exec/rpc/RpcException.java | 10 ++--
 .../org/apache/drill/exec/util/VectorUtil.java  | 52 ++++++++++------
 8 files changed, 126 insertions(+), 38 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/DrillHBaseTable.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/DrillHBaseTable.java
b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/DrillHBaseTable.java
new file mode 100644
index 0000000..ff6991f
--- /dev/null
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/DrillHBaseTable.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.hbase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Set;
+
+import org.apache.drill.exec.planner.logical.DrillTable;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.eigenbase.reltype.RelDataType;
+import org.eigenbase.reltype.RelDataTypeFactory;
+import org.eigenbase.sql.type.SqlTypeName;
+
+public class DrillHBaseTable extends DrillTable implements DrillHBaseConstants {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillHBaseTable.class);
+
+  private HTableDescriptor table;
+
+  public DrillHBaseTable(String storageEngineName, HBaseStoragePlugin plugin, HBaseScanSpec
scanSpec) {
+    super(storageEngineName, plugin, scanSpec);
+    try(HBaseAdmin admin = new HBaseAdmin(plugin.getConfig().getHBaseConf())) {
+      table = admin.getTableDescriptor(HBaseUtils.getBytes(scanSpec.getTableName()));
+    } catch (IOException e) {
+      logger.warn("Failure while loading table names for database '{}'.", storageEngineName,
e);
+    }
+  }
+
+  @Override
+  public RelDataType getRowType(RelDataTypeFactory typeFactory) {
+    ArrayList<RelDataType> typeList = new ArrayList<>();
+    ArrayList<String> fieldNameList = new ArrayList<>();
+
+    fieldNameList.add(ROW_KEY);
+    typeList.add(typeFactory.createSqlType(SqlTypeName.ANY));
+
+    Set<byte[]> families = table.getFamiliesKeys();
+    for (byte[] family : families) {
+      fieldNameList.add(Bytes.toString(family));
+      typeList.add(typeFactory.createMapType(typeFactory.createSqlType(SqlTypeName.VARCHAR),
typeFactory.createSqlType(SqlTypeName.ANY)));
+    }
+    return typeFactory.createStructType(typeList, fieldNameList);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java
b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java
index 84f363b..7b76092 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseSchemaFactory.java
@@ -25,7 +25,6 @@ import net.hydromatic.optiq.Schema;
 import net.hydromatic.optiq.SchemaPlus;
 import net.hydromatic.optiq.Table;
 
-import org.apache.drill.exec.planner.logical.DynamicDrillTable;
 import org.apache.drill.exec.rpc.user.UserSession;
 import org.apache.drill.exec.store.AbstractSchema;
 import org.apache.drill.exec.store.SchemaFactory;
@@ -74,8 +73,8 @@ public class HBaseSchemaFactory implements SchemaFactory {
 
     @Override
     public Table getTable(String name) {
-      Object selection = new HBaseScanSpec(name);
-      return new DynamicDrillTable(plugin, schemaName, selection);
+      HBaseScanSpec scanSpec = new HBaseScanSpec(name);
+      return new DrillHBaseTable(schemaName, plugin, scanSpec);
     }
 
     @Override
@@ -99,4 +98,5 @@ public class HBaseSchemaFactory implements SchemaFactory {
     }
 
   }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
index 9e07d9f..96f0c4a 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/BaseHBaseTest.java
@@ -45,7 +45,7 @@ public class BaseHBaseTest extends BaseTestQuery {
 
   @Rule public TestName TEST_NAME = new TestName();
 
-  private int columnWidth = 8;
+  private int[] columnWidths = new int[] { 8 };
 
   @Before
   public void printID() throws Exception {
@@ -58,7 +58,7 @@ public class BaseHBaseTest extends BaseTestQuery {
      * Change the following to HBaseTestsSuite.configure(false, true)
      * if you want to test against an externally running HBase cluster.
      */
-    HBaseTestsSuite.configure(true, true);
+    HBaseTestsSuite.configure(false, true);
 
     HBaseTestsSuite.initCluster();
     HBaseStoragePlugin plugin = (HBaseStoragePlugin) bit.getContext().getStorage().getPlugin("hbase");
@@ -71,9 +71,13 @@ public class BaseHBaseTest extends BaseTestQuery {
   }
 
   protected void setColumnWidth(int columnWidth) {
-    this.columnWidth = columnWidth;
+    this.columnWidths = new int[] { columnWidth };
   }
 
+  protected void setColumnWidths(int[] columnWidths) {
+    this.columnWidths = columnWidths;
+  }
+  
   protected String getPlanText(String planFile, String tableName) throws IOException {
     return Files.toString(FileUtils.getResourceAsFile(planFile), Charsets.UTF_8)
         .replaceFirst("\"hbase\\.zookeeper\\.property\\.clientPort\".*:.*\\d+", "\"hbase.zookeeper.property.clientPort\"
: " + HBaseTestsSuite.getZookeeperPort())
@@ -102,7 +106,7 @@ public class BaseHBaseTest extends BaseTestQuery {
       if (loader.getRecordCount() <= 0) {
         break;
       }
-      VectorUtil.showVectorAccessibleContent(loader, columnWidth);
+      VectorUtil.showVectorAccessibleContent(loader, columnWidths);
       loader.clear();
       result.release();
     }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
index 90404b7..2071e90 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseFilterPushDown.java
@@ -23,6 +23,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
 
   @Test
   public void testFilterPushDownRowKeyEqual() throws Exception {
+    setColumnWidths(new int[] {8, 38, 38});
     runSQLVerifyCount("SELECT\n"
         + "  *\n"
         + "FROM\n"
@@ -34,6 +35,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
 
   @Test
   public void testFilterPushDownRowKeyGreaterThan() throws Exception {
+    setColumnWidths(new int[] {8, 38, 38});
     runSQLVerifyCount("SELECT\n"
         + "  *\n"
         + "FROM\n"
@@ -45,6 +47,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
 
   @Test
   public void testFilterPushDownRowKeyBetween() throws Exception {
+    setColumnWidths(new int[] {8, 74, 38});
     runSQLVerifyCount("SELECT\n"
         + "  *\n"
         + "FROM\n"
@@ -56,6 +59,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
 
   @Test
   public void testFilterPushDownMultiColumns() throws Exception {
+    setColumnWidths(new int[] {8, 74, 38});
     runSQLVerifyCount("SELECT\n"
         + "  *\n"
         + "FROM\n"
@@ -67,6 +71,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
 
   @Test
   public void testFilterPushDownConvertExpression() throws Exception {
+    setColumnWidths(new int[] {8, 38, 38});
     runSQLVerifyCount("SELECT\n"
         + "  *\n"
         + "FROM\n"
@@ -78,6 +83,7 @@ public class TestHBaseFilterPushDown extends BaseHBaseTest {
 
   @Test
   public void testFilterPushDownRowKeyLessThanOrEqualTo() throws Exception {
+    setColumnWidths(new int[] {8, 74, 38});
     runSQLVerifyCount("SELECT\n"
         + "  *\n"
         + "FROM\n"

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
index b66d2ed..7037b9a 100644
--- a/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
+++ b/contrib/storage-hbase/src/test/java/org/apache/drill/hbase/TestHBaseProjectPushDown.java
@@ -23,6 +23,7 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest {
 
   @Test
   public void testRowKeyPushDown() throws Exception{
+    setColumnWidth(8);
     runSQLVerifyCount("SELECT\n"
         + "row_key\n"
         + "FROM\n"
@@ -32,6 +33,7 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest {
 
   @Test
   public void testColumnWith1RowPushDown() throws Exception{
+    setColumnWidth(6);
     runSQLVerifyCount("SELECT\n"
         + "f2['c7'] as `f[c7]`\n"
         + "FROM\n"
@@ -41,7 +43,7 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest {
 
   @Test
   public void testRowKeyAndColumnPushDown() throws Exception{
-    setColumnWidth(9);
+    setColumnWidths(new int[] {8, 9, 6, 2, 6});
     runSQLVerifyCount("SELECT\n"
         + "row_key, f['c1']*31 as `f[c1]*31`, f['c2'] as `f[c2]`, 5 as `5`, 'abc' as `'abc'`\n"
         + "FROM\n"
@@ -51,9 +53,9 @@ public class TestHBaseProjectPushDown extends BaseHBaseTest {
 
   @Test
   public void testColumnFamilyPushDown() throws Exception{
-    setColumnWidth(74);
+    setColumnWidths(new int[] {8, 74, 38});
     runSQLVerifyCount("SELECT\n"
-        + "f, f2\n"
+        + "row_key, f, f2\n"
         + "FROM\n"
         + "  hbase.`[TABLE_NAME]` tableName"
         , 6);

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
index c34869d..313a81d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/fragment/SimpleParallelizer.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.planner.fragment;
 import java.util.Collection;
 import java.util.List;
 
+import org.apache.commons.lang3.StringEscapeUtils;
 import org.apache.drill.common.exceptions.ExecutionSetupException;
 import org.apache.drill.common.exceptions.PhysicalOperatorSetupException;
 import org.apache.drill.exec.exception.FragmentSetupException;
@@ -38,9 +39,6 @@ import org.apache.drill.exec.work.QueryWorkUnit;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
-import org.apache.drill.exec.expr.fn.impl.DateUtility;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
 
 /**
  * The simple parallelizer determines the level of parallelization of a plan based on the
cost of the underlying
@@ -143,11 +141,11 @@ public class SimpleParallelizer {
             .build();
 
         if (isRootNode) {
-          logger.debug("Root fragment:\n {}", fragment);
+          logger.debug("Root fragment:\n {}", StringEscapeUtils.unescapeJava(fragment.toString()));
           rootFragment = fragment;
           rootOperator = root;
         } else {
-          logger.debug("Remote fragment:\n {}", fragment);
+          logger.debug("Remote fragment:\n {}", StringEscapeUtils.unescapeJava(fragment.toString()));
           fragments.add(fragment);
         }
       }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java
index dba72a1..9b5eb1d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcException.java
@@ -19,13 +19,14 @@ package org.apache.drill.exec.rpc;
 
 import java.util.concurrent.ExecutionException;
 
-import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.lang3.StringEscapeUtils;
 import org.apache.drill.common.exceptions.DrillIOException;
 
 /**
  * Parent class for all rpc exceptions.
  */
 public class RpcException extends DrillIOException{
+  private static final long serialVersionUID = -5964230316010502319L;
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RpcException.class);
 
   public RpcException() {
@@ -47,17 +48,16 @@ public class RpcException extends DrillIOException{
   public RpcException(Throwable cause) {
     super(cause);
   }
-  
+
   public static RpcException mapException(Throwable t){
     while(t instanceof ExecutionException) t = ((ExecutionException)t).getCause();
     if(t instanceof RpcException) return ((RpcException) t);
     return new RpcException(t);
   }
-  
+
   public static RpcException mapException(String message, Throwable t){
     while(t instanceof ExecutionException) t = ((ExecutionException)t).getCause();
     return new RpcException(message, t);
   }
-  
-  
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ae570aad/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
index a43225e..bdba1d8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/VectorUtil.java
@@ -17,14 +17,11 @@
  */
 package org.apache.drill.exec.util;
 
-import java.util.Arrays;
 import java.util.List;
 
 import org.apache.commons.lang.StringUtils;
-import org.apache.drill.exec.exception.SchemaChangeException;
 import org.apache.drill.exec.record.VectorAccessible;
 import org.apache.drill.exec.record.VectorWrapper;
-import org.apache.drill.exec.rpc.RpcException;
 
 import com.beust.jcommander.internal.Lists;
 
@@ -37,7 +34,7 @@ public class VectorUtil {
     int rows = va.getRecordCount();
     List<String> columns = Lists.newArrayList();
     for (VectorWrapper<?> vw : va) {
-      columns.add(vw.getValueVector().getField().getAsSchemaPath().getAsUnescapedPath());
+      columns.add(vw.getValueVector().getField().getPath().getAsUnescapedPath());
     }
 
     int width = columns.size();
@@ -71,55 +68,74 @@ public class VectorUtil {
   }
 
   public static void showVectorAccessibleContent(VectorAccessible va) {
-      showVectorAccessibleContent(va, DEFAULT_COLUMN_WIDTH);
+    showVectorAccessibleContent(va, DEFAULT_COLUMN_WIDTH);
   }
+
   public static void showVectorAccessibleContent(VectorAccessible va, int columnWidth) {
+    showVectorAccessibleContent(va, new int[]{ columnWidth });
+  }
 
-    int rows = va.getRecordCount();
+  public static void showVectorAccessibleContent(VectorAccessible va, int[] columnWidths)
{
+    int width = 0;
+    int columnIndex = 0;
     List<String> columns = Lists.newArrayList();
+    List<String> formats = Lists.newArrayList();
     for (VectorWrapper<?> vw : va) {
-      columns.add(vw.getValueVector().getField().getAsSchemaPath().getAsUnescapedPath());
+      int columnWidth = getColumnWidth(columnWidths, columnIndex);
+      width += columnWidth + 2;
+      formats.add("| %-" + columnWidth + "s");
+      columns.add(vw.getValueVector().getField().getPath().getAsUnescapedPath());
+      columnIndex++;
     }
 
-    int width = columns.size() * (columnWidth + 2);
-
-    String format = ("| %-" + columnWidth + "s");
-
+    int rows = va.getRecordCount();
     for (int row = 0; row < rows; row++) {
+      // header, every 50 rows.
       if (row%50 == 0) {
         System.out.println(StringUtils.repeat("-", width + 1));
+        columnIndex = 0;
         for (String column : columns) {
-          System.out.printf(format, column.length() <= columnWidth ? column : column.substring(0,
columnWidth - 1));
+          int columnWidth = getColumnWidth(columnWidths, columnIndex);
+          System.out.printf(formats.get(columnIndex), column.length() <= columnWidth ?
column : column.substring(0, columnWidth - 1));
+          columnIndex++;
         }
         System.out.printf("|\n");
         System.out.println(StringUtils.repeat("-", width + 1));
       }
+      // column values
+      columnIndex = 0;
       for (VectorWrapper<?> vw : va) {
+        int columnWidth = getColumnWidth(columnWidths, columnIndex);
         Object o = vw.getValueVector().getAccessor().getObject(row);
         if (o == null) {
           //null value
-          System.out.printf(format, "");
+          System.out.printf(formats.get(columnIndex), "");
         }
         else if (o instanceof byte[]) {
           String value = new String((byte[]) o);
-          System.out.printf(format, value.length() <= columnWidth ? value : value.substring(0,
columnWidth - 1));
+          System.out.printf(formats.get(columnIndex), value.length() <= columnWidth ?
value : value.substring(0, columnWidth - 1));
         } else if (o instanceof List) {
           System.out.printf("| %s", o);
         } else {
           String value = o.toString();
-          System.out.printf(format, value.length() <= columnWidth ? value : value.substring(0,columnWidth
- 1));
+          System.out.printf(formats.get(columnIndex), value.length() <= columnWidth ?
value : value.substring(0,columnWidth - 1));
         }
+        columnIndex++;
       }
       System.out.printf("|\n");
     }
+    if (rows > 0) {
+      System.out.println(StringUtils.repeat("-", width + 1));
+    }
 
     for (VectorWrapper<?> vw : va) {
       vw.clear();
     }
-
-    if (rows > 0 )
-      System.out.println(StringUtils.repeat("-", width + 1));
   }
 
+  private static int getColumnWidth(int[] columnWidths, int columnIndex) {
+    return (columnWidths == null) ? DEFAULT_COLUMN_WIDTH
+        : (columnWidths.length > columnIndex) ? columnWidths[columnIndex] : columnWidths[0];
+  }
 
 }


Mime
View raw message