drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j...@apache.org
Subject drill git commit: DRILL-3263: Read Smallint and Tinyint columns in Hive tables as Integer.
Date Thu, 18 Jun 2015 18:59:09 GMT
Repository: drill
Updated Branches:
  refs/heads/master fb25973b4 -> 437706f75


DRILL-3263: Read Smallint and Tinyint columns in Hive tables as Integer.

Smallint and Tinyint are not fully implemented, this will be addressed when DRILL-2470 is
fixed. Until these types are ready for full use throughout Drill, we will be reading smallint
and tinyint data as integers, as we have much more thorough support and testing for the integer
type.

Disabled unit tests for Hive functions that take tinyint and smallint as input or produce
them as output.


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/437706f7
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/437706f7
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/437706f7

Branch: refs/heads/master
Commit: 437706f750b0ec50b60582ea2c47e7017e2718e3
Parents: fb25973
Author: Jason Altekruse <altekrusejason@gmail.com>
Authored: Mon Jun 8 17:22:03 2015 -0700
Committer: Jason Altekruse <altekrusejason@gmail.com>
Committed: Thu Jun 18 09:20:41 2015 -0700

----------------------------------------------------------------------
 .../exec/store/hive/HiveFieldConverter.java     | 37 ++++++++++++--------
 .../drill/exec/store/hive/HiveRecordReader.java | 33 +++++------------
 .../exec/store/hive/schema/DrillHiveTable.java  |  4 +--
 .../drill/exec/fn/hive/HiveTestUDFImpls.java    |  2 ++
 .../drill/exec/fn/hive/TestSampleHiveUDFs.java  |  2 ++
 .../apache/drill/exec/hive/TestHiveStorage.java | 12 +++----
 6 files changed, 43 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/437706f7/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java
b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java
index 658dd79..a59d37b 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveFieldConverter.java
@@ -71,6 +71,12 @@ public abstract class HiveFieldConverter {
 
   private static Map<PrimitiveCategory, Class< ? extends HiveFieldConverter>>
primMap = Maps.newHashMap();
 
+  // TODO (DRILL-2470)
+  // Byte and short (tinyint and smallint in SQL types) are currently read as integers
+  // as these smaller integer types are not fully supported in Drill today.
+  // Here the same types are used, as we have to read out of the correct typed converter
+  // from the hive side, in the FieldConverter classes below for Byte and Short we convert
+  // to integer when writing into Drill's vectors.
   static {
     primMap.put(PrimitiveCategory.BINARY, Binary.class);
     primMap.put(PrimitiveCategory.BOOLEAN, Boolean.class);
@@ -143,14 +149,6 @@ public abstract class HiveFieldConverter {
     }
   }
 
-  public static class Byte extends HiveFieldConverter {
-    @Override
-    public void setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV,
int outputIndex) {
-      final byte value = (byte) ((ByteObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
-      ((NullableTinyIntVector) outputVV).getMutator().setSafe(outputIndex, value);
-    }
-  }
-
   public static class Decimal9 extends HiveFieldConverter {
     private final Decimal9Holder holder = new Decimal9Holder();
 
@@ -247,19 +245,30 @@ public abstract class HiveFieldConverter {
     }
   }
 
-  public static class Long extends HiveFieldConverter {
+  // TODO (DRILL-2470)
+  // Byte and short (tinyint and smallint in SQL types) are currently read as integers
+  // as these smaller integer types are not fully supported in Drill today.
+  public static class Short extends HiveFieldConverter {
     @Override
     public void setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV,
int outputIndex) {
-      final long value = (long) ((LongObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
-      ((NullableBigIntVector) outputVV).getMutator().setSafe(outputIndex, value);
+      final int value = (short) ((ShortObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+      ((NullableIntVector) outputVV).getMutator().setSafe(outputIndex, value);
     }
   }
 
-  public static class Short extends HiveFieldConverter {
+  public static class Byte extends HiveFieldConverter {
     @Override
     public void setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV,
int outputIndex) {
-      final short value = (short) ((ShortObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
-      ((NullableSmallIntVector) outputVV).getMutator().setSafe(outputIndex, value);
+      final int value = (byte)((ByteObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+      ((NullableIntVector) outputVV).getMutator().setSafe(outputIndex, value);
+    }
+  }
+
+  public static class Long extends HiveFieldConverter {
+    @Override
+    public void setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV,
int outputIndex) {
+      final long value = (long) ((LongObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
+      ((NullableBigIntVector) outputVV).getMutator().setSafe(outputIndex, value);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/437706f7/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
index 3c8b9ba..9f63e05 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
@@ -359,8 +359,6 @@ public class HiveRecordReader extends AbstractRecordReader {
         return TypeProtos.MinorType.VARBINARY;
       case BOOLEAN:
         return MinorType.BIT;
-      case BYTE:
-        return MinorType.TINYINT;
       case DECIMAL: {
 
         if (context.getOptions().getOption(PlannerSettings.ENABLE_DECIMAL_DATA_TYPE_KEY).bool_val
== false) {
@@ -375,12 +373,15 @@ public class HiveRecordReader extends AbstractRecordReader {
         return MinorType.FLOAT8;
       case FLOAT:
         return MinorType.FLOAT4;
+      // TODO (DRILL-2470)
+      // Byte and short (tinyint and smallint in SQL types) are currently read as integers
+      // as these smaller integer types are not fully supported in Drill today.
+      case SHORT:
+      case BYTE:
       case INT:
         return MinorType.INT;
       case LONG:
         return MinorType.BIGINT;
-      case SHORT:
-        return MinorType.SMALLINT;
       case STRING:
       case VARCHAR:
         return MinorType.VARCHAR;
@@ -451,14 +452,6 @@ public class HiveRecordReader extends AbstractRecordReader {
           }
           break;
         }
-        case BYTE: {
-          TinyIntVector v = (TinyIntVector) vector;
-          byte value = (byte) val;
-          for (int j = 0; j < recordCount; j++) {
-            v.getMutator().setSafe(j, value);
-          }
-          break;
-        }
         case DOUBLE: {
           Float8Vector v = (Float8Vector) vector;
           double value = (double) val;
@@ -475,6 +468,8 @@ public class HiveRecordReader extends AbstractRecordReader {
           }
           break;
         }
+        case BYTE:
+        case SHORT:
         case INT: {
           IntVector v = (IntVector) vector;
           int value = (int) val;
@@ -491,14 +486,6 @@ public class HiveRecordReader extends AbstractRecordReader {
           }
           break;
         }
-        case SHORT: {
-          SmallIntVector v = (SmallIntVector) vector;
-          short value = (short) val;
-          for (int j = 0; j < recordCount; j++) {
-            v.getMutator().setSafe(j, value);
-          }
-          break;
-        }
         case VARCHAR:
         case STRING: {
           VarCharVector v = (VarCharVector) vector;
@@ -603,8 +590,6 @@ public class HiveRecordReader extends AbstractRecordReader {
         return value.getBytes();
       case BOOLEAN:
         return Boolean.parseBoolean(value);
-      case BYTE:
-        return Byte.parseByte(value);
       case DECIMAL: {
         DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) typeInfo;
         return HiveDecimalUtils.enforcePrecisionScale(HiveDecimal.create(value),
@@ -614,12 +599,12 @@ public class HiveRecordReader extends AbstractRecordReader {
         return Double.parseDouble(value);
       case FLOAT:
         return Float.parseFloat(value);
+      case BYTE:
+      case SHORT:
       case INT:
         return Integer.parseInt(value);
       case LONG:
         return Long.parseLong(value);
-      case SHORT:
-        return Short.parseShort(value);
       case STRING:
       case VARCHAR:
         return value.getBytes();

http://git-wip-us.apache.org/repos/asf/drill/blob/437706f7/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
index 0da28e0..d0ea143 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/schema/DrillHiveTable.java
@@ -82,10 +82,8 @@ public class DrillHiveTable extends DrillTable{
         return typeFactory.createSqlType(SqlTypeName.BOOLEAN);
 
       case BYTE:
-        return typeFactory.createSqlType(SqlTypeName.TINYINT);
-
       case SHORT:
-        return typeFactory.createSqlType(SqlTypeName.SMALLINT);
+        return typeFactory.createSqlType(SqlTypeName.INTEGER);
 
       case INT:
         return typeFactory.createSqlType(SqlTypeName.INTEGER);

http://git-wip-us.apache.org/repos/asf/drill/blob/437706f7/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/HiveTestUDFImpls.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/HiveTestUDFImpls.java
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/HiveTestUDFImpls.java
index 31e4715..c5699eb 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/HiveTestUDFImpls.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/HiveTestUDFImpls.java
@@ -151,6 +151,7 @@ public class HiveTestUDFImpls {
     }
   }
 
+  // TODO(DRILL-2470) - re-enable the test case for this function in TestSampleHiveUDFs
   @Description(name = "testHiveUDFBYTE", value = "_FUNC_(BYTE) - Tests byte data as input
and output")
   public static class GenericUDFTestBYTE extends GenericUDFTestBase {
     public GenericUDFTestBYTE() {
@@ -158,6 +159,7 @@ public class HiveTestUDFImpls {
     }
   }
 
+  // TODO(DRILL-2470) - re-enable the test case for this function in TestSampleHiveUDFs
   @Description(name = "testHiveUDFSHORT", value = "_FUNC_(SHORT) - Tests short data as input
and output")
   public static class GenericUDFTestSHORT extends GenericUDFTestBase {
     public GenericUDFTestSHORT() {

http://git-wip-us.apache.org/repos/asf/drill/blob/437706f7/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java
index 86a78e5..6a38e1c 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/fn/hive/TestSampleHiveUDFs.java
@@ -47,6 +47,7 @@ public class TestSampleHiveUDFs extends HiveTestBase {
     helper(query, expected);
   }
 
+  @Ignore("DRILL-2470")
   @Test
   public void byteInOut() throws Exception{
     String query = "SELECT testHiveUDFByte(tinyint_field) as col1 FROM hive.readtest";
@@ -54,6 +55,7 @@ public class TestSampleHiveUDFs extends HiveTestBase {
     helper(query, expected);
   }
 
+  @Ignore("DRILL-2470")
   @Test
   public void shortInOut() throws Exception{
     String query = "SELECT testHiveUDFShort(smallint_field) as col1 FROM hive.readtest";

http://git-wip-us.apache.org/repos/asf/drill/blob/437706f7/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
index 39e8f54..a5bca93 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
@@ -92,7 +92,7 @@ public class TestHiveStorage extends HiveTestBase {
               .baselineValues(
                   "binaryfield",
                   false,
-                  (byte) 34,
+                  34,
                   new BigDecimal("66"),
                   new BigDecimal("2347.92"),
                   new BigDecimal("2758725827.99990"),
@@ -102,14 +102,14 @@ public class TestHiveStorage extends HiveTestBase {
                   4.67f,
                   123456,
                   234235L,
-                  (short) 3455,
+                  3455,
                   "stringfield",
                   "varcharfield",
                   new DateTime(Timestamp.valueOf("2013-07-05 17:01:00").getTime()),
                   new DateTime(Date.valueOf("2013-07-05").getTime()),
                   "binary",
                   true,
-                  (byte) 64,
+                  64,
                   new BigDecimal("37"),
                   new BigDecimal("36.90"),
                   new BigDecimal("3289379872.94565"),
@@ -119,7 +119,7 @@ public class TestHiveStorage extends HiveTestBase {
                   4.67f,
                   123456,
                   234235L,
-                  (short) 3455,
+                  3455,
                   "string",
                   "varchar",
                   new DateTime(Timestamp.valueOf("2013-07-05 17:01:00").getTime()),
@@ -128,7 +128,7 @@ public class TestHiveStorage extends HiveTestBase {
                   null, null, null, null, null, null, null, null, null, null, null, null,
null, null, null, null, null,
                   "binary",
                   true,
-                  (byte) 64,
+                  64,
                   new BigDecimal("37"),
                   new BigDecimal("36.90"),
                   new BigDecimal("3289379872.94565"),
@@ -138,7 +138,7 @@ public class TestHiveStorage extends HiveTestBase {
                   4.67f,
                   123456,
                   234235L,
-                  (short) 3455,
+                  3455,
                   "string",
                   "varchar",
                   new DateTime(Timestamp.valueOf("2013-07-05 17:01:00").getTime()),


Mime
View raw message