drill-commits mailing list archives

Site index · List index

Message view	« Date » · « Thread »
Top	« Date » · « Thread »
From	amansi...@apache.org
Subject	[2/4] drill git commit: DRILL-4264: Allow field names to include dots
Date	Tue, 05 Sep 2017 04:38:56 GMT
http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java index 0394029..2d3c13c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/FieldIdUtil.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -163,7 +163,7 @@ public class FieldIdUtil { } public static TypedFieldId getFieldId(ValueVector vector, int id, SchemaPath expectedPath, boolean hyper) { - if (!expectedPath.getRootSegment().getNameSegment().getPath().equalsIgnoreCase(vector.getField().getPath())) { + if (!expectedPath.getRootSegment().getPath().equalsIgnoreCase(vector.getField().getName())) { return null; } PathSegment seg = expectedPath.getRootSegment(); http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapUtility.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapUtility.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapUtility.java index 7a5863a..72c094a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapUtility.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapUtility.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -222,7 +222,7 @@ public class MapUtility { } } catch (ClassCastException e) { final MaterializedField field = fieldReader.getField(); - throw new DrillRuntimeException(String.format(TYPE_MISMATCH_ERROR, field.getPath(), field.getType())); + throw new DrillRuntimeException(String.format(TYPE_MISMATCH_ERROR, field.getName(), field.getType())); } } } http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java index 4d49c7b..f7f41fc 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java +++ b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java @@ -626,6 +626,11 @@ public class BaseTestQuery extends ExecTest { copyMetaDataCacheToTempWithReplacements(srcFileOnClassPath, destFolderInTmp, metaFileName, null); } + protected static void copyMetaDataCacheToTempReplacingInternalPaths(Path srcFileOnClassPath, String destFolderInTmp, + String metaFileName) throws IOException { + copyMetaDataCacheToTempReplacingInternalPaths(srcFileOnClassPath.toUri().getPath(), destFolderInTmp, metaFileName); + } + /** * Old metadata cache files include full paths to the files that have been scanned. * <p> http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java b/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java index 64aeef8..2bc78d4 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java +++ b/exec/java-exec/src/test/java/org/apache/drill/DrillTestWrapper.java @@ -14,7 +14,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - *****************************************************************************/ + / package org.apache.drill; import static org.junit.Assert.assertEquals; @@ -33,7 +33,6 @@ import java.util.Map; import java.util.Set; import java.util.TreeMap; -import com.google.common.base.Preconditions; import org.apache.commons.lang3.tuple.Pair; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.types.TypeProtos; @@ -239,7 +238,7 @@ public class DrillTestWrapper { logger.debug("reading batch with " + loader.getRecordCount() + " rows, total read so far " + totalRecords); totalRecords += loader.getRecordCount(); for (VectorWrapper<?> w : loader) { - String field = SchemaPath.getSimplePath(w.getField().getPath()).toExpr(); + String field = SchemaPath.getSimplePath(w.getField().getName()).toExpr(); if (!combinedVectors.containsKey(field)) { MaterializedField mf = w.getField(); ValueVector[] vvList = (ValueVector[]) Array.newInstance(mf.getValueClass(), 1); @@ -350,7 +349,7 @@ public class DrillTestWrapper { if (schema == null) { schema = loader.getSchema(); for (MaterializedField mf : schema) { - combinedVectors.put(SchemaPath.getSimplePath(mf.getPath()).toExpr(), new ArrayList<Object>()); + combinedVectors.put(SchemaPath.getSimplePath(mf.getName()).toExpr(), new ArrayList<>()); } } else { // TODO - actually handle schema changes, this is just to get access to the SelectionVectorMode @@ -361,7 +360,7 @@ public class DrillTestWrapper { logger.debug("reading batch with " + loader.getRecordCount() + " rows, total read so far " + totalRecords); totalRecords += loader.getRecordCount(); for (VectorWrapper<?> w : loader) { - String field = SchemaPath.getSimplePath(w.getField().getPath()).toExpr(); + String field = SchemaPath.getSimplePath(w.getField().getName()).toExpr(); ValueVector[] vectors; if (w.isHyper()) { vectors = w.getValueVectors(); @@ -428,18 +427,18 @@ public class DrillTestWrapper { final BatchSchema schema = loader.getSchema(); final List<Pair<SchemaPath, TypeProtos.MajorType>> expectedSchema = testBuilder.getExpectedSchema(); - if(schema.getFieldCount() != expectedSchema.size()) { + if (schema.getFieldCount() != expectedSchema.size()) { throw new Exception("Expected and actual numbers of columns do not match."); } - for(int i = 0; i < schema.getFieldCount(); ++i) { - final String actualSchemaPath = schema.getColumn(i).getPath(); + for (int i = 0; i < schema.getFieldCount(); ++i) { + final String actualSchemaPath = schema.getColumn(i).getName(); final TypeProtos.MajorType actualMajorType = schema.getColumn(i).getType(); - final String expectedSchemaPath = expectedSchema.get(i).getLeft().getAsUnescapedPath(); + final String expectedSchemaPath = expectedSchema.get(i).getLeft().getRootSegmentPath(); final TypeProtos.MajorType expectedMajorType = expectedSchema.get(i).getValue(); - if(!actualSchemaPath.equals(expectedSchemaPath) + if (!actualSchemaPath.equals(expectedSchemaPath) \|\| !actualMajorType.equals(expectedMajorType)) { throw new Exception(String.format("Schema path or type mismatch for column #%d:\n" + "Expected schema path: %s\nActual schema path: %s\nExpected type: %s\nActual type: %s", @@ -448,8 +447,8 @@ public class DrillTestWrapper { } } - } finally { - if(batch != null) { + } finally { + if (batch != null) { batch.release(); } loader.clear(); @@ -609,7 +608,7 @@ public class DrillTestWrapper { private Map<SchemaPath, TypeProtos.MajorType> getTypeMapFromBatch(QueryDataBatch batch) { Map<SchemaPath, TypeProtos.MajorType> typeMap = new HashMap<>(); for (int i = 0; i < batch.getHeader().getDef().getFieldCount(); i++) { - typeMap.put(SchemaPath.getSimplePath(MaterializedField.create(batch.getHeader().getDef().getField(i)).getPath()), + typeMap.put(SchemaPath.getSimplePath(MaterializedField.create(batch.getHeader().getDef().getField(i)).getName()), batch.getHeader().getDef().getField(i).getMajorType()); } return typeMap; @@ -646,9 +645,9 @@ public class DrillTestWrapper { if (obj instanceof Text) { obj = obj.toString(); } - record.put(SchemaPath.getSimplePath(w.getField().getPath()).toExpr(), obj); + record.put(SchemaPath.getSimplePath(w.getField().getName()).toExpr(), obj); } - record.put(SchemaPath.getSimplePath(w.getField().getPath()).toExpr(), obj); + record.put(SchemaPath.getSimplePath(w.getField().getName()).toExpr(), obj); } materializedRecords.add(record); } http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java index e422a77..eb11532 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java +++ b/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -349,7 +349,7 @@ public class PlanTestBase extends BaseTestQuery { } if (!silent) { - System.out.println(vw.getValueVector().getField().getPath()); + System.out.println(vw.getValueVector().getField().getName()); } final ValueVector vv = vw.getValueVector(); for (int i = 0; i < vv.getAccessor().getValueCount(); i++) { http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java b/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java index ee350ce..6965ab5 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java +++ b/exec/java-exec/src/test/java/org/apache/drill/TestUnionAll.java @@ -28,9 +28,12 @@ import org.apache.drill.exec.work.foreman.SqlUnsupportedException; import org.apache.drill.exec.work.foreman.UnsupportedRelOperatorException; import org.junit.Test; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; import java.util.List; -public class TestUnionAll extends BaseTestQuery{ +public class TestUnionAll extends BaseTestQuery { private static final String sliceTargetSmall = "alter session set `planner.slice_target` = 1"; private static final String sliceTargetDefault = "alter session reset `planner.slice_target`"; @@ -1189,4 +1192,41 @@ public class TestUnionAll extends BaseTestQuery{ .go(); } + @Test // DRILL-4264 + public void testFieldWithDots() throws Exception { + File directory = new File(BaseTestQuery.getTempDir("json/input")); + try { + directory.mkdirs(); + String fileName = "table.json"; + try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(directory, fileName)))) { + writer.write("{\"rk.q\": \"a\", \"m\": {\"a.b\":\"1\", \"a\":{\"b\":\"2\"}, \"c\":\"3\"}}"); + } + + String query = String.format("select * from (" + + "(select t.m.`a.b` as a,\n" + + "t.m.a.b as b,\n" + + "t.m['a.b'] as c,\n" + + "t.rk.q as d,\n" + + "t.`rk.q` as e\n" + + "from dfs_test.`%1$s/%2$s` t)\n" + + "union all\n" + + "(select t.m.`a.b` as a,\n" + + "t.m.a.b as b,\n" + + "t.m['a.b'] as c,\n" + + "t.rk.q as d,\n" + + "t.`rk.q` as e\n" + + "from dfs_test.`%1$s/%2$s` t))", + directory.toPath().toString(), fileName); + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("a", "b", "c", "d", "e") + .baselineValues("1", "2", "1", null, "a") + .baselineValues("1", "2", "1", null, "a") + .go(); + + } finally { + org.apache.commons.io.FileUtils.deleteQuietly(directory); + } + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java index c2ab18a..cf7fd90 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -93,4 +93,23 @@ public class TestSchemaPathMaterialization extends BaseTestQuery { .go(); } + @Test //DRILL-4264 + public void testFieldNameWithDot() throws Exception { + final String tableName = "dfs_test.tmp.table_with_dot_field"; + try { + test("create table %s as select o_custkey as `x.y.z` from cp.`tpch/orders.parquet`", tableName); + + final String query = "select * from %s t where `x.y.z`=1091"; + + testBuilder() + .sqlQuery(query, tableName) + .unOrdered() + .baselineColumns("`x.y.z`") + .baselineValues(1091) + .baselineValues(1091) + .go(); + } finally { + test("drop table if exists %s", tableName); + } + } } http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java index 36ee1b9..cfb1c5f 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestAggregateFunctions.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -30,6 +30,9 @@ import org.apache.drill.exec.rpc.user.QueryDataBatch; import org.junit.Ignore; import org.junit.Test; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; import java.util.List; import java.util.Map; @@ -570,4 +573,32 @@ public class TestAggregateFunctions extends BaseTestQuery { } } + @Test // DRILL-4264 + public void testCountOnFieldWithDots() throws Exception { + File directory = new File(BaseTestQuery.getTempDir("json/input")); + try { + directory.mkdirs(); + String fileName = "table.json"; + try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(directory, fileName)))) { + writer.write("{\"rk.q\": \"a\", \"m\": {\"a.b\":\"1\", \"a\":{\"b\":\"2\"}, \"c\":\"3\"}}"); + } + + String query = String.format("select count(t.m.`a.b`) as a,\n" + + "count(t.m.a.b) as b,\n" + + "count(t.m['a.b']) as c,\n" + + "count(t.rk.q) as d,\n" + + "count(t.`rk.q`) as e\n" + + "from dfs_test.`%s/%s` t", + directory.toPath().toString(), fileName); + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("a", "b", "c", "d", "e") + .baselineValues(1L, 1L, 1L, 0L, 1L) + .go(); + + } finally { + org.apache.commons.io.FileUtils.deleteQuietly(directory); + } + } } http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java index e016b04..9701c7d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestOptiqPlans.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -142,7 +142,7 @@ public class TestOptiqPlans extends ExecTest { System.out.println(String.format("Got %d results", b.getHeader().getRowCount())); loader.load(b.getHeader().getDef(), b.getData()); for (final VectorWrapper<?> vw : loader) { - System.out.println(vw.getValueVector().getField().getPath()); + System.out.println(vw.getValueVector().getField().getName()); final ValueVector vv = vw.getValueVector(); for (int i = 0; i < vv.getAccessor().getValueCount(); i++) { final Object o = vv.getAccessor().getObject(i); @@ -171,7 +171,7 @@ public class TestOptiqPlans extends ExecTest { System.out.println(String.format("Got %d results", b.getHeader().getRowCount())); loader.load(b.getHeader().getDef(), b.getData()); for (final VectorWrapper<?> vw : loader) { - System.out.println(vw.getValueVector().getField().getPath()); + System.out.println(vw.getValueVector().getField().getName()); final ValueVector vv = vw.getValueVector(); for (int i = 0; i < vv.getAccessor().getValueCount(); i++) { final Object o = vv.getAccessor().getObject(i); @@ -200,7 +200,7 @@ public class TestOptiqPlans extends ExecTest { System.out.println(String.format("Got %d results", b.getHeader().getRowCount())); loader.load(b.getHeader().getDef(), b.getData()); for (final VectorWrapper<?> vw : loader) { - System.out.println(vw.getValueVector().getField().getPath()); + System.out.println(vw.getValueVector().getField().getName()); final ValueVector vv = vw.getValueVector(); for (int i = 0; i < vv.getAccessor().getValueCount(); i++) { final Object o = vv.getAccessor().getObject(i); @@ -239,7 +239,7 @@ public class TestOptiqPlans extends ExecTest { System.out.println(String.format("Got %d results", b.getHeader().getRowCount())); loader.load(b.getHeader().getDef(), b.getData()); for (final VectorWrapper vw : loader) { - System.out.println(vw.getValueVector().getField().getPath()); + System.out.println(vw.getValueVector().getField().getName()); final ValueVector vv = vw.getValueVector(); for (int i = 0; i < vv.getAccessor().getValueCount(); i++) { final Object o = vv.getAccessor().getObject(i); @@ -278,7 +278,7 @@ public class TestOptiqPlans extends ExecTest { System.out.println(String.format("Got %d results", b.getHeader().getRowCount())); loader.load(b.getHeader().getDef(), b.getData()); for (final VectorWrapper vw : loader) { - System.out.println(vw.getValueVector().getField().getPath()); + System.out.println(vw.getValueVector().getField().getName()); final ValueVector vv = vw.getValueVector(); for (int i = 0; i < vv.getAccessor().getValueCount(); i++) { final Object o = vv.getAccessor().getObject(i); http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java index 182e19e..3b8ab3f 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -46,7 +46,7 @@ public class TestSimpleFragmentRun extends PopUnitTestBase { public void runNoExchangeFragment() throws Exception { try (final RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet(); final Drillbit bit = new Drillbit(CONFIG, serviceSet); - final DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());) { + final DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) { // run query. bit.run(); @@ -72,7 +72,7 @@ public class TestSimpleFragmentRun extends PopUnitTestBase { } else { System.out.print("\t"); } - System.out.print(value.getField().getPath()); + System.out.print(value.getField().getName()); System.out.print("["); System.out.print(value.getField().getType().getMinorType()); System.out.print("]"); @@ -147,7 +147,7 @@ public class TestSimpleFragmentRun extends PopUnitTestBase { } else { System.out.print("\t"); } - System.out.print(v.getField().getPath()); + System.out.print(v.getField().getName()); System.out.print("["); System.out.print(v.getField().getType().getMinorType()); System.out.print("]"); http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinAdvanced.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinAdvanced.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinAdvanced.java index a70a3f8..16df3ac 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinAdvanced.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestHashJoinAdvanced.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -22,9 +22,12 @@ package org.apache.drill.exec.physical.impl.join; import org.apache.drill.BaseTestQuery; import org.junit.AfterClass; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; + public class TestHashJoinAdvanced extends BaseTestQuery { // Have to disable merge join, if this testcase is to test "HASH-JOIN". @@ -88,6 +91,7 @@ public class TestHashJoinAdvanced extends BaseTestQuery { .build() .run(); } + @Test public void testJoinWithDifferentTypesInCondition() throws Exception { String query = "select t1.full_name from cp.`employee.json` t1, cp.`department.json` t2 " + @@ -112,7 +116,7 @@ public class TestHashJoinAdvanced extends BaseTestQuery { .optionSettingQueriesForTestQuery("alter session set `planner.enable_hashjoin` = true") .unOrdered() .baselineColumns("bigint_col") - .baselineValues(1l) + .baselineValues(1L) .go(); query = "select count() col1 from " + @@ -123,7 +127,38 @@ public class TestHashJoinAdvanced extends BaseTestQuery { .sqlQuery(query) .unOrdered() .baselineColumns("col1") - .baselineValues(4l) + .baselineValues(4L) .go(); } + + @Test //DRILL-2197 Left Join with complex type in projection + public void testJoinWithMapAndDotField() throws Exception { + File directory = new File(BaseTestQuery.getTempDir("json/input")); + try { + directory.mkdirs(); + String fileName = "table.json"; + try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(directory, fileName)))) { + writer.write("{\"rk.q\": \"a\", \"m\": {\"a.b\":\"1\", \"a\":{\"b\":\"2\"}, \"c\":\"3\"}}"); + } + + String query = String.format("select t1.m.`a.b` as a,\n" + + "t2.m.a.b as b,\n" + + "t1.m['a.b'] as c,\n" + + "t2.rk.q as d,\n" + + "t1.`rk.q` as e\n" + + "from dfs_test.`%1$s/%2$s` t1,\n" + + "dfs_test.`%1$s/%2$s` t2\n" + + "where t1.m.`a.b`=t2.m.`a.b` and t1.m.a.b=t2.m.a.b", + directory.toPath().toString(), fileName); + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("a", "b", "c", "d", "e") + .baselineValues("1", "2", "1", null, "a") + .go(); + + } finally { + org.apache.commons.io.FileUtils.deleteQuietly(directory); + } + } } http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java index 53c0a67..8ba442d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/join/TestMergeJoin.java @@ -1,4 +1,4 @@ -/* +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -74,7 +74,7 @@ public class TestMergeJoin extends PopUnitTestBase { while (exec.next()) { totalRecordCount += exec.getRecordCount(); for (final ValueVector v : exec) { - System.out.print("[" + v.getField().getPath() + "] "); + System.out.print("[" + v.getField().getName() + "] "); } System.out.println("\n"); for (int valueIdx = 0; valueIdx < exec.getRecordCount(); valueIdx++) { @@ -131,7 +131,7 @@ public class TestMergeJoin extends PopUnitTestBase { for (int valueIdx = 0; valueIdx < exec.getRecordCount(); valueIdx++) { final List<Object> row = Lists.newArrayList(); for (final ValueVector v : exec) { - row.add(v.getField().getPath() + ":" + v.getAccessor().getObject(valueIdx)); + row.add(v.getField().getName() + ":" + v.getAccessor().getObject(valueIdx)); } for (final Object cell : row) { if (cell == null) { @@ -182,7 +182,7 @@ public class TestMergeJoin extends PopUnitTestBase { for (int valueIdx = 0; valueIdx < exec.getRecordCount(); valueIdx++) { final List<Object> row = Lists.newArrayList(); for (final ValueVector v : exec) { - row.add(v.getField().getPath() + ":" + v.getAccessor().getObject(valueIdx)); + row.add(v.getField().getName() + ":" + v.getAccessor().getObject(valueIdx)); } for (final Object cell : row) { if (cell == null) { @@ -232,7 +232,7 @@ public class TestMergeJoin extends PopUnitTestBase { for (int valueIdx = 0; valueIdx < exec.getRecordCount(); valueIdx++) { final List<Object> row = Lists.newArrayList(); for (final ValueVector v : exec) { - row.add(v.getField().getPath() + ":" + v.getAccessor().getObject(valueIdx)); + row.add(v.getField().getName() + ":" + v.getAccessor().getObject(valueIdx)); } for (final Object cell : row) { if (cell == null) { http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/mergereceiver/TestMergingReceiver.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/mergereceiver/TestMergingReceiver.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/mergereceiver/TestMergingReceiver.java index e4a96bd..6fb7f72 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/mergereceiver/TestMergingReceiver.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/mergereceiver/TestMergingReceiver.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -49,7 +49,7 @@ public class TestMergingReceiver extends PopUnitTestBase { try (final Drillbit bit1 = new Drillbit(CONFIG, serviceSet); final Drillbit bit2 = new Drillbit(CONFIG, serviceSet); - final DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());) { + final DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) { bit1.run(); bit2.run(); client.connect(); @@ -78,7 +78,7 @@ public class TestMergingReceiver extends PopUnitTestBase { try (final Drillbit bit1 = new Drillbit(CONFIG, serviceSet); final Drillbit bit2 = new Drillbit(CONFIG, serviceSet); - final DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());) { + final DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) { bit1.run(); bit2.run(); @@ -102,9 +102,9 @@ public class TestMergingReceiver extends PopUnitTestBase { final ValueVector.Accessor va = vv.getAccessor(); final MaterializedField materializedField = vv.getField(); final int numValues = va.getValueCount(); - for(int valueIdx = 0; valueIdx < numValues; ++valueIdx) { - if (materializedField.getPath().equals("blue")) { - final long longValue = ((Long) va.getObject(valueIdx)).longValue(); + for (int valueIdx = 0; valueIdx < numValues; ++valueIdx) { + if (materializedField.getName().equals("blue")) { + final long longValue = (Long) va.getObject(valueIdx); // check that order is ascending if (lastBlueValue != null) { assertTrue(longValue >= lastBlueValue); @@ -127,7 +127,7 @@ public class TestMergingReceiver extends PopUnitTestBase { try (final Drillbit bit1 = new Drillbit(CONFIG, serviceSet); final Drillbit bit2 = new Drillbit(CONFIG, serviceSet); - final DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());) { + final DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator())) { bit1.run(); bit2.run(); http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/MiniPlanUnitTestBase.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/MiniPlanUnitTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/MiniPlanUnitTestBase.java index 302d0e5..4f0fcbf 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/MiniPlanUnitTestBase.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/unit/MiniPlanUnitTestBase.java @@ -23,14 +23,12 @@ import com.google.common.collect.Lists; import mockit.NonStrictExpectations; import org.apache.drill.DrillTestWrapper; import org.apache.drill.common.expression.SchemaPath; -import org.apache.drill.exec.physical.base.AbstractBase; import org.apache.drill.exec.physical.base.PhysicalOperator; import org.apache.drill.exec.physical.impl.BatchCreator; import org.apache.drill.exec.physical.impl.ScanBatch; import org.apache.drill.exec.record.BatchSchema; import org.apache.drill.exec.record.MaterializedField; import org.apache.drill.exec.record.RecordBatch; -import org.apache.drill.exec.record.VectorAccessible; import org.apache.drill.exec.rpc.NamedThreadFactory; import org.apache.drill.exec.store.RecordReader; import org.apache.drill.exec.store.dfs.DrillFileSystem; @@ -54,7 +52,6 @@ import java.util.concurrent.Executors; import static org.apache.drill.exec.physical.base.AbstractBase.INIT_ALLOCATION; import static org.apache.drill.exec.physical.base.AbstractBase.MAX_ALLOCATION; -import static org.apache.drill.exec.physical.unit.TestMiniPlan.fs; /** * A MiniPlanUnitTestBase extends PhysicalOpUnitTestBase, to construct MiniPlan (aka plan fragment). @@ -112,7 +109,7 @@ public class MiniPlanUnitTestBase extends PhysicalOpUnitTestBase { "Must supply the same number of baseline values as columns in expected schema."); for (MaterializedField field : expectedSchema) { - ret.put(SchemaPath.getSimplePath(field.getPath()).toExpr(), baselineValues[i]); + ret.put(SchemaPath.getSimplePath(field.getName()).toExpr(), baselineValues[i]); i++; } http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestMaterializedField.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestMaterializedField.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestMaterializedField.java index 3380a52..0ea552f 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestMaterializedField.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestMaterializedField.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -17,9 +17,9 @@ / package org.apache.drill.exec.record; -import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.Types; + import static org.junit.Assert.assertTrue; import org.junit.Before; @@ -71,7 +71,7 @@ public class TestMaterializedField { final MaterializedField clone = field.withPathAndType(path, type); - final boolean isPathEqual = path.equals(clone.getPath()); + final boolean isPathEqual = path.equals(clone.getName()); assertTrue("Cloned path does not match the original", isPathEqual); final boolean isTypeEqual = type.equals(clone.getType()); @@ -83,5 +83,4 @@ public class TestMaterializedField { } } - } http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java index 8ac7c45..0dc4149 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestLoad.java @@ -1,4 +1,4 @@ -/* +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -24,8 +24,6 @@ import io.netty.buffer.DrillBuf; import java.util.List; import org.apache.drill.common.config.DrillConfig; -import org.apache.drill.common.expression.ExpressionPosition; -import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.types.TypeProtos; import org.apache.drill.common.types.TypeProtos.MinorType; import org.apache.drill.common.types.Types; @@ -68,14 +66,14 @@ public class TestLoad extends ExecTest { final RecordBatchLoader batchLoader = new RecordBatchLoader(allocator); final ByteBuf[] byteBufs = writableBatch.getBuffers(); int bytes = 0; - for (int i = 0; i < byteBufs.length; i++) { - bytes += byteBufs[i].writerIndex(); + for (ByteBuf buf : byteBufs) { + bytes += buf.writerIndex(); } final DrillBuf byteBuf = allocator.buffer(bytes); int index = 0; - for (int i = 0; i < byteBufs.length; i++) { - byteBufs[i].readBytes(byteBuf, index, byteBufs[i].writerIndex()); - index += byteBufs[i].writerIndex(); + for (ByteBuf buf : byteBufs) { + buf.readBytes(byteBuf, index, buf.writerIndex()); + index += buf.writerIndex(); } byteBuf.writerIndex(bytes); @@ -88,7 +86,7 @@ public class TestLoad extends ExecTest { } else { System.out.print("\t"); } - System.out.print(v.getField().getPath()); + System.out.print(v.getField().getName()); System.out.print("["); System.out.print(v.getField().getType().getMinorType()); System.out.print("]"); http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java index 0c5ff49..4e553de 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestOutputMutator.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -66,7 +66,7 @@ public class TestOutputMutator implements OutputMutator, Iterable<VectorWrapper< List<ValueVector> vectors = Lists.newArrayList(); for (VectorWrapper w : container) { ValueVector vector = w.getValueVector(); - if (vector.getField().getPath().equals(schemaPath)) { + if (vector.getField().getName().equals(schemaPath.getRootSegmentPath())) { vectors.add(newVector); } else { vectors.add(w.getValueVector()); http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetResultListener.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetResultListener.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetResultListener.java index 8b9cd28..bd3a9e8 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetResultListener.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetResultListener.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -14,7 +14,7 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - *****************************************************************************/ + / package org.apache.drill.exec.store.parquet; import static org.junit.Assert.assertEquals; @@ -126,12 +126,12 @@ public class ParquetResultListener implements UserResultsListener { for (final VectorWrapper vw : batchLoader) { final ValueVector vv = vw.getValueVector(); - currentField = props.fields.get(vv.getField().getPath()); - if (!valuesChecked.containsKey(vv.getField().getPath())) { - valuesChecked.put(vv.getField().getPath(), 0); + currentField = props.fields.get(vv.getField().getName()); + if (!valuesChecked.containsKey(vv.getField().getName())) { + valuesChecked.put(vv.getField().getName(), 0); columnValCounter = 0; } else { - columnValCounter = valuesChecked.get(vv.getField().getPath()); + columnValCounter = valuesChecked.get(vv.getField().getName()); } printColumnMajor(vv); @@ -145,9 +145,9 @@ public class ParquetResultListener implements UserResultsListener { columnValCounter += vv.getAccessor().getValueCount(); } - valuesChecked.remove(vv.getField().getPath()); + valuesChecked.remove(vv.getField().getName()); assertEquals("Mismatched value count for vectors in the same batch.", valueCount, vv.getAccessor().getValueCount()); - valuesChecked.put(vv.getField().getPath(), columnValCounter); + valuesChecked.put(vv.getField().getName(), columnValCounter); } if (ParquetRecordReaderTest.VERBOSE_DEBUG){ @@ -184,7 +184,7 @@ public class ParquetResultListener implements UserResultsListener { public void printColumnMajor(ValueVector vv) { if (ParquetRecordReaderTest.VERBOSE_DEBUG){ - System.out.println("\n" + vv.getField().getPath()); + System.out.println("\n" + vv.getField().getName()); } for (int j = 0; j < vv.getAccessor().getValueCount(); j++) { if (ParquetRecordReaderTest.VERBOSE_DEBUG){ @@ -211,7 +211,7 @@ public class ParquetResultListener implements UserResultsListener { System.out.println(); for (VectorWrapper vw : batchLoader) { ValueVector v = vw.getValueVector(); - System.out.print(Strings.padStart(v.getField().getPath(), 20, ' ') + " "); + System.out.print(Strings.padStart(v.getField().getName(), 20, ' ') + " "); } System.out.println(); http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java index 6397ef7..301374f 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -18,7 +18,6 @@ package org.apache.drill.exec.store.parquet; import org.apache.drill.BaseTestQuery; -import org.apache.drill.exec.proto.UserBitShared; import org.junit.Test; public class TestParquetComplex extends BaseTestQuery { @@ -180,8 +179,11 @@ public class TestParquetComplex extends BaseTestQuery { @Test //DRILL-3533 public void notxistsField() throws Exception { - String query = String.format("select t.`marketing_info`.notexists as notexists, t.`marketing_info`.camp_id as id from %s t", DATAFILE); - String[] columns = {"notexists", "id"}; + String query = String.format("select t.`marketing_info`.notexists as notexists1,\n" + + "t.`marketing_info`.camp_id as id,\n" + + "t.`marketing_info.camp_id` as notexists2\n" + + "from %s t", DATAFILE); + String[] columns = {"notexists1", "id", "notexists2"}; testBuilder() .sqlQuery(query) .unOrdered() http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java index b6f1408..7578476 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetMetadataCache.java @@ -17,6 +17,7 @@ / package org.apache.drill.exec.store.parquet; +import com.google.common.io.Resources; import mockit.Mock; import mockit.MockUp; import mockit.integration.junit4.JMockit; @@ -465,36 +466,43 @@ public class TestParquetMetadataCache extends PlanTestBase { } @Test - public void testMetadataCacheAbsolutePaths() throws Exception { - final String absolutePathsMetadata = "absolute_paths_metadata"; - try { - test("use dfs_test.tmp"); - // creating two inner directories to leverage METADATA_DIRECTORIES_FILENAME metadata file as well - final String absolutePathsMetadataT1 = absolutePathsMetadata + "/t1"; - final String absolutePathsMetadataT2 = absolutePathsMetadata + "/t2"; - test("create table `%s` as select from cp.`tpch/nation.parquet`", absolutePathsMetadataT1); - test("create table `%s` as select * from cp.`tpch/nation.parquet`", absolutePathsMetadataT2); - copyMetaDataCacheToTempReplacingInternalPaths("parquet/metadata_with_absolute_path/" + - "metadata_directories_with_absolute_paths.requires_replace.txt", absolutePathsMetadata, Metadata.METADATA_DIRECTORIES_FILENAME); - copyMetaDataCacheToTempReplacingInternalPaths("parquet/metadata_with_absolute_path/" + - "metadata_table_with_absolute_paths.requires_replace.txt", absolutePathsMetadata, Metadata.METADATA_FILENAME); - copyMetaDataCacheToTempReplacingInternalPaths("parquet/metadata_with_absolute_path/" + - "metadata_table_with_absolute_paths_t1.requires_replace.txt", absolutePathsMetadataT1, Metadata.METADATA_FILENAME); - copyMetaDataCacheToTempReplacingInternalPaths("parquet/metadata_with_absolute_path/" + - "metadata_table_with_absolute_paths_t2.requires_replace.txt", absolutePathsMetadataT2, Metadata.METADATA_FILENAME); - String query = String.format("select * from %s", absolutePathsMetadata); - int expectedRowCount = 50; - int expectedNumFiles = 1; // point to selectionRoot since no pruning is done in this query - int actualRowCount = testSql(query); - assertEquals("An incorrect result was obtained while querying a table with metadata cache files", - expectedRowCount, actualRowCount); - String numFilesPattern = "numFiles=" + expectedNumFiles; - String usedMetaPattern = "usedMetadataFile=true"; - String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", getDfsTestTmpSchemaLocation(), absolutePathsMetadata); - PlanTestBase.testPlanMatchingPatterns(query, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern}, - new String[] {"Filter"}); - } finally { - test("drop table if exists %s", absolutePathsMetadata); + public void testOldMetadataVersions() throws Exception { + final String tablePath = "absolute_paths_metadata"; + String rootMetadataPath = new Path("parquet", "metadata_files_with_old_versions").toUri().getPath(); + // gets folders with different metadata cache versions + String[] metadataPaths = new File(Resources.getResource(rootMetadataPath).getFile()).list(); + for (String metadataPath : metadataPaths) { + try { + test("use dfs_test.tmp"); + // creating two inner directories to leverage METADATA_DIRECTORIES_FILENAME metadata file as well + final String absolutePathsMetadataT1 = new Path(tablePath, "t1").toUri().getPath(); + final String absolutePathsMetadataT2 = new Path(tablePath, "t2").toUri().getPath(); + String createQuery = "create table `%s` as select * from cp.`tpch/nation.parquet`"; + test(createQuery, absolutePathsMetadataT1); + test(createQuery, absolutePathsMetadataT2); + Path relativePath = new Path(rootMetadataPath, metadataPath); + copyMetaDataCacheToTempReplacingInternalPaths(new Path(relativePath, "metadata_directories.requires_replace.txt"), + tablePath, Metadata.METADATA_DIRECTORIES_FILENAME); + copyMetaDataCacheToTempReplacingInternalPaths(new Path(relativePath, "metadata_table.requires_replace.txt"), + tablePath, Metadata.METADATA_FILENAME); + copyMetaDataCacheToTempReplacingInternalPaths(new Path(relativePath, "metadata_table_t1.requires_replace.txt"), + absolutePathsMetadataT1, Metadata.METADATA_FILENAME); + copyMetaDataCacheToTempReplacingInternalPaths(new Path(relativePath, "metadata_table_t2.requires_replace.txt"), + absolutePathsMetadataT2, Metadata.METADATA_FILENAME); + String query = String.format("select * from %s", tablePath); + int expectedRowCount = 50; + int expectedNumFiles = 1; // point to selectionRoot since no pruning is done in this query + int actualRowCount = testSql(query); + assertEquals("An incorrect result was obtained while querying a table with metadata cache files", + expectedRowCount, actualRowCount); + String numFilesPattern = "numFiles=" + expectedNumFiles; + String usedMetaPattern = "usedMetadataFile=true"; + String cacheFileRootPattern = String.format("cacheFileRoot=%s/%s", getDfsTestTmpSchemaLocation(), tablePath); + PlanTestBase.testPlanMatchingPatterns(query, new String[]{numFilesPattern, usedMetaPattern, cacheFileRootPattern}, + new String[]{"Filter"}); + } finally { + test("drop table if exists %s", tablePath); + } } } @@ -681,6 +689,31 @@ public class TestParquetMetadataCache extends PlanTestBase { } } + @Test // DRILL-4264 + public void testMetadataCacheFieldWithDots() throws Exception { + final String tableWithDots = "dfs_test.tmp.`complex_table`"; + try { + test("create table %s as\n" + + "select cast(1 as int) as `column.with.dots`, t.`column`.`with.dots`\n" + + "from cp.`store/parquet/complex/complex.parquet` t limit 1", tableWithDots); + + String query = String.format("select * from %s", tableWithDots); + int expectedRowCount = 1; + + int actualRowCount = testSql(query); + assertEquals("Row count does not match the expected value", expectedRowCount, actualRowCount); + PlanTestBase.testPlanMatchingPatterns(query, new String[]{"usedMetadataFile=false"}, null); + + test("refresh table metadata %s", tableWithDots); + + actualRowCount = testSql(query); + assertEquals("Row count does not match the expected value", expectedRowCount, actualRowCount); + PlanTestBase.testPlanMatchingPatterns(query, new String[]{"usedMetadataFile=true"}, null); + } finally { + test(String.format("drop table if exists %s", tableWithDots)); + } + } + /** * Helper method for checking the metadata file existence * http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetPhysicalPlan.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetPhysicalPlan.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetPhysicalPlan.java index 8714b30..e023ecb 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetPhysicalPlan.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetPhysicalPlan.java @@ -1,4 +1,4 @@ -/** +/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -54,7 +54,7 @@ public class TestParquetPhysicalPlan extends ExecTest { RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet(); DrillConfig config = DrillConfig.create(); - try (Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator());) { + try (Drillbit bit1 = new Drillbit(config, serviceSet); DrillClient client = new DrillClient(config, serviceSet.getCoordinator())) { bit1.run(); client.connect(); List<QueryDataBatch> results = client.runQuery(org.apache.drill.exec.proto.UserBitShared.QueryType.PHYSICAL, Resources.toString(Resources.getResource(fileName),Charsets.UTF_8)); @@ -65,7 +65,7 @@ public class TestParquetPhysicalPlan extends ExecTest { count += b.getHeader().getRowCount(); loader.load(b.getHeader().getDef(), b.getData()); for (VectorWrapper vw : loader) { - System.out.print(vw.getValueVector().getField().getPath() + ": "); + System.out.print(vw.getValueVector().getField().getName() + ": "); ValueVector vv = vw.getValueVector(); for (int i = 0; i < vv.getAccessor().getValueCount(); i++) { Object o = vv.getAccessor().getObject(i); @@ -124,7 +124,7 @@ public class TestParquetPhysicalPlan extends ExecTest { public void testParseParquetPhysicalPlanRemote() throws Exception { DrillConfig config = DrillConfig.create(); - try(DrillClient client = new DrillClient(config);) { + try (DrillClient client = new DrillClient(config)) { client.connect(); ParquetResultsListener listener = new ParquetResultsListener(); Stopwatch watch = Stopwatch.createStarted(); http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java index 7c0b345..9064c5c 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java @@ -740,4 +740,33 @@ public class TestJsonReader extends BaseTestQuery { org.apache.commons.io.FileUtils.deleteQuietly(directory); } } + + @Test // DRILL-4264 + public void testFieldWithDots() throws Exception { + File directory = new File(BaseTestQuery.getTempDir("json/input")); + try { + directory.mkdirs(); + String fileName = "table.json"; + try (BufferedWriter writer = new BufferedWriter(new FileWriter(new File(directory, fileName)))) { + writer.write("{\"rk.q\": \"a\", \"m\": {\"a.b\":\"1\", \"a\":{\"b\":\"2\"}, \"c\":\"3\"}}"); + } + + String query = String.format("select t.m.`a.b` as a,\n" + + "t.m.a.b as b,\n" + + "t.m['a.b'] as c,\n" + + "t.rk.q as d,\n" + + "t.`rk.q` as e\n" + + "from dfs_test.`%s/%s` t", + directory.toPath().toString(), fileName); + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("a", "b", "c", "d", "e") + .baselineValues("1", "2", "1", null, "a") + .go(); + + } finally { + org.apache.commons.io.FileUtils.deleteQuietly(directory); + } + } } http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java index 601abb1..42a7e63 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/RowSetPrinter.java @@ -73,7 +73,7 @@ public class RowSetPrinter { if (i > 0) { out.print(", "); } - out.print(schema.column(i).getLastName()); + out.print(schema.column(i).getName()); } out.println(); } http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java index 03417ff..af35cdf 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/test/rowSet/test/RowSetTest.java @@ -50,46 +50,46 @@ public class RowSetTest extends SubOperatorTest { * Test a simple physical schema with no maps. / - @Test - public void testSchema() { - BatchSchema batchSchema = new SchemaBuilder() - .add("c", MinorType.INT) - .add("a", MinorType.INT, DataMode.REPEATED) - .addNullable("b", MinorType.VARCHAR) - .build(); - - assertEquals("c", batchSchema.getColumn(0).getName()); - assertEquals("a", batchSchema.getColumn(1).getName()); - assertEquals("b", batchSchema.getColumn(2).getName()); - - RowSetSchema schema = new RowSetSchema(batchSchema); - TupleSchema access = schema.hierarchicalAccess(); - assertEquals(3, access.count()); - - crossCheck(access, 0, "c", MinorType.INT); - assertEquals(DataMode.REQUIRED, access.column(0).getDataMode()); - assertEquals(DataMode.REQUIRED, access.column(0).getType().getMode()); - assertTrue(! access.column(0).isNullable()); - - crossCheck(access, 1, "a", MinorType.INT); - assertEquals(DataMode.REPEATED, access.column(1).getDataMode()); - assertEquals(DataMode.REPEATED, access.column(1).getType().getMode()); - assertTrue(! access.column(1).isNullable()); - - crossCheck(access, 2, "b", MinorType.VARCHAR); - assertEquals(MinorType.VARCHAR, access.column(2).getType().getMinorType()); - assertEquals(DataMode.OPTIONAL, access.column(2).getDataMode()); - assertEquals(DataMode.OPTIONAL, access.column(2).getType().getMode()); - assertTrue(access.column(2).isNullable()); - - // No maps: physical schema is the same as access schema. - - PhysicalSchema physical = schema.physical(); - assertEquals(3, physical.count()); - assertEquals("c", physical.column(0).field().getName()); - assertEquals("a", physical.column(1).field().getName()); - assertEquals("b", physical.column(2).field().getName()); - } +// @Test +// public void testSchema() { +// BatchSchema batchSchema = new SchemaBuilder() +// .add("c", MinorType.INT) +// .add("a", MinorType.INT, DataMode.REPEATED) +// .addNullable("b", MinorType.VARCHAR) +// .build(); +// +// assertEquals("c", batchSchema.getColumn(0).getName()); +// assertEquals("a", batchSchema.getColumn(1).getName()); +// assertEquals("b", batchSchema.getColumn(2).getName()); +// +// RowSetSchema schema = new RowSetSchema(batchSchema); +// TupleSchema access = schema.hierarchicalAccess(); +// assertEquals(3, access.count()); +// +// crossCheck(access, 0, "c", MinorType.INT); +// assertEquals(DataMode.REQUIRED, access.column(0).getDataMode()); +// assertEquals(DataMode.REQUIRED, access.column(0).getType().getMode()); +// assertTrue(! access.column(0).isNullable()); +// +// crossCheck(access, 1, "a", MinorType.INT); +// assertEquals(DataMode.REPEATED, access.column(1).getDataMode()); +// assertEquals(DataMode.REPEATED, access.column(1).getType().getMode()); +// assertTrue(! access.column(1).isNullable()); +// +// crossCheck(access, 2, "b", MinorType.VARCHAR); +// assertEquals(MinorType.VARCHAR, access.column(2).getType().getMinorType()); +// assertEquals(DataMode.OPTIONAL, access.column(2).getDataMode()); +// assertEquals(DataMode.OPTIONAL, access.column(2).getType().getMode()); +// assertTrue(access.column(2).isNullable()); +// +// // No maps: physical schema is the same as access schema. +// +// PhysicalSchema physical = schema.physical(); +// assertEquals(3, physical.count()); +// assertEquals("c", physical.column(0).field().getName()); +// assertEquals("a", physical.column(1).field().getName()); +// assertEquals("b", physical.column(2).field().getName()); +// } /* * Validate that the actual column metadata is as expected by @@ -102,89 +102,89 @@ public class RowSetTest extends SubOperatorTest { * @param type expected type / - public void crossCheck(TupleSchema schema, int index, String fullName, MinorType type) { - String name = null; - for (String part : Splitter.on(".").split(fullName)) { - name = part; - } - assertEquals(name, schema.column(index).getName()); - assertEquals(index, schema.columnIndex(fullName)); - assertSame(schema.column(index), schema.column(fullName)); - assertEquals(type, schema.column(index).getType().getMinorType()); - } +// public void crossCheck(TupleSchema schema, int index, String fullName, MinorType type) { +// String name = null; +// for (String part : Splitter.on(".").split(fullName)) { +// name = part; +// } +// assertEquals(name, schema.column(index).getName()); +// assertEquals(index, schema.columnIndex(fullName)); +// assertSame(schema.column(index), schema.column(fullName)); +// assertEquals(type, schema.column(index).getType().getMinorType()); +// } /* * Verify that a nested map schema works as expected. / - @Test - public void testMapSchema() { - BatchSchema batchSchema = new SchemaBuilder() - .add("c", MinorType.INT) - .addMap("a") - .addNullable("b", MinorType.VARCHAR) - .add("d", MinorType.INT) - .addMap("e") - .add("f", MinorType.VARCHAR) - .buildMap() - .add("g", MinorType.INT) - .buildMap() - .add("h", MinorType.BIGINT) - .build(); - - RowSetSchema schema = new RowSetSchema(batchSchema); - - // Access schema: flattened with maps removed - - FlattenedSchema access = schema.flatAccess(); - assertEquals(6, access.count()); - crossCheck(access, 0, "c", MinorType.INT); - crossCheck(access, 1, "a.b", MinorType.VARCHAR); - crossCheck(access, 2, "a.d", MinorType.INT); - crossCheck(access, 3, "a.e.f", MinorType.VARCHAR); - crossCheck(access, 4, "a.g", MinorType.INT); - crossCheck(access, 5, "h", MinorType.BIGINT); - - // Should have two maps. - - assertEquals(2, access.mapCount()); - assertEquals("a", access.map(0).getName()); - assertEquals("e", access.map(1).getName()); - assertEquals(0, access.mapIndex("a")); - assertEquals(1, access.mapIndex("a.e")); - - // Verify physical schema: should mirror the schema created above. - - PhysicalSchema physical = schema.physical(); - assertEquals(3, physical.count()); - assertEquals("c", physical.column(0).field().getName()); - assertEquals("c", physical.column(0).fullName()); - assertFalse(physical.column(0).isMap()); - assertNull(physical.column(0).mapSchema()); - - assertEquals("a", physical.column(1).field().getName()); - assertEquals("a", physical.column(1).fullName()); - assertTrue(physical.column(1).isMap()); - assertNotNull(physical.column(1).mapSchema()); - - assertEquals("h", physical.column(2).field().getName()); - assertEquals("h", physical.column(2).fullName()); - assertFalse(physical.column(2).isMap()); - assertNull(physical.column(2).mapSchema()); - - PhysicalSchema aSchema = physical.column(1).mapSchema(); - assertEquals(4, aSchema.count()); - assertEquals("b", aSchema.column(0).field().getName()); - assertEquals("a.b", aSchema.column(0).fullName()); - assertEquals("d", aSchema.column(1).field().getName()); - assertEquals("e", aSchema.column(2).field().getName()); - assertEquals("g", aSchema.column(3).field().getName()); - - PhysicalSchema eSchema = aSchema.column(2).mapSchema(); - assertEquals(1, eSchema.count()); - assertEquals("f", eSchema.column(0).field().getName()); - assertEquals("a.e.f", eSchema.column(0).fullName()); - } +// @Test +// public void testMapSchema() { +// BatchSchema batchSchema = new SchemaBuilder() +// .add("c", MinorType.INT) +// .addMap("a") +// .addNullable("b", MinorType.VARCHAR) +// .add("d", MinorType.INT) +// .addMap("e") +// .add("f", MinorType.VARCHAR) +// .buildMap() +// .add("g", MinorType.INT) +// .buildMap() +// .add("h", MinorType.BIGINT) +// .build(); +// +// RowSetSchema schema = new RowSetSchema(batchSchema); +// +// // Access schema: flattened with maps removed +// +// FlattenedSchema access = schema.flatAccess(); +// assertEquals(6, access.count()); +// crossCheck(access, 0, "c", MinorType.INT); +// crossCheck(access, 1, "a.b", MinorType.VARCHAR); +// crossCheck(access, 2, "a.d", MinorType.INT); +// crossCheck(access, 3, "a.e.f", MinorType.VARCHAR); +// crossCheck(access, 4, "a.g", MinorType.INT); +// crossCheck(access, 5, "h", MinorType.BIGINT); +// +// // Should have two maps. +// +// assertEquals(2, access.mapCount()); +// assertEquals("a", access.map(0).getName()); +// assertEquals("e", access.map(1).getName()); +// assertEquals(0, access.mapIndex("a")); +// assertEquals(1, access.mapIndex("a.e")); +// +// // Verify physical schema: should mirror the schema created above. +// +// PhysicalSchema physical = schema.physical(); +// assertEquals(3, physical.count()); +// assertEquals("c", physical.column(0).field().getName()); +// assertEquals("c", physical.column(0).fullName()); +// assertFalse(physical.column(0).isMap()); +// assertNull(physical.column(0).mapSchema()); +// +// assertEquals("a", physical.column(1).field().getName()); +// assertEquals("a", physical.column(1).fullName()); +// assertTrue(physical.column(1).isMap()); +// assertNotNull(physical.column(1).mapSchema()); +// +// assertEquals("h", physical.column(2).field().getName()); +// assertEquals("h", physical.column(2).fullName()); +// assertFalse(physical.column(2).isMap()); +// assertNull(physical.column(2).mapSchema()); +// +// PhysicalSchema aSchema = physical.column(1).mapSchema(); +// assertEquals(4, aSchema.count()); +// assertEquals("b", aSchema.column(0).field().getName()); +// assertEquals("a.b", aSchema.column(0).fullName()); +// assertEquals("d", aSchema.column(1).field().getName()); +// assertEquals("e", aSchema.column(2).field().getName()); +// assertEquals("g", aSchema.column(3).field().getName()); +// +// PhysicalSchema eSchema = aSchema.column(2).mapSchema(); +// assertEquals(1, eSchema.count()); +// assertEquals("f", eSchema.column(0).field().getName()); +// assertEquals("a.e.f", eSchema.column(0).fullName()); +// } /* * Verify that simple scalar (non-repeated) column readers @@ -348,33 +348,33 @@ public class RowSetTest extends SubOperatorTest { * Map fields are flattened into a logical schema. / - @Test - public void testMap() { - BatchSchema batchSchema = new SchemaBuilder() - .add("a", MinorType.INT) - .addMap("b") - .add("c", MinorType.INT) - .add("d", MinorType.INT) - .buildMap() - .build(); - SingleRowSet rs = fixture.rowSetBuilder(batchSchema) - .add(10, 20, 30) - .add(40, 50, 60) - .build(); - RowSetReader reader = rs.reader(); - assertTrue(reader.next()); - assertEquals(10, reader.column(0).getInt()); - assertEquals(20, reader.column(1).getInt()); - assertEquals(30, reader.column(2).getInt()); - assertEquals(10, reader.column("a").getInt()); - assertEquals(30, reader.column("b.d").getInt()); - assertTrue(reader.next()); - assertEquals(40, reader.column(0).getInt()); - assertEquals(50, reader.column(1).getInt()); - assertEquals(60, reader.column(2).getInt()); - assertFalse(reader.next()); - rs.clear(); - } +// @Test +// public void testMap() { +// BatchSchema batchSchema = new SchemaBuilder() +// .add("a", MinorType.INT) +// .addMap("b") +// .add("c", MinorType.INT) +// .add("d", MinorType.INT) +// .buildMap() +// .build(); +// SingleRowSet rs = fixture.rowSetBuilder(batchSchema) +// .add(10, 20, 30) +// .add(40, 50, 60) +// .build(); +// RowSetReader reader = rs.reader(); +// assertTrue(reader.next()); +// assertEquals(10, reader.column(0).getInt()); +// assertEquals(20, reader.column(1).getInt()); +// assertEquals(30, reader.column(2).getInt()); +// assertEquals(10, reader.column("a").getInt()); +// assertEquals(30, reader.column("b.d").getInt()); +// assertTrue(reader.next()); +// assertEquals(40, reader.column(0).getInt()); +// assertEquals(50, reader.column(1).getInt()); +// assertEquals(60, reader.column(2).getInt()); +// assertFalse(reader.next()); +// rs.clear(); +// } /* * Test an array of ints (as an example fixed-width type) http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_directories.requires_replace.txt ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_directories.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_directories.requires_replace.txt new file mode 100644 index 0000000..8a9989d --- /dev/null +++ b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_directories.requires_replace.txt @@ -0,0 +1,3 @@ +{ + "directories" : [ "file:REPLACED_IN_TEST/absolute_paths_metadata/t1", "file:REPLACED_IN_TEST/absolute_paths_metadata/t2" ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_table.requires_replace.txt ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_table.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_table.requires_replace.txt new file mode 100644 index 0000000..e3734f3 --- /dev/null +++ b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_table.requires_replace.txt @@ -0,0 +1,108 @@ +{ + "metadata_version" : "v3", + "columnTypeInfo" : { + "n_name" : { + "name" : [ "n_name" ], + "primitiveType" : "BINARY", + "originalType" : "UTF8", + "precision" : 0, + "scale" : 0, + "repetitionLevel" : 0, + "definitionLevel" : 0 + }, + "n_nationkey" : { + "name" : [ "n_nationkey" ], + "primitiveType" : "INT32", + "originalType" : null, + "precision" : 0, + "scale" : 0, + "repetitionLevel" : 0, + "definitionLevel" : 0 + }, + "n_regionkey" : { + "name" : [ "n_regionkey" ], + "primitiveType" : "INT32", + "originalType" : null, + "precision" : 0, + "scale" : 0, + "repetitionLevel" : 0, + "definitionLevel" : 0 + }, + "n_comment" : { + "name" : [ "n_comment" ], + "primitiveType" : "BINARY", + "originalType" : "UTF8", + "precision" : 0, + "scale" : 0, + "repetitionLevel" : 0, + "definitionLevel" : 0 + } + }, + "files" : [ { + "path" : "REPLACED_IN_TEST/absolute_paths_metadata/t1/0_0_0.parquet", + "length" : 2424, + "rowGroups" : [ { + "start" : 4, + "length" : 1802, + "rowCount" : 25, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "n_nationkey" ], + "minValue" : 0, + "maxValue" : 24, + "nulls" : 0 + }, { + "name" : [ "n_name" ], + "minValue" : "ALGERIA", + "maxValue" : "VIETNAM", + "nulls" : 0 + }, { + "name" : [ "n_regionkey" ], + "minValue" : 0, + "maxValue" : 4, + "nulls" : 0 + }, { + "name" : [ "n_comment" ], + "minValue" : " haggle. carefully final deposits detect slyly agai", + "maxValue" : "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be", + "nulls" : 0 + } ] + } ] + }, { + "path" : "REPLACED_IN_TEST/absolute_paths_metadata/t2/0_0_0.parquet", + "length" : 2424, + "rowGroups" : [ { + "start" : 4, + "length" : 1802, + "rowCount" : 25, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "n_nationkey" ], + "minValue" : 0, + "maxValue" : 24, + "nulls" : 0 + }, { + "name" : [ "n_name" ], + "minValue" : "ALGERIA", + "maxValue" : "VIETNAM", + "nulls" : 0 + }, { + "name" : [ "n_regionkey" ], + "minValue" : 0, + "maxValue" : 4, + "nulls" : 0 + }, { + "name" : [ "n_comment" ], + "minValue" : " haggle. carefully final deposits detect slyly agai", + "maxValue" : "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be", + "nulls" : 0 + } ] + } ] + } ], + "directories" : [ "file:REPLACED_IN_TEST/absolute_paths_metadata/t1", "file:REPLACED_IN_TEST/absolute_paths_metadata/t2" ], + "drillVersion" : "1.11.0" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_table_t1.requires_replace.txt ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_table_t1.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_table_t1.requires_replace.txt new file mode 100644 index 0000000..62a8c80 --- /dev/null +++ b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_table_t1.requires_replace.txt @@ -0,0 +1,76 @@ +{ + "metadata_version" : "v3", + "columnTypeInfo" : { + "n_name" : { + "name" : [ "n_name" ], + "primitiveType" : "BINARY", + "originalType" : "UTF8", + "precision" : 0, + "scale" : 0, + "repetitionLevel" : 0, + "definitionLevel" : 0 + }, + "n_nationkey" : { + "name" : [ "n_nationkey" ], + "primitiveType" : "INT32", + "originalType" : null, + "precision" : 0, + "scale" : 0, + "repetitionLevel" : 0, + "definitionLevel" : 0 + }, + "n_regionkey" : { + "name" : [ "n_regionkey" ], + "primitiveType" : "INT32", + "originalType" : null, + "precision" : 0, + "scale" : 0, + "repetitionLevel" : 0, + "definitionLevel" : 0 + }, + "n_comment" : { + "name" : [ "n_comment" ], + "primitiveType" : "BINARY", + "originalType" : "UTF8", + "precision" : 0, + "scale" : 0, + "repetitionLevel" : 0, + "definitionLevel" : 0 + } + }, + "files" : [ { + "path" : "REPLACED_IN_TEST/absolute_paths_metadata/t1/0_0_0.parquet", + "length" : 2424, + "rowGroups" : [ { + "start" : 4, + "length" : 1802, + "rowCount" : 25, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "n_nationkey" ], + "minValue" : 0, + "maxValue" : 24, + "nulls" : 0 + }, { + "name" : [ "n_name" ], + "minValue" : "ALGERIA", + "maxValue" : "VIETNAM", + "nulls" : 0 + }, { + "name" : [ "n_regionkey" ], + "minValue" : 0, + "maxValue" : 4, + "nulls" : 0 + }, { + "name" : [ "n_comment" ], + "minValue" : " haggle. carefully final deposits detect slyly agai", + "maxValue" : "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be", + "nulls" : 0 + } ] + } ] + } ], + "directories" : [ ], + "drillVersion" : "1.11.0" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_table_t2.requires_replace.txt ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_table_t2.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_table_t2.requires_replace.txt new file mode 100644 index 0000000..b70c8fa --- /dev/null +++ b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/V3/metadata_table_t2.requires_replace.txt @@ -0,0 +1,76 @@ +{ + "metadata_version" : "v3", + "columnTypeInfo" : { + "n_name" : { + "name" : [ "n_name" ], + "primitiveType" : "BINARY", + "originalType" : "UTF8", + "precision" : 0, + "scale" : 0, + "repetitionLevel" : 0, + "definitionLevel" : 0 + }, + "n_nationkey" : { + "name" : [ "n_nationkey" ], + "primitiveType" : "INT32", + "originalType" : null, + "precision" : 0, + "scale" : 0, + "repetitionLevel" : 0, + "definitionLevel" : 0 + }, + "n_regionkey" : { + "name" : [ "n_regionkey" ], + "primitiveType" : "INT32", + "originalType" : null, + "precision" : 0, + "scale" : 0, + "repetitionLevel" : 0, + "definitionLevel" : 0 + }, + "n_comment" : { + "name" : [ "n_comment" ], + "primitiveType" : "BINARY", + "originalType" : "UTF8", + "precision" : 0, + "scale" : 0, + "repetitionLevel" : 0, + "definitionLevel" : 0 + } + }, + "files" : [ { + "path" : "REPLACED_IN_TEST/absolute_paths_metadata/t2/0_0_0.parquet", + "length" : 2424, + "rowGroups" : [ { + "start" : 4, + "length" : 1802, + "rowCount" : 25, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "n_nationkey" ], + "minValue" : 0, + "maxValue" : 24, + "nulls" : 0 + }, { + "name" : [ "n_name" ], + "minValue" : "ALGERIA", + "maxValue" : "VIETNAM", + "nulls" : 0 + }, { + "name" : [ "n_regionkey" ], + "minValue" : 0, + "maxValue" : 4, + "nulls" : 0 + }, { + "name" : [ "n_comment" ], + "minValue" : " haggle. carefully final deposits detect slyly agai", + "maxValue" : "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be", + "nulls" : 0 + } ] + } ] + } ], + "directories" : [ ], + "drillVersion" : "1.11.0" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_directories.requires_replace.txt ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_directories.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_directories.requires_replace.txt new file mode 100644 index 0000000..ad40340 --- /dev/null +++ b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_directories.requires_replace.txt @@ -0,0 +1,3 @@ +{ + "directories" : [ "t2", "t1" ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_table.requires_replace.txt ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_table.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_table.requires_replace.txt new file mode 100644 index 0000000..6c84901 --- /dev/null +++ b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_table.requires_replace.txt @@ -0,0 +1,108 @@ +{ + "metadata_version" : "3.1", + "columnTypeInfo" : { + "n_name" : { + "name" : [ "n_name" ], + "primitiveType" : "BINARY", + "originalType" : "UTF8", + "precision" : 0, + "scale" : 0, + "repetitionLevel" : 0, + "definitionLevel" : 0 + }, + "n_nationkey" : { + "name" : [ "n_nationkey" ], + "primitiveType" : "INT32", + "originalType" : null, + "precision" : 0, + "scale" : 0, + "repetitionLevel" : 0, + "definitionLevel" : 0 + }, + "n_regionkey" : { + "name" : [ "n_regionkey" ], + "primitiveType" : "INT32", + "originalType" : null, + "precision" : 0, + "scale" : 0, + "repetitionLevel" : 0, + "definitionLevel" : 0 + }, + "n_comment" : { + "name" : [ "n_comment" ], + "primitiveType" : "BINARY", + "originalType" : "UTF8", + "precision" : 0, + "scale" : 0, + "repetitionLevel" : 0, + "definitionLevel" : 0 + } + }, + "files" : [ { + "path" : "t2/0_0_0.parquet", + "length" : 2424, + "rowGroups" : [ { + "start" : 4, + "length" : 1802, + "rowCount" : 25, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "n_nationkey" ], + "minValue" : 0, + "maxValue" : 24, + "nulls" : 0 + }, { + "name" : [ "n_name" ], + "minValue" : "ALGERIA", + "maxValue" : "VIETNAM", + "nulls" : 0 + }, { + "name" : [ "n_regionkey" ], + "minValue" : 0, + "maxValue" : 4, + "nulls" : 0 + }, { + "name" : [ "n_comment" ], + "minValue" : " haggle. carefully final deposits detect slyly agai", + "maxValue" : "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be", + "nulls" : 0 + } ] + } ] + }, { + "path" : "t1/0_0_0.parquet", + "length" : 2424, + "rowGroups" : [ { + "start" : 4, + "length" : 1802, + "rowCount" : 25, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "n_nationkey" ], + "minValue" : 0, + "maxValue" : 24, + "nulls" : 0 + }, { + "name" : [ "n_name" ], + "minValue" : "ALGERIA", + "maxValue" : "VIETNAM", + "nulls" : 0 + }, { + "name" : [ "n_regionkey" ], + "minValue" : 0, + "maxValue" : 4, + "nulls" : 0 + }, { + "name" : [ "n_comment" ], + "minValue" : " haggle. carefully final deposits detect slyly agai", + "maxValue" : "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be", + "nulls" : 0 + } ] + } ] + } ], + "directories" : [ "t2", "t1" ], + "drillVersion" : "1.12.0-SNAPSHOT" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/drill/blob/d105950a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_table_t1.requires_replace.txt ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_table_t1.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_table_t1.requires_replace.txt new file mode 100644 index 0000000..b611d13 --- /dev/null +++ b/exec/java-exec/src/test/resources/parquet/metadata_files_with_old_versions/v3_1/metadata_table_t1.requires_replace.txt @@ -0,0 +1,76 @@ +{ + "metadata_version" : "3.1", + "columnTypeInfo" : { + "n_name" : { + "name" : [ "n_name" ], + "primitiveType" : "BINARY", + "originalType" : "UTF8", + "precision" : 0, + "scale" : 0, + "repetitionLevel" : 0, + "definitionLevel" : 0 + }, + "n_nationkey" : { + "name" : [ "n_nationkey" ], + "primitiveType" : "INT32", + "originalType" : null, + "precision" : 0, + "scale" : 0, + "repetitionLevel" : 0, + "definitionLevel" : 0 + }, + "n_regionkey" : { + "name" : [ "n_regionkey" ], + "primitiveType" : "INT32", + "originalType" : null, + "precision" : 0, + "scale" : 0, + "repetitionLevel" : 0, + "definitionLevel" : 0 + }, + "n_comment" : { + "name" : [ "n_comment" ], + "primitiveType" : "BINARY", + "originalType" : "UTF8", + "precision" : 0, + "scale" : 0, + "repetitionLevel" : 0, + "definitionLevel" : 0 + } + }, + "files" : [ { + "path" : "0_0_0.parquet", + "length" : 2424, + "rowGroups" : [ { + "start" : 4, + "length" : 1802, + "rowCount" : 25, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "n_nationkey" ], + "minValue" : 0, + "maxValue" : 24, + "nulls" : 0 + }, { + "name" : [ "n_name" ], + "minValue" : "ALGERIA", + "maxValue" : "VIETNAM", + "nulls" : 0 + }, { + "name" : [ "n_regionkey" ], + "minValue" : 0, + "maxValue" : 4, + "nulls" : 0 + }, { + "name" : [ "n_comment" ], + "minValue" : " haggle. carefully final deposits detect slyly agai", + "maxValue" : "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be", + "nulls" : 0 + } ] + } ] + } ], + "directories" : [ ], + "drillVersion" : "1.12.0-SNAPSHOT" +} \ No newline at end of file
Mime	Unnamed text/plain (inline, 7-Bit, 89273 bytes)
	View raw message