From drill-commits-return-588-apmail-incubator-drill-commits-archive=incubator.apache.org@incubator.apache.org Wed Jul 30 05:27:15 2014 Return-Path: X-Original-To: apmail-incubator-drill-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-drill-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 3159C11C96 for ; Wed, 30 Jul 2014 05:27:15 +0000 (UTC) Received: (qmail 24247 invoked by uid 500); 30 Jul 2014 05:27:15 -0000 Delivered-To: apmail-incubator-drill-commits-archive@incubator.apache.org Received: (qmail 24195 invoked by uid 500); 30 Jul 2014 05:27:15 -0000 Mailing-List: contact drill-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: drill-commits@incubator.apache.org Delivered-To: mailing list drill-commits@incubator.apache.org Received: (qmail 24178 invoked by uid 99); 30 Jul 2014 05:27:15 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 30 Jul 2014 05:27:15 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id C940B950423; Wed, 30 Jul 2014 05:27:14 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jacques@apache.org To: drill-commits@incubator.apache.org Date: Wed, 30 Jul 2014 05:27:14 -0000 Message-Id: <29ce48d50f004e719e1c0f08b145aa46@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [01/14] git commit: DRILL-1195: Querying nested array elements in JSON returns only null values Repository: incubator-drill Updated Branches: refs/heads/master cfa0b639b -> 0d6befca0 DRILL-1195: Querying nested array elements in JSON returns only null values Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/a7440105 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/a7440105 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/a7440105 Branch: refs/heads/master Commit: a74401054907b1a924bdb1d819d50c6df7ea7046 Parents: cfa0b63 Author: Aditya Kishore Authored: Mon Jul 28 17:37:39 2014 -0700 Committer: Aditya Kishore Committed: Tue Jul 29 11:44:29 2014 -0700 ---------------------------------------------------------------------- .../vector/complex/AbstractContainerVector.java | 62 ++++++++++---------- .../exec/nested/TestNestedComplexSchema.java | 40 +++++++++++++ .../src/test/resources/nested/nested_1.json | 5 ++ .../src/test/resources/nested/nested_2.json | 5 ++ .../src/test/resources/nested/nested_3.json | 5 ++ 5 files changed, 87 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a7440105/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java index 4f7cf52..9667fd2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java @@ -42,67 +42,69 @@ public abstract class AbstractContainerVector implements ValueVector{ public abstract VectorWithOrdinal getVectorWithOrdinal(String name); - public TypedFieldId getFieldIdIfMatches(TypedFieldId.Builder builder, boolean addToBreadCrumb, PathSegment seg){ - if(seg == null){ + public TypedFieldId getFieldIdIfMatches(TypedFieldId.Builder builder, boolean addToBreadCrumb, PathSegment seg) { + if (seg == null) { if(addToBreadCrumb) builder.intermediateType(this.getField().getType()); return builder.finalType(this.getField().getType()).build(); } - if(seg.isArray()){ - - if(seg.isLastPath()){ - - //if(addToBreadCrumb) builder.intermediateType(this.getField().getType()); - + if (seg.isArray()) { + if (seg.isLastPath()) { return builder // .remainder(seg) // .withIndex() // .finalType(getLastPathType()) // .build(); - }else{ - if(addToBreadCrumb){ + } else { + if (addToBreadCrumb) { addToBreadCrumb = false; builder.remainder(seg); } // this is a complex array reference, which means it doesn't correspond directly to a vector by itself. seg = seg.getChild(); - } - - }else{ + } else { // name segment. } VectorWithOrdinal vord = getVectorWithOrdinal(seg.isArray() ? null : seg.getNameSegment().getPath()); - if(vord == null) return null; + if (vord == null) return null; ValueVector v = vord.vector; - - if(addToBreadCrumb){ - //builder.intermediateType(this.getField().getType()); + if (addToBreadCrumb) { builder.intermediateType(v.getField().getType()); builder.addId(vord.ordinal); } - if(v instanceof AbstractContainerVector){ + if (v instanceof AbstractContainerVector) { // we're looking for a multi path. AbstractContainerVector c = (AbstractContainerVector) v; return c.getFieldIdIfMatches(builder, addToBreadCrumb, seg.getChild()); - }else{ - if (seg != null && seg.isLastPath() && ! seg.isArray()) { + } else { + if (seg.isNamed()) { if(addToBreadCrumb) builder.intermediateType(v.getField().getType()); - return builder.finalType(v.getField().getType()).build(); - }else if(seg != null && seg.isLastPath() && seg.isArray()){ - //if(addToBreadCrumb) builder.intermediateType(v.getField().getType()); - return builder - .finalType(v.getField().getType().toBuilder().setMode(DataMode.OPTIONAL).build()) - .build(); - }else{ - logger.warn("You tried to request a complex type inside a scalar object or path or type is wrong."); - return null; + builder.finalType(v.getField().getType()); + } else { + builder.finalType(v.getField().getType().toBuilder().setMode(DataMode.OPTIONAL).build()); } - } + if (seg.isLastPath()) { + return builder.build(); + } else { + PathSegment child = seg.getChild(); + if (child.isLastPath() && child.isArray()) { + if (addToBreadCrumb) { + builder.remainder(child); + } + builder.withIndex(); + builder.finalType(v.getField().getType().toBuilder().setMode(DataMode.OPTIONAL).build()); + return builder.build(); + } else { + logger.warn("You tried to request a complex type inside a scalar object or path or type is wrong."); + return null; + } + } + } } private MajorType getLastPathType() { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a7440105/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestNestedComplexSchema.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestNestedComplexSchema.java b/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestNestedComplexSchema.java new file mode 100644 index 0000000..1865120 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestNestedComplexSchema.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.nested; + +import org.apache.drill.BaseTestQuery; +import org.junit.Test; + +public class TestNestedComplexSchema extends BaseTestQuery { + + @Test + public void testNested1() throws Exception { + test("select tbl.arrayval[0] from cp.`nested/nested_1.json` tbl"); + } + + @Test + public void testNested2() throws Exception { + test("select tbl.a.arrayval[0] from cp.`nested/nested_2.json` tbl"); + } + + @Test + public void testNested3() throws Exception { + test("select tbl.a.arrayval[0].val1[0] from cp.`nested/nested_3.json` tbl"); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a7440105/exec/java-exec/src/test/resources/nested/nested_1.json ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/nested/nested_1.json b/exec/java-exec/src/test/resources/nested/nested_1.json new file mode 100644 index 0000000..60f9414 --- /dev/null +++ b/exec/java-exec/src/test/resources/nested/nested_1.json @@ -0,0 +1,5 @@ +{"rownum":1, "arrayval": [ "a1", "a2", "a3" ]} +{"rownum":2, "arrayval": [ "b1", "b2", "b3" ]} +{"rownum":3, "arrayval": [ "c1", "c2", "c3" ]} +{"rownum":4, "arrayval": [ "d1", "d2", "d3" ]} +{"rownum":5, "arrayval": [ "e1", "e2", "e3" ]} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a7440105/exec/java-exec/src/test/resources/nested/nested_2.json ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/nested/nested_2.json b/exec/java-exec/src/test/resources/nested/nested_2.json new file mode 100644 index 0000000..9e1f584 --- /dev/null +++ b/exec/java-exec/src/test/resources/nested/nested_2.json @@ -0,0 +1,5 @@ +{"rownum":1, "a":{"arrayval": [ "a1", "a2", "a3" ]}} +{"rownum":2, "a":{"arrayval": [ "b1", "b2", "b3" ]}} +{"rownum":3, "a":{"arrayval": [ "c1", "c2", "c3" ]}} +{"rownum":4, "a":{"arrayval": [ "d1", "d2", "d3" ]}} +{"rownum":5, "a":{"arrayval": [ "e1", "e2", "e3" ]}} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a7440105/exec/java-exec/src/test/resources/nested/nested_3.json ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/nested/nested_3.json b/exec/java-exec/src/test/resources/nested/nested_3.json new file mode 100644 index 0000000..7bb244f --- /dev/null +++ b/exec/java-exec/src/test/resources/nested/nested_3.json @@ -0,0 +1,5 @@ +{"rownum":1, "a":{"arrayval": [ {"val1":["a1"]}, {"val2":"[a2]"}, {"val3":"[a3]"} ]}} +{"rownum":2, "a":{"arrayval": [ {"val1":["b1"]}, {"val2":"[b2]"}, {"val3":"[b3]"} ]}} +{"rownum":3, "a":{"arrayval": [ {"val1":["c1"]}, {"val2":"[c2]"}, {"val3":"[c3]"} ]}} +{"rownum":4, "a":{"arrayval": [ {"val1":["d1"]}, {"val2":"[d2]"}, {"val3":"[d3]"} ]}} +{"rownum":5, "a":{"arrayval": [ {"val1":["e1"]}, {"val2":"[e2]"}, {"val3":"[e3]"} ]}} \ No newline at end of file