From commits-return-2004-apmail-sqoop-commits-archive=sqoop.apache.org@sqoop.apache.org Tue Jan 13 05:18:07 2015 Return-Path: X-Original-To: apmail-sqoop-commits-archive@www.apache.org Delivered-To: apmail-sqoop-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 9AD9217493 for ; Tue, 13 Jan 2015 05:18:07 +0000 (UTC) Received: (qmail 82254 invoked by uid 500); 13 Jan 2015 05:18:08 -0000 Delivered-To: apmail-sqoop-commits-archive@sqoop.apache.org Received: (qmail 82220 invoked by uid 500); 13 Jan 2015 05:18:08 -0000 Mailing-List: contact commits-help@sqoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@sqoop.apache.org Delivered-To: mailing list commits@sqoop.apache.org Received: (qmail 82211 invoked by uid 99); 13 Jan 2015 05:18:08 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 13 Jan 2015 05:18:08 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 578DBA05A23; Tue, 13 Jan 2015 05:18:08 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: abe@apache.org To: commits@sqoop.apache.org Message-Id: <6363f8d9020c42458843760e7ec76455@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: sqoop git commit: SQOOP-1936: Sqoop2: Sort by comparing IDF data in shuffle phase Date: Tue, 13 Jan 2015 05:18:08 +0000 (UTC) Repository: sqoop Updated Branches: refs/heads/sqoop2 2d54e26a0 -> 7631d2933 SQOOP-1936: Sqoop2: Sort by comparing IDF data in shuffle phase (Veena Basavaraj via Abraham Elmahrek) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/7631d293 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/7631d293 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/7631d293 Branch: refs/heads/sqoop2 Commit: 7631d293335855fd42a319c1fcb56530c27f78a4 Parents: 2d54e26 Author: Abraham Elmahrek Authored: Mon Jan 12 21:16:13 2015 -0800 Committer: Abraham Elmahrek Committed: Mon Jan 12 21:16:13 2015 -0800 ---------------------------------------------------------------------- .../connector/idf/CSVIntermediateDataFormat.java | 7 +++---- .../sqoop/connector/idf/IntermediateDataFormat.java | 16 +++++++++++++++- .../connector/idf/JSONIntermediateDataFormat.java | 4 ++++ .../idf/TestCSVIntermediateDataFormat.java | 3 +-- .../java/org/apache/sqoop/job/io/SqoopWritable.java | 4 ++-- .../org/apache/sqoop/job/io/TestSqoopWritable.java | 9 --------- 6 files changed, 25 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/7631d293/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java index 2af6acd..4870fae 100644 --- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java +++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/CSVIntermediateDataFormat.java @@ -58,8 +58,7 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat { */ @Override public String getCSVTextData() { - // TODO:SQOOP-1936 to enable schema validation after we use compareTo - return this.data; + return super.getData(); } /** @@ -81,14 +80,14 @@ public class CSVIntermediateDataFormat extends IntermediateDataFormat { if (csvStringArray == null) { return null; } + Column[] columns = schema.getColumnsArray(); - if (csvStringArray.length != schema.getColumnsArray().length) { + if (csvStringArray.length != columns.length) { throw new SqoopException(IntermediateDataFormatError.INTERMEDIATE_DATA_FORMAT_0001, "The data " + getCSVTextData() + " has the wrong number of fields."); } Object[] objectArray = new Object[csvStringArray.length]; - Column[] columns = schema.getColumnsArray(); for (int i = 0; i < csvStringArray.length; i++) { if (csvStringArray[i].equals(NULL_VALUE) && !columns[i].isNullable()) { throw new SqoopException(IntermediateDataFormatError.INTERMEDIATE_DATA_FORMAT_0005, http://git-wip-us.apache.org/repos/asf/sqoop/blob/7631d293/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java index 6063320..6f945c2 100644 --- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java +++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/IntermediateDataFormat.java @@ -44,10 +44,13 @@ import java.util.Set; * Any conversion to the format dictated by the corresponding data source from the native or CSV text format * has to be done by the connector themselves both in FROM and TO * + * NOTE: we cannot use the generic for comparable, since the comparison can be arbitrary for instance, + * purely based on text format * @param - Each data format may have a native representation of the * data, represented by the parameter. */ -public abstract class IntermediateDataFormat { +@SuppressWarnings("rawtypes") +public abstract class IntermediateDataFormat implements Comparable { protected volatile T data; @@ -203,4 +206,15 @@ public abstract class IntermediateDataFormat { return true; } + @Override + public String toString() { + return this.data.toString(); + } + + @Override + public int compareTo(Object o) { + IntermediateDataFormat idf = (IntermediateDataFormat) o; + return toString().compareTo(idf.toString()); + } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/sqoop/blob/7631d293/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/JSONIntermediateDataFormat.java ---------------------------------------------------------------------- diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/JSONIntermediateDataFormat.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/JSONIntermediateDataFormat.java index 3cfd356..c8df6e0 100644 --- a/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/JSONIntermediateDataFormat.java +++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/connector/idf/JSONIntermediateDataFormat.java @@ -419,4 +419,8 @@ public class JSONIntermediateDataFormat extends IntermediateDataFormat