Author: arvind
Date: Tue Mar 6 23:01:02 2012
New Revision: 1297782
URL: http://svn.apache.org/viewvc?rev=1297782&view=rev
Log:
SQOOP-352. Export of avro data fails on year values.
(Bilung Lee via Arvind Prabhakar)
Modified:
incubator/sqoop/trunk/src/java/org/apache/sqoop/hive/TableDefWriter.java
incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/ConnManager.java
incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/MySQLManager.java
incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/OracleManager.java
incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/SqlManager.java
incubator/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java
incubator/sqoop/trunk/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java
incubator/sqoop/trunk/src/java/org/apache/sqoop/orm/ClassWriter.java
incubator/sqoop/trunk/src/test/com/cloudera/sqoop/manager/MySQLCompatTest.java
Modified: incubator/sqoop/trunk/src/java/org/apache/sqoop/hive/TableDefWriter.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/hive/TableDefWriter.java?rev=1297782&r1=1297781&r2=1297782&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/hive/TableDefWriter.java (original)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/hive/TableDefWriter.java Tue Mar 6 23:01:02
2012
@@ -165,7 +165,7 @@ public class TableDefWriter {
Integer colType = columnTypes.get(col);
String hiveColType = userMapping.getProperty(col);
if (hiveColType == null) {
- hiveColType = connManager.toHiveType(colType);
+ hiveColType = connManager.toHiveType(col, colType);
}
if (null == hiveColType) {
throw new IOException("Hive does not support the SQL type for column "
Modified: incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/ConnManager.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/ConnManager.java?rev=1297782&r1=1297781&r2=1297782&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/ConnManager.java (original)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/ConnManager.java Tue Mar 6 23:01:02
2012
@@ -23,6 +23,7 @@ import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
+import java.sql.Types;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedHashSet;
@@ -31,10 +32,16 @@ import java.util.Map;
import java.util.Set;
import java.util.StringTokenizer;
+import org.apache.avro.Schema.Type;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.io.BytesWritable;
import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.hive.HiveTypes;
+import com.cloudera.sqoop.lib.BlobRef;
+import com.cloudera.sqoop.lib.ClobRef;
+import com.cloudera.sqoop.manager.SqlManager;
import com.cloudera.sqoop.util.ExportException;
import com.cloudera.sqoop.util.ImportException;
@@ -76,18 +83,154 @@ public abstract class ConnManager {
public abstract String getPrimaryKey(String tableName);
/**
+ * Resolve a database-specific type to the Java type that should contain it.
+ * @param sqlType sql type
+ * @return the name of a Java type to hold the sql datatype, or null if none.
+ */
+ public String toJavaType(int sqlType) {
+ // Mappings taken from:
+ // http://java.sun.com/j2se/1.3/docs/guide/jdbc/getstart/mapping.html
+ if (sqlType == Types.INTEGER) {
+ return "Integer";
+ } else if (sqlType == Types.VARCHAR) {
+ return "String";
+ } else if (sqlType == Types.CHAR) {
+ return "String";
+ } else if (sqlType == Types.LONGVARCHAR) {
+ return "String";
+ } else if (sqlType == Types.NVARCHAR) {
+ return "String";
+ } else if (sqlType == Types.NCHAR) {
+ return "String";
+ } else if (sqlType == Types.LONGNVARCHAR) {
+ return "String";
+ } else if (sqlType == Types.NUMERIC) {
+ return "java.math.BigDecimal";
+ } else if (sqlType == Types.DECIMAL) {
+ return "java.math.BigDecimal";
+ } else if (sqlType == Types.BIT) {
+ return "Boolean";
+ } else if (sqlType == Types.BOOLEAN) {
+ return "Boolean";
+ } else if (sqlType == Types.TINYINT) {
+ return "Integer";
+ } else if (sqlType == Types.SMALLINT) {
+ return "Integer";
+ } else if (sqlType == Types.BIGINT) {
+ return "Long";
+ } else if (sqlType == Types.REAL) {
+ return "Float";
+ } else if (sqlType == Types.FLOAT) {
+ return "Double";
+ } else if (sqlType == Types.DOUBLE) {
+ return "Double";
+ } else if (sqlType == Types.DATE) {
+ return "java.sql.Date";
+ } else if (sqlType == Types.TIME) {
+ return "java.sql.Time";
+ } else if (sqlType == Types.TIMESTAMP) {
+ return "java.sql.Timestamp";
+ } else if (sqlType == Types.BINARY
+ || sqlType == Types.VARBINARY) {
+ return BytesWritable.class.getName();
+ } else if (sqlType == Types.CLOB) {
+ return ClobRef.class.getName();
+ } else if (sqlType == Types.BLOB
+ || sqlType == Types.LONGVARBINARY) {
+ return BlobRef.class.getName();
+ } else {
+ // TODO(aaron): Support DISTINCT, ARRAY, STRUCT, REF, JAVA_OBJECT.
+ // Return null indicating database-specific manager should return a
+ // java data type if it can find one for any nonstandard type.
+ return null;
+ }
+ }
+
+ /**
+ * Resolve a database-specific type to Hive data type.
+ * @param sqlType sql type
+ * @return hive type
+ */
+ public String toHiveType(int sqlType) {
+ return HiveTypes.toHiveType(sqlType);
+ }
+
+ /**
+ * Resolve a database-specific type to Avro data type.
+ * @param sqlType sql type
+ * @return avro type
+ */
+ public Type toAvroType(int sqlType) {
+ switch (sqlType) {
+ case Types.TINYINT:
+ case Types.SMALLINT:
+ case Types.INTEGER:
+ return Type.INT;
+ case Types.BIGINT:
+ return Type.LONG;
+ case Types.BIT:
+ case Types.BOOLEAN:
+ return Type.BOOLEAN;
+ case Types.REAL:
+ return Type.FLOAT;
+ case Types.FLOAT:
+ case Types.DOUBLE:
+ return Type.DOUBLE;
+ case Types.NUMERIC:
+ case Types.DECIMAL:
+ return Type.STRING;
+ case Types.CHAR:
+ case Types.VARCHAR:
+ case Types.LONGVARCHAR:
+ case Types.LONGNVARCHAR:
+ case Types.NVARCHAR:
+ case Types.NCHAR:
+ return Type.STRING;
+ case Types.DATE:
+ case Types.TIME:
+ case Types.TIMESTAMP:
+ return Type.LONG;
+ case Types.BINARY:
+ case Types.VARBINARY:
+ return Type.BYTES;
+ default:
+ throw new IllegalArgumentException("Cannot convert SQL type "
+ + sqlType);
+ }
+ }
+
+ /**
* Return java type for SQL type.
+ * @param columnName column name
* @param sqlType sql type
* @return java type
*/
- public abstract String toJavaType(int sqlType);
+ public String toJavaType(String columnName, int sqlType) {
+ // ignore column name by default.
+ return toJavaType(sqlType);
+ }
/**
* Return hive type for SQL type.
+ * @param columnName column name
* @param sqlType sql type
* @return hive type
*/
- public abstract String toHiveType(int sqlType);
+ public String toHiveType(String columnName, int sqlType) {
+ // ignore column name by default.
+ return toHiveType(sqlType);
+ }
+
+ /**
+ * Return avro type for SQL type.
+ * @param columnName column name
+ * @param sqlType sql type
+ * @return avro type
+ */
+ public Type toAvroType(String columnName, int sqlType) {
+ // ignore column name by default.
+ return toAvroType(sqlType);
+ }
/**
* Return an unordered mapping from colname to sqltype for
@@ -126,6 +269,51 @@ public abstract class ConnManager {
}
/**
+ * Return an unordered mapping from colname to sql type name for
+ * all columns in a table.
+ */
+ public Map<String, String> getColumnTypeNamesForTable(String tableName) {
+ LOG.error("This database does not support column type names.");
+ return null;
+ }
+
+ /**
+ * Return an unordered mapping from colname to sql type name for
+ * all columns in a query.
+ */
+ public Map<String, String> getColumnTypeNamesForQuery(String query) {
+ LOG.error("This database does not support free-form query"
+ + " column type names.");
+ return null;
+ }
+
+ /**
+ * Return an unordered mapping from colname to sql type name for
+ * all columns in a table or query.
+ *
+ * @param tableName the name of the table
+ * @param sqlQuery the SQL query to use if tableName is null
+ */
+ public Map<String, String> getColumnTypeNames(String tableName,
+ String sqlQuery) {
+ Map<String, String> columnTypeNames;
+ if (null != tableName) {
+ // We're generating a class based on a table import.
+ columnTypeNames = getColumnTypeNamesForTable(tableName);
+ } else {
+ // This is based on an arbitrary query.
+ String query = sqlQuery;
+ if (query.indexOf(SqlManager.SUBSTITUTE_TOKEN) == -1) {
+ throw new RuntimeException("Query [" + query + "] must contain '"
+ + SqlManager.SUBSTITUTE_TOKEN + "' in WHERE clause.");
+ }
+
+ columnTypeNames = getColumnTypeNamesForQuery(query);
+ }
+ return columnTypeNames;
+ }
+
+ /**
* This method allows various connection managers to indicate if they support
* staging data for export jobs. The managers that do support this must
* override this method and return <tt>true</tt>.
Modified: incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/MySQLManager.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/MySQLManager.java?rev=1297782&r1=1297781&r2=1297782&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/MySQLManager.java (original)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/MySQLManager.java Tue Mar 6 23:01:02
2012
@@ -24,11 +24,14 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.sql.Types;
+import java.util.Map;
+import org.apache.avro.Schema.Type;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
import org.apache.hadoop.util.StringUtils;
+
import com.cloudera.sqoop.SqoopOptions;
import com.cloudera.sqoop.util.ImportException;
@@ -224,5 +227,50 @@ public class MySQLManager
protected String getSchemaQuery() {
return "SELECT SCHEMA()";
}
+
+ private Map<String, String> colTypeNames;
+ private static final int YEAR_TYPE_OVERWRITE = Types.SMALLINT;
+
+ @Override
+ public String toJavaType(String columnName, int sqlType) {
+ if (colTypeNames == null) {
+ colTypeNames = getColumnTypeNames(options.getTableName(),
+ options.getSqlQuery());
+ }
+
+ if ("YEAR".equalsIgnoreCase(colTypeNames.get(columnName))) {
+ sqlType = YEAR_TYPE_OVERWRITE;
+ }
+
+ return super.toJavaType(columnName, sqlType);
+ }
+
+ @Override
+ public String toHiveType(String columnName, int sqlType) {
+ if (colTypeNames == null) {
+ colTypeNames = getColumnTypeNames(options.getTableName(),
+ options.getSqlQuery());
+ }
+
+ if ("YEAR".equalsIgnoreCase(colTypeNames.get(columnName))) {
+ sqlType = YEAR_TYPE_OVERWRITE;
+ }
+
+ return super.toHiveType(columnName, sqlType);
+ }
+
+ @Override
+ public Type toAvroType(String columnName, int sqlType) {
+ if (colTypeNames == null) {
+ colTypeNames = getColumnTypeNames(options.getTableName(),
+ options.getSqlQuery());
+ }
+
+ if ("YEAR".equalsIgnoreCase(colTypeNames.get(columnName))) {
+ sqlType = YEAR_TYPE_OVERWRITE;
+ }
+
+ return super.toAvroType(columnName, sqlType);
+ }
}
Modified: incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/OracleManager.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/OracleManager.java?rev=1297782&r1=1297781&r2=1297782&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/OracleManager.java (original)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/OracleManager.java Tue Mar 6
23:01:02 2012
@@ -474,8 +474,8 @@ public class OracleManager
* @param sqlType
* @return the name of a Java type to hold the sql datatype, or null if none.
*/
- public String toJavaType(int sqlType) {
- String defaultJavaType = super.toJavaType(sqlType);
+ public String toJavaType(String columnName, int sqlType) {
+ String defaultJavaType = super.toJavaType(columnName, sqlType);
return (defaultJavaType == null) ? dbToJavaType(sqlType) : defaultJavaType;
}
@@ -511,8 +511,8 @@ public class OracleManager
* @param sqlType sql data type
* @return hive data type
*/
- public String toHiveType(int sqlType) {
- String defaultHiveType = super.toHiveType(sqlType);
+ public String toHiveType(String columnName, int sqlType) {
+ String defaultHiveType = super.toHiveType(columnName, sqlType);
return (defaultHiveType == null) ? dbToHiveType(sqlType) : defaultHiveType;
}
Modified: incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/SqlManager.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/SqlManager.java?rev=1297782&r1=1297781&r2=1297782&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/SqlManager.java (original)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/SqlManager.java Tue Mar 6 23:01:02
2012
@@ -22,14 +22,12 @@ import java.sql.Timestamp;
import com.cloudera.sqoop.SqoopOptions;
import com.cloudera.sqoop.hbase.HBaseUtil;
-import com.cloudera.sqoop.hive.HiveTypes;
-import com.cloudera.sqoop.lib.BlobRef;
-import com.cloudera.sqoop.lib.ClobRef;
import com.cloudera.sqoop.mapreduce.DataDrivenImportJob;
import com.cloudera.sqoop.mapreduce.HBaseImportJob;
import com.cloudera.sqoop.mapreduce.ImportJobBase;
import com.cloudera.sqoop.mapreduce.JdbcExportJob;
import com.cloudera.sqoop.mapreduce.JdbcUpdateExportJob;
+import com.cloudera.sqoop.mapreduce.db.DataDrivenDBInputFormat;
import com.cloudera.sqoop.util.ExportException;
import com.cloudera.sqoop.util.ImportException;
import com.cloudera.sqoop.util.ResultSetPrinter;
@@ -52,9 +50,7 @@ import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.util.StringUtils;
-import com.cloudera.sqoop.mapreduce.db.DataDrivenDBInputFormat;
/**
* ConnManager implementation for generic SQL-compliant database.
@@ -236,6 +232,61 @@ public abstract class SqlManager
}
@Override
+ public Map<String, String> getColumnTypeNamesForTable(String tableName) {
+ String stmt = getColTypesQuery(tableName);
+ return getColumnTypeNamesForRawQuery(stmt);
+ }
+
+ @Override
+ public Map<String, String> getColumnTypeNamesForQuery(String query) {
+ // Manipulate the query to return immediately, with zero rows.
+ String rawQuery = query.replace(SUBSTITUTE_TOKEN, " (1 = 0) ");
+ return getColumnTypeNamesForRawQuery(rawQuery);
+ }
+
+ protected Map<String, String> getColumnTypeNamesForRawQuery(String stmt) {
+ ResultSet results;
+ try {
+ results = execute(stmt);
+ } catch (SQLException sqlE) {
+ LOG.error("Error executing statement: " + sqlE.toString(), sqlE);
+ release();
+ return null;
+ }
+
+ try {
+ Map<String, String> colTypeNames = new HashMap<String, String>();
+
+ int cols = results.getMetaData().getColumnCount();
+ ResultSetMetaData metadata = results.getMetaData();
+ for (int i = 1; i < cols + 1; i++) {
+ String colTypeName = metadata.getColumnTypeName(i);
+
+ String colName = metadata.getColumnName(i);
+ if (colName == null || colName.equals("")) {
+ colName = metadata.getColumnLabel(i);
+ }
+
+ colTypeNames.put(colName, colTypeName);
+ }
+
+ return colTypeNames;
+ } catch (SQLException sqlException) {
+ LOG.error("Error reading from database: " + sqlException.toString());
+ return null;
+ } finally {
+ try {
+ results.close();
+ getConnection().commit();
+ } catch (SQLException sqlE) {
+ LOG.warn("SQLException closing ResultSet: " + sqlE.toString());
+ }
+
+ release();
+ }
+ }
+
+ @Override
public ResultSet readTable(String tableName, String[] columns)
throws SQLException {
if (columns == null) {
@@ -496,79 +547,6 @@ public abstract class SqlManager
return execute(stmt, options.getFetchSize(), args);
}
- /**
- * Resolve a database-specific type to the Java type that should contain it.
- * @param sqlType
- * @return the name of a Java type to hold the sql datatype, or null if none.
- */
- public String toJavaType(int sqlType) {
- // Mappings taken from:
- // http://java.sun.com/j2se/1.3/docs/guide/jdbc/getstart/mapping.html
- if (sqlType == Types.INTEGER) {
- return "Integer";
- } else if (sqlType == Types.VARCHAR) {
- return "String";
- } else if (sqlType == Types.CHAR) {
- return "String";
- } else if (sqlType == Types.LONGVARCHAR) {
- return "String";
- } else if (sqlType == Types.NVARCHAR) {
- return "String";
- } else if (sqlType == Types.NCHAR) {
- return "String";
- } else if (sqlType == Types.LONGNVARCHAR) {
- return "String";
- } else if (sqlType == Types.NUMERIC) {
- return "java.math.BigDecimal";
- } else if (sqlType == Types.DECIMAL) {
- return "java.math.BigDecimal";
- } else if (sqlType == Types.BIT) {
- return "Boolean";
- } else if (sqlType == Types.BOOLEAN) {
- return "Boolean";
- } else if (sqlType == Types.TINYINT) {
- return "Integer";
- } else if (sqlType == Types.SMALLINT) {
- return "Integer";
- } else if (sqlType == Types.BIGINT) {
- return "Long";
- } else if (sqlType == Types.REAL) {
- return "Float";
- } else if (sqlType == Types.FLOAT) {
- return "Double";
- } else if (sqlType == Types.DOUBLE) {
- return "Double";
- } else if (sqlType == Types.DATE) {
- return "java.sql.Date";
- } else if (sqlType == Types.TIME) {
- return "java.sql.Time";
- } else if (sqlType == Types.TIMESTAMP) {
- return "java.sql.Timestamp";
- } else if (sqlType == Types.BINARY
- || sqlType == Types.VARBINARY) {
- return BytesWritable.class.getName();
- } else if (sqlType == Types.CLOB) {
- return ClobRef.class.getName();
- } else if (sqlType == Types.BLOB
- || sqlType == Types.LONGVARBINARY) {
- return BlobRef.class.getName();
- } else {
- // TODO(aaron): Support DISTINCT, ARRAY, STRUCT, REF, JAVA_OBJECT.
- // Return null indicating database-specific manager should return a
- // java data type if it can find one for any nonstandard type.
- return null;
- }
- }
-
- /**
- * Resolve a database-specific type to Hive data type.
- * @param sqlType sql type
- * @return hive type
- */
- public String toHiveType(int sqlType) {
- return HiveTypes.toHiveType(sqlType);
- }
-
public void close() throws SQLException {
release();
}
Modified: incubator/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java?rev=1297782&r1=1297781&r2=1297782&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java (original)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java Tue Mar
6 23:01:02 2012
@@ -73,7 +73,8 @@ public class JdbcExportJob extends Expor
MapWritable columnTypes = new MapWritable();
for (Map.Entry<String, Integer> e : columnTypeInts.entrySet()) {
Text columnName = new Text(e.getKey());
- Text columnText = new Text(connManager.toJavaType(e.getValue()));
+ Text columnText = new Text(
+ connManager.toJavaType(e.getKey(), e.getValue()));
columnTypes.put(columnName, columnText);
}
DefaultStringifier.store(job.getConfiguration(), columnTypes,
Modified: incubator/sqoop/trunk/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java?rev=1297782&r1=1297781&r2=1297782&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java (original)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java Tue Mar
6 23:01:02 2012
@@ -19,24 +19,28 @@
package org.apache.sqoop.orm;
import java.io.IOException;
-import java.sql.Types;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import org.apache.avro.Schema;
import org.apache.avro.Schema.Field;
import org.apache.avro.Schema.Type;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import com.cloudera.sqoop.SqoopOptions;
import com.cloudera.sqoop.manager.ConnManager;
-import java.util.Properties;
/**
* Creates an Avro schema to represent a table from a database.
*/
public class AvroSchemaGenerator {
+ public static final Log LOG =
+ LogFactory.getLog(AvroSchemaGenerator.class.getName());
+
private final SqoopOptions options;
private final ConnManager connManager;
private final String tableName;
@@ -78,58 +82,6 @@ public class AvroSchemaGenerator {
return schema;
}
- private Type toAvroType(int sqlType) {
- switch (sqlType) {
- case Types.TINYINT:
- case Types.SMALLINT:
- case Types.INTEGER:
- return Type.INT;
- case Types.BIGINT:
- return Type.LONG;
- case Types.BIT:
- case Types.BOOLEAN:
- return Type.BOOLEAN;
- case Types.REAL:
- return Type.FLOAT;
- case Types.FLOAT:
- case Types.DOUBLE:
- return Type.DOUBLE;
- case Types.NUMERIC:
- case Types.DECIMAL:
- return Type.STRING;
- case Types.CHAR:
- case Types.VARCHAR:
- case Types.LONGVARCHAR:
- case Types.LONGNVARCHAR:
- case Types.NVARCHAR:
- case Types.NCHAR:
- return Type.STRING;
- case Types.DATE:
- case Types.TIME:
- case Types.TIMESTAMP:
- return Type.LONG;
- case Types.BINARY:
- case Types.VARBINARY:
- return Type.BYTES;
- default:
- throw new IllegalArgumentException("Cannot convert SQL type "
- + sqlType);
- }
- }
-
- private Type toAvroType(String type) {
- if (type.equalsIgnoreCase("INTEGER")) { return Type.INT; }
- if (type.equalsIgnoreCase("LONG")) { return Type.LONG; }
- if (type.equalsIgnoreCase("BOOLEAN")) { return Type.BOOLEAN; }
- if (type.equalsIgnoreCase("FLOAT")) { return Type.FLOAT; }
- if (type.equalsIgnoreCase("DOUBLE")) { return Type.DOUBLE; }
- if (type.equalsIgnoreCase("STRING")) { return Type.STRING; }
- if (type.equalsIgnoreCase("BYTES")) { return Type.BYTES; }
-
- // Mapping was not found
- throw new IllegalArgumentException("Cannot convert to AVRO type " + type);
- }
-
/**
* Will create union, because each type is assumed to be nullable.
*
@@ -138,18 +90,8 @@ public class AvroSchemaGenerator {
* @return Schema
*/
public Schema toAvroSchema(int sqlType, String columnName) {
- Properties mappingJava = options.getMapColumnJava();
-
- // Try to apply any user specified mapping
- Type targetType;
- if (columnName != null && mappingJava.containsKey(columnName)) {
- targetType = toAvroType((String)mappingJava.get(columnName));
- } else {
- targetType = toAvroType(sqlType);
- }
-
List<Schema> childSchemas = new ArrayList<Schema>();
- childSchemas.add(Schema.create(targetType));
+ childSchemas.add(Schema.create(toAvroType(columnName, sqlType)));
childSchemas.add(Schema.create(Schema.Type.NULL));
return Schema.createUnion(childSchemas);
}
@@ -157,4 +99,28 @@ public class AvroSchemaGenerator {
public Schema toAvroSchema(int sqlType) {
return toAvroSchema(sqlType, null);
}
+
+ private Type toAvroType(String columnName, int sqlType) {
+ Properties mapping = options.getMapColumnJava();
+
+ if (mapping.containsKey(columnName)) {
+ String type = mapping.getProperty(columnName);
+ if (LOG.isDebugEnabled()) {
+ LOG.info("Overriding type of column " + columnName + " to " + type);
+ }
+
+ if (type.equalsIgnoreCase("INTEGER")) { return Type.INT; }
+ if (type.equalsIgnoreCase("LONG")) { return Type.LONG; }
+ if (type.equalsIgnoreCase("BOOLEAN")) { return Type.BOOLEAN; }
+ if (type.equalsIgnoreCase("FLOAT")) { return Type.FLOAT; }
+ if (type.equalsIgnoreCase("DOUBLE")) { return Type.DOUBLE; }
+ if (type.equalsIgnoreCase("STRING")) { return Type.STRING; }
+ if (type.equalsIgnoreCase("BYTES")) { return Type.BYTES; }
+
+ // Mapping was not found
+ throw new IllegalArgumentException("Cannot convert to AVRO type " + type);
+ }
+
+ return connManager.toAvroType(columnName, sqlType);
+ }
}
Modified: incubator/sqoop/trunk/src/java/org/apache/sqoop/orm/ClassWriter.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/orm/ClassWriter.java?rev=1297782&r1=1297781&r2=1297782&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/orm/ClassWriter.java (original)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/orm/ClassWriter.java Tue Mar 6 23:01:02
2012
@@ -250,7 +250,7 @@ public class ClassWriter {
return type;
}
- return connManager.toJavaType(sqlType);
+ return connManager.toJavaType(columnName, sqlType);
}
/**
Modified: incubator/sqoop/trunk/src/test/com/cloudera/sqoop/manager/MySQLCompatTest.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/test/com/cloudera/sqoop/manager/MySQLCompatTest.java?rev=1297782&r1=1297781&r2=1297782&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/test/com/cloudera/sqoop/manager/MySQLCompatTest.java (original)
+++ incubator/sqoop/trunk/src/test/com/cloudera/sqoop/manager/MySQLCompatTest.java Tue Mar
6 23:01:02 2012
@@ -27,6 +27,8 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
import com.cloudera.sqoop.SqoopOptions;
import com.cloudera.sqoop.testutil.ManagerCompatTestCase;
@@ -189,5 +191,10 @@ public class MySQLCompatTest extends Man
protected String getDecimalDbOutput(String numAsInserted) {
return getNumericDbOutput(numAsInserted);
}
+
+ @Test
+ public void testYear() {
+ verifyType("YEAR", "2012", "2012-01-01", "2012");
+ }
}
|