Repository: incubator-carbondata
Updated Branches:
refs/heads/master bbb5919a6 -> ecf29472e
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d73f4bfe/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
index 4f8bf1a..f166025 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala
@@ -50,16 +50,16 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte
TimeStampGranularityConstants.TIME_GRAN_SEC.toString
)
CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "true")
+ sql("drop table if exists directDictionaryTable")
+ sql("drop table if exists directDictionaryTable_hive")
sql(
- "CREATE TABLE if not exists directDictionaryTable (empno int,doj Timestamp, " +
- "salary int) " +
+ "CREATE TABLE if not exists directDictionaryTable (empno int,doj Timestamp, salary
int) " +
"STORED BY 'org.apache.carbondata.format'"
)
sql(
- "CREATE TABLE if not exists directDictionaryTable_hive (empno int,doj Timestamp,
" +
- "salary int) " +
- "row format delimited fields terminated by ','"
+ "CREATE TABLE if not exists directDictionaryTable_hive (empno int,doj Timestamp,
salary int) " +
+ "row format delimited fields terminated by ','"
)
CarbonProperties.getInstance()
@@ -68,8 +68,8 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte
.getCanonicalPath
val csvFilePath = currentDirectory + "/src/test/resources/datasample.csv"
sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE directDictionaryTable
OPTIONS" +
- "('DELIMITER'= ',', 'QUOTECHAR'= '\"')");
- sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE directDictionaryTable_hive");
+ "('DELIMITER'= ',', 'QUOTECHAR'= '\"')")
+ sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE directDictionaryTable_hive")
} catch {
case x: Throwable => CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
@@ -97,8 +97,8 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte
test("test direct dictionary for not equals condition") {
checkAnswer(
- sql("select doj from directDictionaryTable where doj != '2016-04-14 15:00:09.0'"),
- Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0"))
+ sql("select doj from directDictionaryTable where doj != '2016-04-14 15:00:09'"),
+ Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09"))
)
)
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d73f4bfe/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
index 22678af..991b1bf 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala
@@ -88,8 +88,8 @@ class TimestampDataTypeDirectDictionaryWithNoDictTestCase extends QueryTest
with
)
}
-
- test("select doj from directDictionaryTable with greater than filter") {
+
+ test("select doj from directDictionaryTable with greater than filter") {
checkAnswer(
sql("select doj from directDictionaryTable where doj>'2016-03-14 15:00:09'"),
Seq(Row(Timestamp.valueOf("2016-04-14 15:00:09")))
@@ -99,7 +99,7 @@ class TimestampDataTypeDirectDictionaryWithNoDictTestCase extends QueryTest
with
override def afterAll {
- sql("drop table directDictionaryTable")
+ sql("drop table directDictionaryTable")
CarbonProperties.getInstance()
.addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy")
CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "false")
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d73f4bfe/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
index f1a9ccb..9b7359f 100644
--- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
+++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala
@@ -50,7 +50,7 @@ class TimestampDataTypeNullDataTest extends QueryTest with BeforeAndAfterAll
{
)
sql(
"""CREATE TABLE IF NOT EXISTS timestampTyeNullData
- (ID Int, date Timestamp, country String,
+ (ID Int, dateField Timestamp, country String,
name String, phonetype String, serialname String, salary Int)
STORED BY 'org.apache.carbondata.format'"""
)
@@ -68,18 +68,18 @@ class TimestampDataTypeNullDataTest extends QueryTest with BeforeAndAfterAll
{
}
}
- test("SELECT max(date) FROM timestampTyeNullData where date is not null") {
+ test("SELECT max(dateField) FROM timestampTyeNullData where dateField is not null") {
checkAnswer(
- sql("SELECT max(date) FROM timestampTyeNullData where date is not null"),
+ sql("SELECT max(dateField) FROM timestampTyeNullData where dateField is not null"),
Seq(Row(Timestamp.valueOf("2015-07-23 00:00:00.0"))
)
)
}
- test("SELECT * FROM timestampTyeNullData where date is null") {
- checkAnswer(
- sql("SELECT date FROM timestampTyeNullData where date is null"),
- Seq(Row(null)
- ))
+ test("SELECT * FROM timestampTyeNullData where dateField is null") {
+ checkAnswer(
+ sql("SELECT dateField FROM timestampTyeNullData where dateField is null"),
+ Seq(Row(null)
+ ))
}
override def afterAll {
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d73f4bfe/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
b/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
index 65d583c..1bfcdea 100644
--- a/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
+++ b/integration/spark2/src/main/java/org/apache/carbondata/spark/readsupport/SparkRowReadSupportImpl.java
@@ -39,11 +39,13 @@ public class SparkRowReadSupportImpl extends AbstractDictionaryDecodedReadSuppor
@Override public Row readRow(Object[] data) {
for (int i = 0; i < dictionaries.length; i++) {
- if (dictionaries[i] == null) {
+ if (dictionaries[i] == null && data[i] != null) {
if (carbonColumns[i].hasEncoding(Encoding.DIRECT_DICTIONARY)) {
//convert the long to timestamp in case of direct dictionary column
if (DataType.TIMESTAMP == carbonColumns[i].getDataType()) {
- data[i] = new Timestamp((long) data[i] / 1000);
+ data[i] = new Timestamp((long) data[i]);
+ } else if(DataType.DATE == carbonColumns[i].getDataType()) {
+ data[i] = new java.sql.Date((int) data[i]);
}
} else if(dataTypes[i].equals(DataType.INT)) {
data[i] = ((Long)(data[i])).intValue();
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d73f4bfe/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
index 2468962..5db5d14 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDataFrameWriter.scala
@@ -134,6 +134,7 @@ class CarbonDataFrameWriter(sqlContext: SQLContext, val dataFrame: DataFrame)
{
case DoubleType => CarbonType.DOUBLE.getName
case BooleanType => CarbonType.DOUBLE.getName
case TimestampType => CarbonType.TIMESTAMP.getName
+ case DateType => CarbonType.DATE.getName
case other => sys.error(s"unsupported type: $other")
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d73f4bfe/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
index 940c6d7..3b63021 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDictionaryDecoder.scala
@@ -104,6 +104,7 @@ case class CarbonDictionaryDecoder(
DecimalType(precision, scale)
}
case DataType.TIMESTAMP => TimestampType
+ case DataType.DATE => DateType
case DataType.STRUCT =>
CarbonMetastoreTypes
.toDataType(s"struct<${ relation.getStructChildren(carbonDimension.getColName)
}>")
@@ -261,6 +262,7 @@ class CarbonDecoderRDD(
DecimalType(precision, scale)
}
case DataType.TIMESTAMP => TimestampType
+ case DataType.DATE => DateType
case DataType.STRUCT =>
CarbonMetastoreTypes
.toDataType(s"struct<${ relation.getStructChildren(carbonDimension.getColName)
}>")
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d73f4bfe/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
index b14a95c..2ba8a03 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonSource.scala
@@ -102,12 +102,10 @@ class CarbonSource extends CreatableRelationProvider
private def createTableIfNotExists(sparkSession: SparkSession, parameters: Map[String,
String],
dataSchema: StructType): String = {
- val (dbName, tableName) = parameters.get("path") match {
- case Some(path) =>
- val p = path.split(File.separator)
- ("default", p(p.length - 1))
- case _ => throw new Exception("do not have dbname and tablename for carbon table")
- }
+
+ val dbName: String = parameters.getOrElse("dbName", CarbonCommonConstants.DATABASE_DEFAULT_NAME)
+ val tableName: String = parameters.getOrElse("tableName", "default_table")
+
try {
CarbonEnv.get.carbonMetastore.lookupRelation(Option(dbName), tableName)(sparkSession)
CarbonEnv.get.carbonMetastore.storePath + s"/$dbName/$tableName"
@@ -134,7 +132,8 @@ class CarbonSource extends CreatableRelationProvider
val cm = TableCreator.prepareTableModel(false, Option(dbName), tableName, fields,
Nil, map)
CreateTable(cm).run(sparkSession)
CarbonEnv.get.carbonMetastore.storePath + s"/$dbName/$tableName"
- case _ => throw new Exception("do not have dbname and tablename for carbon table")
+ case ex: Exception =>
+ throw new Exception("do not have dbname and tablename for carbon table", ex)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d73f4bfe/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
index e375710..1faaafa 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/TableCreator.scala
@@ -39,7 +39,7 @@ object TableCreator {
// detects whether double or decimal column is part of dictionary_exclude
def isStringAndTimestampColDictionaryExclude(columnDataType: String): Boolean = {
- val dataTypes = Array("string", "timestamp")
+ val dataTypes = Array("string", "timestamp", "date", "stringtype", "timestamptype", "datetype")
dataTypes.exists(x => x.equalsIgnoreCase(columnDataType))
}
@@ -47,7 +47,7 @@ object TableCreator {
def isDetectAsDimentionDatatype(dimensionDatatype: String): Boolean = {
val dimensionType =
Array("string", "stringtype", "array", "arraytype", "struct",
- "structtype", "timestamp", "timestamptype")
+ "structtype", "timestamp", "timestamptype", "date", "datetype")
dimensionType.exists(x => x.equalsIgnoreCase(dimensionDatatype))
}
@@ -111,7 +111,8 @@ object TableCreator {
fields.foreach(field => {
if (dictExcludeCols.toSeq.exists(x => x.equalsIgnoreCase(field.column))) {
- if (DataTypeUtil.getDataType(field.dataType.get.toUpperCase()) != DataType.TIMESTAMP)
{
+ if (DataTypeUtil.getDataType(field.dataType.get.toUpperCase()) != DataType.TIMESTAMP
&&
+ DataTypeUtil.getDataType(field.dataType.get.toUpperCase()) != DataType.DATE)
{
noDictionaryDims :+= field.column
}
dimFields += field
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/d73f4bfe/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
index 24601f4..9467804 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonMetastore.scala
@@ -609,6 +609,7 @@ object CarbonMetastoreTypes extends RegexParsers {
fixedDecimalType |
"decimal" ^^^ "decimal" ^^^ DecimalType(18, 2) |
"varchar\\((\\d+)\\)".r ^^^ StringType |
+ "date" ^^^ DateType |
"timestamp" ^^^ TimestampType
protected lazy val fixedDecimalType: Parser[DataType] =
@@ -668,6 +669,7 @@ object CarbonMetastoreTypes extends RegexParsers {
case BooleanType => "boolean"
case DecimalType() => "decimal"
case TimestampType => "timestamp"
+ case DateType => "date"
}
}
}
|