carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject [2/3] incubator-carbondata git commit: Cast Push Down Optimization
Date Mon, 01 May 2017 01:08:14 GMT
Cast Push Down Optimization

Furthur Optimize Cast Expression with Early Evaluation


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/96e5b5cc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/96e5b5cc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/96e5b5cc

Branch: refs/heads/master
Commit: 96e5b5ccd9d712a577e4e4b291e1486293383be8
Parents: 472f9c9
Author: sounakr <sounakr@gmail.com>
Authored: Tue Apr 4 15:16:46 2017 +0530
Committer: ravipesala <ravi.pesala@gmail.com>
Committed: Mon May 1 06:36:53 2017 +0530

----------------------------------------------------------------------
 .../detailquery/CastColumnTestCase.scala        | 973 +++++++++++++++++++
 .../RangeFilterAllDataTypesTestCases.scala      | 131 ++-
 .../detailquery/RangeFilterTestCase.scala       |   4 +-
 .../apache/carbondata/spark/CarbonFilters.scala | 370 -------
 .../apache/spark/sql/CarbonBoundReference.scala |   7 +-
 .../spark/sql/optimizer/CarbonFilters.scala     | 373 +++++++
 .../apache/spark/sql/CarbonBoundReference.scala |  51 +
 .../scala/org/apache/spark/sql/CarbonScan.scala |  14 +-
 .../spark/sql/SparkUnknownExpression.scala      |  27 +-
 .../execution/CarbonLateDecodeStrategy.scala    |  41 +-
 .../execution/CastExpressionOptimization.scala  | 373 +++++++
 .../spark/sql/optimizer/CarbonFilters.scala     | 386 ++++++++
 .../carbondata/query/TestNotEqualToFilter.scala |   3 +
 13 files changed, 2340 insertions(+), 413 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96e5b5cc/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/CastColumnTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/CastColumnTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/CastColumnTestCase.scala
new file mode 100644
index 0000000..bc94350
--- /dev/null
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/CastColumnTestCase.scala
@@ -0,0 +1,973 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark.testsuite.detailquery
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.common.util.QueryTest
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants
+import org.apache.carbondata.core.util.CarbonProperties
+
+/**
+ * Test Class for Range Filters.
+ */
+class CastColumnTestCase extends QueryTest with BeforeAndAfterAll {
+
+  override def beforeAll {
+    //For the Hive table creation and data loading
+    sql("drop table if exists DICTIONARY_CARBON_1")
+    sql("drop table if exists DICTIONARY_HIVE_1")
+    sql("drop table if exists NO_DICTIONARY_CARBON_2")
+    sql("drop table if exists NO_DICTIONARY_HIVE_2")
+
+    //For Carbon cube creation.
+    sql("CREATE TABLE DICTIONARY_CARBON_1 (empno string, " +
+        "doj Timestamp, workgroupcategory Int, empname String,workgroupcategoryname String, " +
+        "deptno Int, deptname String, projectcode Int, projectjoindate Timestamp, " +
+        "projectenddate Timestamp, designation String,attendance Int,utilization " +
+        "Int,salary Int) STORED BY 'org.apache.carbondata.format' " +
+        "TBLPROPERTIES('DICTIONARY_INCLUDE'='empname,workgroupcategory, designation')"
+    )
+    sql(
+      s"LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE DICTIONARY_CARBON_1 " +
+      "OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+    )
+
+    sql(
+      "create table DICTIONARY_HIVE_1(empno string,empname string,designation string,doj " +
+      "Timestamp,workgroupcategory int, " +
+      "workgroupcategoryname string,deptno int, deptname string, projectcode int, " +
+      "projectjoindate Timestamp,projectenddate Timestamp,attendance int, "
+      + "utilization int,salary int) row format delimited fields terminated by ',' " +
+      "tblproperties(\"skip.header.line.count\"=\"1\") " +
+      ""
+    )
+
+    sql(
+      s"load data local inpath '$resourcesPath/datawithoutheader.csv' into table " +
+      "DICTIONARY_HIVE_1"
+    )
+
+
+    sql("CREATE TABLE NO_DICTIONARY_CARBON_2 (empno string, " +
+        "doj Timestamp, workgroupcategory Int, empname String,workgroupcategoryname String, " +
+        "deptno Int, deptname String, projectcode Int, projectjoindate Timestamp, " +
+        "projectenddate Timestamp, designation String,attendance Int,utilization " +
+        "Int,salary Int) STORED BY 'org.apache.carbondata.format' " +
+        "TBLPROPERTIES('DICTIONARY_INCLUDE'='workgroupcategory', 'DICTIONARY_EXCLUDE'='empno, designation')"
+    )
+    sql(
+      s"LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE NO_DICTIONARY_CARBON_2 " +
+      "OPTIONS('DELIMITER'= ',', 'QUOTECHAR'= '\"')"
+    )
+
+    sql(
+      "create table NO_DICTIONARY_HIVE_2(empno string,empname string,designation string,doj " +
+      "Timestamp,workgroupcategory int, " +
+      "workgroupcategoryname string,deptno int, deptname string, projectcode int, " +
+      "projectjoindate Timestamp,projectenddate Timestamp,attendance int, "
+      + "utilization int,salary int) row format delimited fields terminated by ',' " +
+      "tblproperties(\"skip.header.line.count\"=\"1\") " +
+      ""
+    )
+
+    sql(
+      s"load data local inpath '$resourcesPath/datawithoutheader.csv' into table " +
+      "NO_DICTIONARY_HIVE_2"
+    );
+
+    CarbonProperties.getInstance()
+      .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "yyyy-MM-dd HH:mm:ss")
+  }
+  test("Dictionary String ") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno = 11"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno = 11")
+    )
+  }
+
+  test("Dictionary String OR") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno = '11' or empno = '15'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno = '11' or empno = '15'")
+    )
+  }
+
+  test("Dictionary String OR Implicit Cast to int") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno = 11 or empno = 15"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno = 11 or empno = 15")
+    )
+  }
+
+  test("Dictionary String OR explicit Cast to int") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(empno as int) = 11 or cast(empno as int) = 15"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(empno as int) = 11 or cast (empno as int) = 15")
+    )
+  }
+
+  test("Dictionary INT OR to implicit String") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory = '1' or  workgroupcategory = '2'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory = '1' or  workgroupcategory = '2'")
+    )
+  }
+
+  test("Dictionary INT OR to excplicit String") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(workgroupcategory as string)= '1' or cast(workgroupcategory as string) = '2'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as string) = '1' or  cast(workgroupcategory as string) = '2'")
+    )
+  }
+
+  test("Dictionary INT OR to explicit double") {
+    sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(workgroupcategory as double)= 1.0 or cast(workgroupcategory as double) = 2.0").show(2000, false)
+    sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) = 1.0 or  cast(workgroupcategory as double) = 2.0 ").show(2000, false)
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(workgroupcategory as double)= 1.0 or cast(workgroupcategory as double) = 2.0"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) = 1.0 or  cast(workgroupcategory as double) = 2.0 ")
+    )
+  }
+
+  test("Dictionary String Not") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno != '11'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno != '11'")
+    )
+  }
+
+  test("Dictionary String Not Implicit Cast to int") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno != 11"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno != 11")
+    )
+  }
+
+  test("Dictionary String Not explicit Cast to int") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(empno as int) != 11"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(empno as int) != 11")
+    )
+  }
+
+  test("Dictionary INT Not to implicit String") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory != '1'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory != '1'")
+    )
+  }
+
+  test("Dictionary INT Not to excplicit String") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(workgroupcategory as string) != '1'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as string) != '1'")
+    )
+  }
+
+  test("Dictionary INT Not to explicit double") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(workgroupcategory as double) != 1.0"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) != 1.0")
+    )
+  }
+
+  test("Dictionary String In") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno in ('11', '15')"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno in ('11', '15')")
+    )
+  }
+
+  test("Dictionary String In Implicit ") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno in (11, 15)"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno in (11, 15)")
+    )
+  }
+
+  test("Dictionary String In Explicit ") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (empno as int) in (11, 15)"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast (empno as int) in (11, 15)")
+    )
+  }
+
+  test("Dictionary String In Explicit double") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (empno as double) in (11.0, 15.0)"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast (empno as double) in (11.0, 15.0)")
+    )
+  }
+
+  test("Dictionary String Not in Explicit double 1") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (empno as double)  not in (11.0, 15.0)"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast (empno as double) not in (11.0, 15.0)")
+    )
+  }
+
+  test("Dictionary INT In to implicit Int") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory in ('1', '2')"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory in ('1', '2')")
+    )
+  }
+
+  test("Dictionary INT In to excplicit String") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(workgroupcategory as string) in ('1', '2')"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as string) in ('1', '2')")
+    )
+  }
+
+  test("Dictionary INT In to explicit double") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(workgroupcategory as double) in (1.0, 2.0)"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) in (1.0, 2.0)")
+    )
+  }
+
+  test("Dictionary String Greater Than") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno > '11'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno > '11'")
+    )
+  }
+
+  test("Dictionary String Greater Than Implicit") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno > 11"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno > 11")
+    )
+  }
+
+  test("Dictionary String Greater Than explicit 1") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(empno as int) > '11'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast (empno as int) > '11'")
+    )
+  }
+
+
+  test("Dictionary String Greater Than explicit 2") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(empno as int) > 11"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast (empno as int) > 11")
+    )
+  }
+
+  test("Dictionary INT Greater Than") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory > 1"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory > 1")
+    )
+  }
+
+
+  test("Dictionary INT Greater Than implicit ") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory > '1'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory > '1'")
+    )
+  }
+
+
+  test("Dictionary INT Greater Than double") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (workgroupcategory as double) > 1.0"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) > 1.0")
+    )
+  }
+
+
+  test("Dictionary INT Greater Than String ") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (workgroupcategory as double) > '1.0'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) > '1.0'")
+    )
+  }
+
+
+  test("Dictionary String Greater Than equal") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno >= '11'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno >= '11'")
+    )
+  }
+
+  test("Dictionary String Greater Than Implicit equal") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno >= 11"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno >= 11")
+    )
+  }
+
+  test("Dictionary String Greater Than equal explicit 1") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(empno as int) >= '11'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast (empno as int) >= '11'")
+    )
+  }
+
+
+  test("Dictionary String Greater Than equal explicit 2") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(empno as int) >= 11"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast (empno as int) >= 11")
+    )
+  }
+
+  test("Dictionary INT Greater Than equal ") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory >= 1"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory >= 1")
+    )
+  }
+
+
+  test("Dictionary INT Greater Than equal implicit ") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory >= '1'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory >= '1'")
+    )
+  }
+
+
+  test("Dictionary INT Greater Than equal 2") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (workgroupcategory as double) >= 1.0"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) >= 1.0")
+    )
+  }
+
+
+  test("Dictionary INT Greater equal Than") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (workgroupcategory as double) >= '1.0'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) >= '1.0'")
+    )
+  }
+
+  test("Dictionary String less Than") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno < '20'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno < '20'")
+    )
+  }
+
+  test("Dictionary String Less Than Implicit") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno < 20"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno < 20")
+    )
+  }
+
+  test("Dictionary String Less Than explicit 1") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(empno as int) < '20'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast (empno as int) < '20'")
+    )
+  }
+
+
+  test("Dictionary String Less Than explicit 2") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(empno as int) < 20"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast (empno as int) < 20")
+    )
+  }
+
+  test("Dictionary INT Less Than") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory < 2"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory < 2")
+    )
+  }
+
+
+  test("Dictionary INT Less Than implicit ") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory < '2'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory < '2'")
+    )
+  }
+
+
+  test("Dictionary INT Less Than double") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (workgroupcategory as double) < 2.0"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) < 2.0")
+    )
+  }
+
+
+  test("Dictionary INT Less Than String ") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (workgroupcategory as double) < '2.0'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) < '2.0'")
+    )
+  }
+
+
+  test("Dictionary String Less Than equal") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno <= '20'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno <= '20'")
+    )
+  }
+
+  test("Dictionary String Less Than Implicit equal") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where empno <= 20"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where empno <= 20")
+    )
+  }
+
+  test("Dictionary String Less Than equal explicit 1") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(empno as int) <= '20'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast (empno as int) <= '20'")
+    )
+  }
+
+
+  test("Dictionary String Less Than equal explicit 2") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast(empno as int) <= 20"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast (empno as int) <= 20")
+    )
+  }
+
+  test("Dictionary INT Less Than equal ") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory <= 2"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory <= 2")
+    )
+  }
+
+
+  test("Dictionary INT Less Than equal implicit ") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory <= '2'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory <= '2'")
+    )
+  }
+
+
+  test("Dictionary INT Less Than equal 2") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (workgroupcategory as double) <= 2.0"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) <= 2.0")
+    )
+  }
+
+
+  test("Dictionary INT Less equal Than") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (workgroupcategory as double) <= '2.0'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) <= '2.0'")
+    )
+  }
+
+
+  test("Dictionary INT greater less Than explicit double") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (workgroupcategory as double) <= '2.0' and cast (workgroupcategory as double) >= '1.0'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as double) <= '2.0' and cast (workgroupcategory as double) >= '1.0'")
+    )
+  }
+
+
+  test("Dictionary INT greater less Than explicit string") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where cast (workgroupcategory as string) <= '2.0' and cast (workgroupcategory as string) >= '1.0'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where cast(workgroupcategory as string) <= '2.0' and cast (workgroupcategory as string) >= '1.0'")
+    )
+  }
+
+
+  test("Dictionary INT greater less Than implicit  string") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory <= '2.0' and workgroupcategory >= '1.0'"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory  <= '2.0' and workgroupcategory >= '1.0'")
+    )
+  }
+
+
+  test("Dictionary INT greater less Than implicit int") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory <= 2 and workgroupcategory >= 1"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory  <= 2 and workgroupcategory >= 1")
+    )
+  }
+
+
+  test("Dictionary INT greater less Than implicit double") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from DICTIONARY_CARBON_1 where workgroupcategory <= 2.0 and workgroupcategory >= 1.0"),
+      sql("select empno,empname,workgroupcategory from DICTIONARY_HIVE_1 where workgroupcategory  <= 2.0 and workgroupcategory >= 1.0")
+    )
+  }
+
+
+  // No dictionary cases
+  test("NO Dictionary String ") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno = 11"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno = 11")
+    )
+  }
+
+  test("NO Dictionary String OR") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno = '11' or empno = '15'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno = '11' or empno = '15'")
+    )
+  }
+
+  test("NO Dictionary String OR Implicit Cast to int") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno = 11 or empno = 15"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno = 11 or empno = 15")
+    )
+  }
+
+  test("NO Dictionary String OR explicit Cast to int") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(empno as int) = 11 or cast(empno as int) = 15"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(empno as int) = 11 or cast (empno as int) = 15")
+    )
+  }
+
+  test("NO Dictionary INT OR to implicit String") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory = '1' or  workgroupcategory = '2'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory = '1' or  workgroupcategory = '2'")
+    )
+  }
+
+  test("NO Dictionary INT OR to excplicit String") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(workgroupcategory as string)= '1' or cast(workgroupcategory as string) = '2'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as string) = '1' or  cast(workgroupcategory as string) = '2'")
+    )
+  }
+
+  test("NO Dictionary INT OR to explicit double") {
+    sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(workgroupcategory as double)= 1.0 or cast(workgroupcategory as double) = 2.0").show(2000, false)
+    sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) = 1.0 or  cast(workgroupcategory as double) = 2.0 ").show(2000, false)
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(workgroupcategory as double)= 1.0 or cast(workgroupcategory as double) = 2.0"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) = 1.0 or  cast(workgroupcategory as double) = 2.0 ")
+    )
+  }
+
+  test("NO Dictionary String Not") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno != '11'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno != '11'")
+    )
+  }
+
+  test("NO Dictionary String Not Implicit Cast to int") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno != 11"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno != 11")
+    )
+  }
+
+  test("NO Dictionary String Not explicit Cast to int") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(empno as int) != 11"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(empno as int) != 11")
+    )
+  }
+
+  test("NO Dictionary INT Not to implicit String") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory != '1'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory != '1'")
+    )
+  }
+
+  test("NO Dictionary INT Not to excplicit String") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(workgroupcategory as string) != '1'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as string) != '1'")
+    )
+  }
+
+  test("NO Dictionary INT Not to explicit double") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(workgroupcategory as double) != 1.0"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) != 1.0")
+    )
+  }
+
+  test("NO Dictionary String In") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno in ('11', '15')"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno in ('11', '15')")
+    )
+  }
+
+  test("NO Dictionary String In Implicit ") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno in (11, 15)"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno in (11, 15)")
+    )
+  }
+
+  test("NO Dictionary String In Explicit ") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (empno as int) in (11, 15)"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast (empno as int) in (11, 15)")
+    )
+  }
+
+  test("NO Dictionary String In Explicit double") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (empno as double) in (11.0, 15.0)"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast (empno as double) in (11.0, 15.0)")
+    )
+  }
+
+  test("NO Dictionary String Not in Explicit double 1") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (empno as double)  not in (11.0, 15.0)"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast (empno as double) not in (11.0, 15.0)")
+    )
+  }
+
+  test("NO Dictionary INT In to implicit Int") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory in ('1', '2')"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory in ('1', '2')")
+    )
+  }
+
+  test("NO Dictionary INT In to excplicit String") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(workgroupcategory as string) in ('1', '2')"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as string) in ('1', '2')")
+    )
+  }
+
+  test("NO Dictionary INT In to explicit double") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(workgroupcategory as double) in (1.0, 2.0)"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) in (1.0, 2.0)")
+    )
+  }
+
+  test("NO Dictionary String Greater Than") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno > '11'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno > '11'")
+    )
+  }
+
+  test("NO Dictionary String Greater Than Implicit") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno > 11"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno > 11")
+    )
+  }
+
+  test("NO Dictionary String Greater Than explicit 1") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(empno as int) > '11'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast (empno as int) > '11'")
+    )
+  }
+
+
+  test("NO Dictionary String Greater Than explicit 2") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(empno as int) > 11"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast (empno as int) > 11")
+    )
+  }
+
+  test("NO Dictionary INT Greater Than") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory > 1"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory > 1")
+    )
+  }
+
+
+  test("NO Dictionary INT Greater Than implicit ") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory > '1'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory > '1'")
+    )
+  }
+
+
+  test("NO Dictionary INT Greater Than double") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (workgroupcategory as double) > 1.0"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) > 1.0")
+    )
+  }
+
+
+  test("NO Dictionary INT Greater Than String ") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (workgroupcategory as double) > '1.0'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) > '1.0'")
+    )
+  }
+
+
+  test("NO Dictionary String Greater Than equal") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno >= '11'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno >= '11'")
+    )
+  }
+
+  test("NO Dictionary String Greater Than Implicit equal") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno >= 11"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno >= 11")
+    )
+  }
+
+  test("NO Dictionary String Greater Than equal explicit 1") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(empno as int) >= '11'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast (empno as int) >= '11'")
+    )
+  }
+
+
+  test("NO Dictionary String Greater Than equal explicit 2") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(empno as int) >= 11"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast (empno as int) >= 11")
+    )
+  }
+
+  test("NO Dictionary INT Greater Than equal ") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory >= 1"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory >= 1")
+    )
+  }
+
+
+  test("NO Dictionary INT Greater Than equal implicit ") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory >= '1'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory >= '1'")
+    )
+  }
+
+
+  test("NO Dictionary INT Greater Than equal 2") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (workgroupcategory as double) >= 1.0"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) >= 1.0")
+    )
+  }
+
+
+  test("NO Dictionary INT Greater equal Than") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (workgroupcategory as double) >= '1.0'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) >= '1.0'")
+    )
+  }
+
+  test("NO Dictionary String less Than") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno < '20'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno < '20'")
+    )
+  }
+
+  test("NO Dictionary String Less Than Implicit") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno < 20"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno < 20")
+    )
+  }
+
+  test("NO Dictionary String Less Than explicit 1") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(empno as int) < '20'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast (empno as int) < '20'")
+    )
+  }
+
+
+  test("NO Dictionary String Less Than explicit 2") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(empno as int) < 20"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast (empno as int) < 20")
+    )
+  }
+
+  test("NO Dictionary INT Less Than") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory < 2"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory < 2")
+    )
+  }
+
+
+  test("NO Dictionary INT Less Than implicit ") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory < '2'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory < '2'")
+    )
+  }
+
+
+  test("NO Dictionary INT Less Than double") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (workgroupcategory as double) < 2.0"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) < 2.0")
+    )
+  }
+
+
+  test("NO Dictionary INT Less Than String ") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (workgroupcategory as double) < '2.0'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) < '2.0'")
+    )
+  }
+
+
+  test("NO Dictionary String Less Than equal") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno <= '20'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno <= '20'")
+    )
+  }
+
+  test("NO Dictionary String Less Than Implicit equal") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where empno <= 20"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where empno <= 20")
+    )
+  }
+
+  test("NO Dictionary String Less Than equal explicit 1") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(empno as int) <= '20'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast (empno as int) <= '20'")
+    )
+  }
+
+
+  test("NO Dictionary String Less Than equal explicit 2") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast(empno as int) <= 20"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast (empno as int) <= 20")
+    )
+  }
+
+  test("NO Dictionary INT Less Than equal ") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory <= 2"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory <= 2")
+    )
+  }
+
+
+  test("NO Dictionary INT Less Than equal implicit ") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory <= '2'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory <= '2'")
+    )
+  }
+
+
+  test("NO Dictionary INT Less Than equal 2") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (workgroupcategory as double) <= 2.0"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) <= 2.0")
+    )
+  }
+
+
+  test("NO Dictionary INT Less equal Than") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (workgroupcategory as double) <= '2.0'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) <= '2.0'")
+    )
+  }
+
+
+  test("NO Dictionary INT greater less Than explicit double") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (workgroupcategory as double) <= '2.0' and cast (workgroupcategory as double) >= '1.0'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as double) <= '2.0' and cast (workgroupcategory as double) >= '1.0'")
+    )
+  }
+
+
+  test("NO Dictionary INT greater less Than explicit string") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where cast (workgroupcategory as string) <= '2.0' and cast (workgroupcategory as string) >= '1.0'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where cast(workgroupcategory as string) <= '2.0' and cast (workgroupcategory as string) >= '1.0'")
+    )
+  }
+
+
+  test("NO Dictionary INT greater less Than implicit  string") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory <= '2.0' and workgroupcategory >= '1.0'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory  <= '2.0' and workgroupcategory >= '1.0'")
+    )
+  }
+
+
+  test("NO Dictionary INT greater less Than implicit int") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory <= 2 and workgroupcategory >= 1"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory  <= 2 and workgroupcategory >= 1")
+    )
+  }
+
+
+  test("NO Dictionary INT greater less Than implicit double") {
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_2 where workgroupcategory <= 2.0 and workgroupcategory >= 1.0"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_2 where workgroupcategory  <= 2.0 and workgroupcategory >= 1.0")
+    )
+  }
+
+  // Direct Dictionary and Timestamp Column.
+
+
+
+
+  override def afterAll {
+    sql("drop table if exists DICTIONARY_CARBON_1")
+    sql("drop table if exists DICTIONARY_HIVE_1")
+    sql("drop table if exists NO_DICTIONARY_CARBON_2")
+    sql("drop table if exists NO_DICTIONARY_HIVE_2")
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96e5b5cc/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
index 68b08a3..3f42d56 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterAllDataTypesTestCases.scala
@@ -242,6 +242,7 @@ class RangeFilterMyTests extends QueryTest with BeforeAndAfterAll {
   }
 
   test("test range filter for direct dictionary"){
+    sql("select doj from directDictionaryTable where doj > cast('2016-03-14 15:00:17' as timestamp)").show(2000,false)
     checkAnswer(
       sql("select doj from directDictionaryTable where doj > '2016-03-14 15:00:17'"),
       sql("select doj from directDictionaryTable_hive where doj > '2016-03-14 15:00:17'")
@@ -256,6 +257,110 @@ class RangeFilterMyTests extends QueryTest with BeforeAndAfterAll {
     )
   }
 
+  test("test range filter for direct dictionary equality"){
+    checkAnswer(
+      sql("select doj from directDictionaryTable where doj = '2016-03-14 15:00:16'"),
+      Seq(Row(Timestamp.valueOf("2016-03-14 15:00:16.0"))
+      )
+    )
+  }
+
+  test("test range filter for direct dictionary not equality"){
+    sql("select doj from directDictionaryTable where doj != '2016-03-14 15:00:16'").show(2000, false)
+    checkAnswer(
+      sql("select doj from directDictionaryTable where doj != '2016-03-14 15:00:16'"),
+      Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0")),
+        Row(Timestamp.valueOf("2016-03-14 15:00:10.0")),
+        Row(Timestamp.valueOf("2016-03-14 15:00:11.0")),
+        Row(Timestamp.valueOf("2016-03-14 15:00:12.0")),
+        Row(Timestamp.valueOf("2016-03-14 15:00:13.0")),
+        Row(Timestamp.valueOf("2016-03-14 15:00:14.0")),
+        Row(Timestamp.valueOf("2016-03-14 15:00:15.0")),
+        Row(Timestamp.valueOf("2016-03-14 15:00:17.0")),
+        Row(Timestamp.valueOf("2016-03-14 15:00:18.0")))
+    )
+  }
+
+  test("test range filter for direct dictionary and with explicit casts"){
+    sql("select doj from directDictionaryTable where doj > cast ('2016-03-14 15:00:16' as timestamp) and doj < cast ('2016-03-14 15:00:18' as timestamp)").show(2000, false)
+    checkAnswer(
+      sql("select doj from directDictionaryTable where doj > cast ('2016-03-14 15:00:16' as timestamp) and doj < cast ('2016-03-14 15:00:18' as timestamp)"),
+      Seq(Row(Timestamp.valueOf("2016-03-14 15:00:17.0"))
+      )
+    )
+  }
+
+  /*
+  Commented this test case
+  test("test range filter for direct dictionary and with DirectVals as long") {
+    checkAnswer(
+      sql(
+        "select doj from directDictionaryTable where doj > cast (1457992816l as timestamp) and doj < cast (1457992818l as timestamp)"),
+      Seq(Row(Timestamp.valueOf("2016-03-14 15:00:17.0"))
+      )
+    )
+  }
+  */
+
+
+  // Test of Cast Optimization
+  test("test range filter for different Timestamp formats"){
+    checkAnswer(
+      sql("select count(*) from directDictionaryTable where doj = '2016-03-14 15:00:180000000'"),
+      Seq(Row(0)
+      )
+    )
+  }
+
+  test("test range filter for different Timestamp formats1"){
+    checkAnswer(
+      sql("select count(*) from directDictionaryTable where doj = '03-03-14 15:00:18'"),
+      Seq(Row(0)
+      )
+    )
+  }
+
+  test("test range filter for different Timestamp formats2"){
+    checkAnswer(
+      sql("select count(*) from directDictionaryTable where doj = '2016-03-14'"),
+      Seq(Row(0)
+      )
+    )
+  }
+
+  test("test range filter for different Timestamp formats3"){
+    checkAnswer(
+      sql("select count(*) from directDictionaryTable where doj > '2016-03-14 15:00:18.000'"),
+      sql("select count(*) from directDictionaryTable_hive where doj > '2016-03-14 15:00:18.000'")
+      )
+  }
+
+  test("test range filter for different Timestamp In format "){
+    sql("select doj from directDictionaryTable").show(200,false)
+    sql("select * from directDictionaryTable where doj in ('2016-03-14 15:00:18', '2016-03-14 15:00:17' )").show(200, false)
+    checkAnswer(
+      sql("select doj from directDictionaryTable where doj in ('2016-03-14 15:00:18', '2016-03-14 15:00:17')"),
+      Seq(Row(Timestamp.valueOf("2016-03-14 15:00:17.0")),
+        Row(Timestamp.valueOf("2016-03-14 15:00:18.0")))
+    )
+  }
+
+  /*
+  test("test range filter for different Timestamp Not In format 5"){
+    sql("select doj from directDictionaryTable where doj not in (null, '2016-03-14 15:00:18', '2016-03-14 15:00:17','2016-03-14 15:00:11', '2016-03-14 15:00:12')").show(200, false)
+    checkAnswer(
+      sql("select doj from directDictionaryTable where doj Not in (null, '2016-03-14 15:00:18', '2016-03-14 15:00:17','2016-03-14 15:00:11', '2016-03-14 15:00:12')"),
+      Seq(Row(Timestamp.valueOf("2016-03-14 15:00:09.0")),
+        Row(Timestamp.valueOf("2016-03-14 15:00:10.0")),
+        Row(Timestamp.valueOf("2016-03-14 15:00:13.0")),
+        Row(Timestamp.valueOf("2016-03-14 15:00:14.0")),
+        Row(Timestamp.valueOf("2016-03-14 15:00:15.0")),
+        Row(Timestamp.valueOf("2016-03-14 15:00:16.0")))
+    )
+  }
+  */
+
+
   test("test range filter for direct dictionary and boundary"){
     checkAnswer(
       sql("select count(*) from directDictionaryTable where doj > '2016-03-14 15:00:18.0' and doj < '2016-03-14 15:00:09.0'"),
@@ -510,6 +615,13 @@ class RangeFilterMyTests extends QueryTest with BeforeAndAfterAll {
     )
   }
 
+  test("test range filter No Dictionary Range"){
+    checkAnswer(
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_8 where empno > '13' and empno < '17'"),
+      sql("select empno,empname,workgroupcategory from NO_DICTIONARY_HIVE_8 where empno > '13' and empno < '17'")
+    )
+  }
+
   test("test range filter for more columns conditions"){
     checkAnswer(
       sql("select empno,empname,workgroupcategory from NO_DICTIONARY_CARBON_8 where empno > '13' and workgroupcategory < 3 and deptno > 12 and empno < '17'"),
@@ -531,25 +643,6 @@ class RangeFilterMyTests extends QueryTest with BeforeAndAfterAll {
     )
   }
 
-  /*
-  test("test range filter with complex datatypes 1"){
-    sql("select proddate.productionDate from complexcarbontable where proddate.productionDate > '30-11-2015' and proddate.productionDate < '31-01-2016' ").show()
-    checkAnswer(
-      sql("select proddate.productionDate from complexcarbontable where proddate.productionDate > '30-11-2015' and proddate.productionDate < '31-01-2016'"),
-      Seq(Row("30-12-2015"))
-    )
-  }
-  */
-
-  /*
-  test("test range filter with complex datatypes 2"){
-    sql("select ROMSize,mobile.imei from complexcarbontable where proddate.productionDate > '30-11-2015' and proddate.productionDate < '31-01-2016' ").show()
-    checkAnswer(
-      sql("select ROMSize,mobile.imei from complexcarbontable where proddate.productionDate > '30-11-2015' and proddate.productionDate < '31-01-2016'"),
-      Seq(Row("8ROM size",""))
-    )
-  }
-  */
 
 
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96e5b5cc/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterTestCase.scala
index 3dd01cc..46b514a 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/RangeFilterTestCase.scala
@@ -89,7 +89,7 @@ class RangeFilterTestCase extends QueryTest with BeforeAndAfterAll {
         "deptno Int, deptname String, projectcode Int, projectjoindate Timestamp, " +
         "projectenddate Timestamp, designation String,attendance Int,utilization " +
         "Int,salary Int) STORED BY 'org.apache.carbondata.format' " +
-        "TBLPROPERTIES('DICTIONARY_EXCLUDE'='empno, empname,designation')"
+        "TBLPROPERTIES('DICTIONARY_INCLUDE'='workgroupcategory','DICTIONARY_EXCLUDE'='empno, empname,designation')"
     )
     sql(
       s"LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE NO_DICTIONARY_CARBON_6 " +
@@ -102,7 +102,7 @@ class RangeFilterTestCase extends QueryTest with BeforeAndAfterAll {
         "deptno Int, deptname String, projectcode Int, projectjoindate Timestamp, " +
         "projectenddate Timestamp, designation String,attendance Int,utilization " +
         "Int,salary Int) STORED BY 'org.apache.carbondata.format' " +
-        "TBLPROPERTIES('DICTIONARY_EXCLUDE'='empname,designation')"
+        "TBLPROPERTIES('DICTIONARY_INCLUDE'='workgroupcategory','DICTIONARY_EXCLUDE'='empname,designation')"
     )
     sql(
       s"LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE DICTIONARY_CARBON_6 " +

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96e5b5cc/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala
deleted file mode 100644
index 5698dab..0000000
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/CarbonFilters.scala
+++ /dev/null
@@ -1,370 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.carbondata.spark
-
-import scala.collection.mutable.ArrayBuffer
-
-import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.optimizer.AttributeReferenceWrapper
-import org.apache.spark.sql.sources
-import org.apache.spark.sql.types.StructType
-
-import org.apache.carbondata.core.metadata.datatype.DataType
-import org.apache.carbondata.core.metadata.schema.table.CarbonTable
-import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn
-import org.apache.carbondata.core.scan.expression.{ColumnExpression => CarbonColumnExpression, Expression => CarbonExpression, LiteralExpression => CarbonLiteralExpression}
-import org.apache.carbondata.core.scan.expression.conditional._
-import org.apache.carbondata.core.scan.expression.logical.{AndExpression, FalseExpression, OrExpression}
-import org.apache.carbondata.spark.util.CarbonScalaUtil
-
-/**
- * All filter conversions are done here.
- */
-object CarbonFilters {
-
-  /**
-   * Converts data sources filters to carbon filter predicates.
-   */
-  def createCarbonFilter(schema: StructType,
-      predicate: sources.Filter): Option[CarbonExpression] = {
-    val dataTypeOf = schema.map(f => f.name -> f.dataType).toMap
-
-    def createFilter(predicate: sources.Filter): Option[CarbonExpression] = {
-      predicate match {
-
-        case sources.EqualTo(name, value) =>
-          Some(new EqualToExpression(getCarbonExpression(name),
-            getCarbonLiteralExpression(name, value)))
-        case sources.Not(sources.EqualTo(name, value)) =>
-          Some(new NotEqualsExpression(getCarbonExpression(name),
-            getCarbonLiteralExpression(name, value)))
-
-        case sources.EqualNullSafe(name, value) =>
-          Some(new EqualToExpression(getCarbonExpression(name),
-            getCarbonLiteralExpression(name, value)))
-        case sources.Not(sources.EqualNullSafe(name, value)) =>
-          Some(new NotEqualsExpression(getCarbonExpression(name),
-            getCarbonLiteralExpression(name, value)))
-
-        case sources.GreaterThan(name, value) =>
-          Some(new GreaterThanExpression(getCarbonExpression(name),
-            getCarbonLiteralExpression(name, value)))
-        case sources.LessThan(name, value) =>
-          Some(new LessThanExpression(getCarbonExpression(name),
-            getCarbonLiteralExpression(name, value)))
-        case sources.GreaterThanOrEqual(name, value) =>
-          Some(new GreaterThanEqualToExpression(getCarbonExpression(name),
-            getCarbonLiteralExpression(name, value)))
-        case sources.LessThanOrEqual(name, value) =>
-          Some(new LessThanEqualToExpression(getCarbonExpression(name),
-            getCarbonLiteralExpression(name, value)))
-
-        case sources.In(name, values) =>
-          Some(new InExpression(getCarbonExpression(name),
-            new ListExpression(
-              convertToJavaList(values.map(f => getCarbonLiteralExpression(name, f)).toList))))
-        case sources.Not(sources.In(name, values)) =>
-          Some(new NotInExpression(getCarbonExpression(name),
-            new ListExpression(
-              convertToJavaList(values.map(f => getCarbonLiteralExpression(name, f)).toList))))
-
-        case sources.IsNull(name) =>
-          Some(new EqualToExpression(getCarbonExpression(name),
-            getCarbonLiteralExpression(name, null), true))
-        case sources.IsNotNull(name) =>
-          Some(new NotEqualsExpression(getCarbonExpression(name),
-            getCarbonLiteralExpression(name, null), true))
-
-        case sources.And(lhs, rhs) =>
-          (createFilter(lhs) ++ createFilter(rhs)).reduceOption(new AndExpression(_, _))
-
-        case sources.Or(lhs, rhs) =>
-          for {
-            lhsFilter <- createFilter(lhs)
-            rhsFilter <- createFilter(rhs)
-          } yield {
-            new OrExpression(lhsFilter, rhsFilter)
-          }
-
-        case _ => None
-      }
-    }
-
-    def getCarbonExpression(name: String) = {
-      new CarbonColumnExpression(name,
-        CarbonScalaUtil.convertSparkToCarbonDataType(dataTypeOf(name)))
-    }
-
-    def getCarbonLiteralExpression(name: String, value: Any): CarbonExpression = {
-      val dataTypeOfAttribute = CarbonScalaUtil.convertSparkToCarbonDataType(dataTypeOf(name))
-      val dataType = if (Option(value).isDefined
-                         && dataTypeOfAttribute == DataType.STRING
-                         && value.isInstanceOf[Double]) {
-        DataType.DOUBLE
-      } else {
-        dataTypeOfAttribute
-      }
-      new CarbonLiteralExpression(value, dataType)
-    }
-
-    createFilter(predicate)
-  }
-
-
-  // Check out which filters can be pushed down to carbon, remaining can be handled in spark layer.
-  // Mostly dimension filters are only pushed down since it is faster in carbon.
-  def selectFilters(filters: Seq[Expression],
-      attrList: java.util.HashSet[AttributeReferenceWrapper],
-      aliasMap: CarbonAliasDecoderRelation): Unit = {
-    def translate(expr: Expression, or: Boolean = false): Option[sources.Filter] = {
-      expr match {
-        case or@ Or(left, right) =>
-
-          val leftFilter = translate(left, or = true)
-          val rightFilter = translate(right, or = true)
-          if (leftFilter.isDefined && rightFilter.isDefined) {
-            Some( sources.Or(leftFilter.get, rightFilter.get))
-          } else {
-            or.collect {
-              case attr: AttributeReference =>
-                attrList.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
-            }
-            None
-          }
-
-        case And(left, right) =>
-          val leftFilter = translate(left, or)
-          val rightFilter = translate(right, or)
-          if (or) {
-            if (leftFilter.isDefined && rightFilter.isDefined) {
-              (leftFilter ++ rightFilter).reduceOption(sources.And)
-            } else {
-              None
-            }
-          } else {
-            (leftFilter ++ rightFilter).reduceOption(sources.And)
-          }
-
-        case EqualTo(a: Attribute, Literal(v, t)) =>
-          Some(sources.EqualTo(a.name, v))
-        case EqualTo(l@Literal(v, t), a: Attribute) =>
-          Some(sources.EqualTo(a.name, v))
-        case Not(EqualTo(a: Attribute, Literal(v, t))) =>
-          Some(sources.Not(sources.EqualTo(a.name, v)))
-        case Not(EqualTo(Literal(v, t), a: Attribute)) =>
-          Some(sources.Not(sources.EqualTo(a.name, v)))
-        case IsNotNull(a: Attribute) =>
-          Some(sources.IsNotNull(a.name))
-        case IsNull(a: Attribute) =>
-          Some(sources.IsNull(a.name))
-        case Not(In(a: Attribute, list)) if !list.exists(!_.isInstanceOf[Literal]) =>
-          val hSet = list.map(e => e.eval(EmptyRow))
-          Some(sources.Not(sources.In(a.name, hSet.toArray)))
-        case In(a: Attribute, list) if !list.exists(!_.isInstanceOf[Literal]) =>
-          val hSet = list.map(e => e.eval(EmptyRow))
-          Some(sources.In(a.name, hSet.toArray))
-        case GreaterThan(a: Attribute, Literal(v, t)) =>
-          Some(sources.GreaterThan(a.name, v))
-        case GreaterThan(Literal(v, t), a: Attribute) =>
-          Some(sources.LessThan(a.name, v))
-        case LessThan(a: Attribute, Literal(v, t)) =>
-          Some(sources.LessThan(a.name, v))
-        case LessThan(Literal(v, t), a: Attribute) =>
-          Some(sources.GreaterThan(a.name, v))
-        case GreaterThanOrEqual(a: Attribute, Literal(v, t)) =>
-          Some(sources.GreaterThanOrEqual(a.name, v))
-        case GreaterThanOrEqual(Literal(v, t), a: Attribute) =>
-          Some(sources.LessThanOrEqual(a.name, v))
-        case LessThanOrEqual(a: Attribute, Literal(v, t)) =>
-          Some(sources.LessThanOrEqual(a.name, v))
-        case LessThanOrEqual(Literal(v, t), a: Attribute) =>
-          Some(sources.GreaterThanOrEqual(a.name, v))
-
-        case others =>
-          if (!or) {
-            others.collect {
-              case attr: AttributeReference =>
-                attrList.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
-            }
-          }
-          None
-      }
-    }
-    filters.flatMap(translate(_, false)).toArray
-  }
-
-  def processExpression(exprs: Seq[Expression],
-      attributesNeedToDecode: java.util.HashSet[AttributeReference],
-      unprocessedExprs: ArrayBuffer[Expression],
-      carbonTable: CarbonTable): Option[CarbonExpression] = {
-    def transformExpression(expr: Expression, or: Boolean = false): Option[CarbonExpression] = {
-      expr match {
-        case orFilter@ Or(left, right) =>
-          val leftFilter = transformExpression(left, or = true)
-          val rightFilter = transformExpression(right, or = true)
-          if (leftFilter.isDefined && rightFilter.isDefined) {
-            Some(new OrExpression(leftFilter.get, rightFilter.get))
-          } else {
-            if (!or) {
-              orFilter.collect {
-                case attr: AttributeReference => attributesNeedToDecode.add(attr)
-              }
-              unprocessedExprs += orFilter
-            }
-            None
-          }
-
-        case And(left, right) =>
-          val leftFilter = transformExpression(left, or)
-          val rightFilter = transformExpression(right, or)
-          if (or) {
-            if (leftFilter.isDefined && rightFilter.isDefined) {
-              (leftFilter ++ rightFilter).reduceOption(new AndExpression(_, _))
-            } else {
-              None
-            }
-          } else {
-            (leftFilter ++ rightFilter).reduceOption(new AndExpression(_, _))
-          }
-
-
-        case EqualTo(a: Attribute, l@Literal(v, t)) =>
-          Some(
-            new EqualToExpression(
-              transformExpression(a).get,
-              transformExpression(l).get
-            )
-          )
-        case EqualTo(l@Literal(v, t), a: Attribute) =>
-          Some(
-            new EqualToExpression(
-              transformExpression(a).get,
-              transformExpression(l).get
-            )
-          )
-
-        case Not(EqualTo(a: Attribute, l@Literal(v, t))) =>
-          Some(
-            new NotEqualsExpression(
-              transformExpression(a).get,
-              transformExpression(l).get
-            )
-          )
-        case Not(EqualTo(l@Literal(v, t), a: Attribute)) =>
-          Some(
-            new NotEqualsExpression(
-              transformExpression(a).get,
-              transformExpression(l).get
-            )
-          )
-        case IsNotNull(child: Attribute) =>
-            Some(new NotEqualsExpression(transformExpression(child).get,
-             transformExpression(Literal(null)).get, true))
-        case IsNull(child: Attribute) =>
-            Some(new EqualToExpression(transformExpression(child).get,
-             transformExpression(Literal(null)).get, true))
-        case Not(In(a: Attribute, list))
-          if !list.exists(!_.isInstanceOf[Literal]) =>
-          if (list.exists(x => isNullLiteral(x.asInstanceOf[Literal]))) {
-            Some(new FalseExpression(transformExpression(a).get))
-          } else {
-            Some(new NotInExpression(transformExpression(a).get,
-              new ListExpression(convertToJavaList(list.map(transformExpression(_).get)))))
-          }
-        case In(a: Attribute, list) if !list.exists(!_.isInstanceOf[Literal]) =>
-          Some(new InExpression(transformExpression(a).get,
-            new ListExpression(convertToJavaList(list.map(transformExpression(_).get)))))
-
-        case GreaterThan(a: Attribute, l@Literal(v, t)) =>
-          Some(new GreaterThanExpression(transformExpression(a).get, transformExpression(l).get))
-        case GreaterThan(l@Literal(v, t), a: Attribute) =>
-          Some(new LessThanExpression(transformExpression(a).get, transformExpression(l).get))
-
-        case LessThan(a: Attribute, l@Literal(v, t)) =>
-          Some(new LessThanExpression(transformExpression(a).get, transformExpression(l).get))
-        case LessThan(l@Literal(v, t), a: Attribute) =>
-          Some(new GreaterThanExpression(transformExpression(a).get, transformExpression(l).get))
-
-        case GreaterThanOrEqual(a: Attribute, l@Literal(v, t)) =>
-          Some(new GreaterThanEqualToExpression(transformExpression(a).get,
-            transformExpression(l).get))
-        case GreaterThanOrEqual(l@Literal(v, t), a: Attribute) =>
-          Some(new LessThanEqualToExpression(transformExpression(a).get,
-            transformExpression(l).get))
-
-        case LessThanOrEqual(a: Attribute, l@Literal(v, t)) =>
-          Some(new LessThanEqualToExpression(transformExpression(a).get,
-            transformExpression(l).get))
-        case LessThanOrEqual(l@Literal(v, t), a: Attribute) =>
-          Some(new GreaterThanEqualToExpression(transformExpression(a).get,
-            transformExpression(l).get))
-
-        case AttributeReference(name, dataType, _, _) =>
-          Some(new CarbonColumnExpression(name,
-            CarbonScalaUtil.convertSparkToCarbonDataType(
-              getActualCarbonDataType(name, carbonTable))))
-        case Literal(name, dataType) => Some(new
-            CarbonLiteralExpression(name, CarbonScalaUtil.convertSparkToCarbonDataType(dataType)))
-        case others =>
-          if (!or) {
-            others.collect {
-              case attr: AttributeReference => attributesNeedToDecode.add(attr)
-            }
-            unprocessedExprs += others
-          }
-          None
-      }
-    }
-    exprs.flatMap(transformExpression(_, false)).reduceOption(new AndExpression(_, _))
-  }
-  private def isNullLiteral(exp: Expression): Boolean = {
-    if (null != exp
-        &&  exp.isInstanceOf[Literal]
-        && (exp.asInstanceOf[Literal].dataType == org.apache.spark.sql.types.DataTypes.NullType)
-        || (exp.asInstanceOf[Literal].value == null)) {
-      true
-    } else {
-      false
-    }
-  }
-  private def getActualCarbonDataType(column: String, carbonTable: CarbonTable) = {
-    var carbonColumn: CarbonColumn =
-      carbonTable.getDimensionByName(carbonTable.getFactTableName, column)
-    val dataType = if (carbonColumn != null) {
-      carbonColumn.getDataType
-    } else {
-      carbonColumn = carbonTable.getMeasureByName(carbonTable.getFactTableName, column)
-      carbonColumn.getDataType match {
-        case DataType.INT => DataType.LONG
-        case DataType.LONG => DataType.LONG
-        case DataType.DECIMAL => DataType.DECIMAL
-        case _ => DataType.DOUBLE
-      }
-    }
-    CarbonScalaUtil.convertCarbonToSparkDataType(dataType)
-  }
-
-  // Convert scala list to java list, Cannot use scalaList.asJava as while deserializing it is
-  // not able find the classes inside scala list and gives ClassNotFoundException.
-  private def convertToJavaList(
-      scalaList: Seq[CarbonExpression]): java.util.List[CarbonExpression] = {
-    val javaList = new java.util.ArrayList[CarbonExpression]()
-    scalaList.foreach(javaList.add)
-    javaList
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96e5b5cc/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
index 3bb1efd..a6c28a9 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonBoundReference.scala
@@ -18,12 +18,14 @@
 package org.apache.spark.sql
 
 import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.{Attribute, ExprId, LeafExpression, NamedExpression}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression, ExprId, LeafExpression, NamedExpression}
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
+import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types.DataType
 
 import org.apache.carbondata.core.scan.expression.ColumnExpression
 
+case class CastExpr(expr: Expression) extends Filter
 
 case class CarbonBoundReference(colExp: ColumnExpression, dataType: DataType, nullable: Boolean)
   extends LeafExpression with NamedExpression with CodegenFallback {
@@ -38,8 +40,7 @@ case class CarbonBoundReference(colExp: ColumnExpression, dataType: DataType, nu
 
   override def toAttribute: Attribute = throw new UnsupportedOperationException
 
-  override def qualifiers: Seq[String] = throw new UnsupportedOperationException
-
   override def exprId: ExprId = throw new UnsupportedOperationException
 
+  override def qualifiers: Seq[String] = throw new UnsupportedOperationException
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/96e5b5cc/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
new file mode 100644
index 0000000..cf8689c
--- /dev/null
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/optimizer/CarbonFilters.scala
@@ -0,0 +1,373 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.spark
+
+import scala.collection.mutable.ArrayBuffer
+
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.optimizer.AttributeReferenceWrapper
+import org.apache.spark.sql.sources
+import org.apache.spark.sql.types.StructType
+
+import org.apache.carbondata.core.metadata.datatype.DataType
+import org.apache.carbondata.core.metadata.schema.table.CarbonTable
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn
+import org.apache.carbondata.core.scan.expression.{ColumnExpression => CarbonColumnExpression, Expression => CarbonExpression, LiteralExpression => CarbonLiteralExpression}
+import org.apache.carbondata.core.scan.expression.conditional._
+import org.apache.carbondata.core.scan.expression.logical.{AndExpression, FalseExpression, OrExpression}
+import org.apache.carbondata.spark.util.CarbonScalaUtil
+
+/**
+ * All filter conversions are done here.
+ */
+object CarbonFilters {
+
+  /**
+   * Converts data sources filters to carbon filter predicates.
+   */
+  def createCarbonFilter(schema: StructType,
+      predicate: sources.Filter): Option[CarbonExpression] = {
+    val dataTypeOf = schema.map(f => f.name -> f.dataType).toMap
+
+    def createFilter(predicate: sources.Filter): Option[CarbonExpression] = {
+      predicate match {
+
+        case sources.EqualTo(name, value) =>
+          Some(new EqualToExpression(getCarbonExpression(name),
+            getCarbonLiteralExpression(name, value)))
+        case sources.Not(sources.EqualTo(name, value)) =>
+          Some(new NotEqualsExpression(getCarbonExpression(name),
+            getCarbonLiteralExpression(name, value)))
+
+        case sources.EqualNullSafe(name, value) =>
+          Some(new EqualToExpression(getCarbonExpression(name),
+            getCarbonLiteralExpression(name, value)))
+        case sources.Not(sources.EqualNullSafe(name, value)) =>
+          Some(new NotEqualsExpression(getCarbonExpression(name),
+            getCarbonLiteralExpression(name, value)))
+
+        case sources.GreaterThan(name, value) =>
+          Some(new GreaterThanExpression(getCarbonExpression(name),
+            getCarbonLiteralExpression(name, value)))
+        case sources.LessThan(name, value) =>
+          Some(new LessThanExpression(getCarbonExpression(name),
+            getCarbonLiteralExpression(name, value)))
+        case sources.GreaterThanOrEqual(name, value) =>
+          Some(new GreaterThanEqualToExpression(getCarbonExpression(name),
+            getCarbonLiteralExpression(name, value)))
+        case sources.LessThanOrEqual(name, value) =>
+          Some(new LessThanEqualToExpression(getCarbonExpression(name),
+            getCarbonLiteralExpression(name, value)))
+
+        case sources.In(name, values) =>
+          Some(new InExpression(getCarbonExpression(name),
+            new ListExpression(
+              convertToJavaList(values.map(f => getCarbonLiteralExpression(name, f)).toList))))
+        case sources.Not(sources.In(name, values)) =>
+          Some(new NotInExpression(getCarbonExpression(name),
+            new ListExpression(
+              convertToJavaList(values.map(f => getCarbonLiteralExpression(name, f)).toList))))
+
+        case sources.IsNull(name) =>
+          Some(new EqualToExpression(getCarbonExpression(name),
+            getCarbonLiteralExpression(name, null), true))
+        case sources.IsNotNull(name) =>
+          Some(new NotEqualsExpression(getCarbonExpression(name),
+            getCarbonLiteralExpression(name, null), true))
+
+        case sources.And(lhs, rhs) =>
+          (createFilter(lhs) ++ createFilter(rhs)).reduceOption(new AndExpression(_, _))
+
+        case sources.Or(lhs, rhs) =>
+          for {
+            lhsFilter <- createFilter(lhs)
+            rhsFilter <- createFilter(rhs)
+          } yield {
+            new OrExpression(lhsFilter, rhsFilter)
+          }
+
+        case _ => None
+      }
+    }
+
+    def getCarbonExpression(name: String) = {
+      new CarbonColumnExpression(name,
+        CarbonScalaUtil.convertSparkToCarbonDataType(dataTypeOf(name)))
+    }
+
+    def getCarbonLiteralExpression(name: String, value: Any): CarbonExpression = {
+      val dataTypeOfAttribute = CarbonScalaUtil.convertSparkToCarbonDataType(dataTypeOf(name))
+      val dataType = if (Option(value).isDefined
+                         && dataTypeOfAttribute == DataType.STRING
+                         && value.isInstanceOf[Double]) {
+        DataType.DOUBLE
+      } else {
+        dataTypeOfAttribute
+      }
+      new CarbonLiteralExpression(value, dataType)
+    }
+
+    createFilter(predicate)
+  }
+
+
+  // Check out which filters can be pushed down to carbon, remaining can be handled in spark layer.
+  // Mostly dimension filters are only pushed down since it is faster in carbon.
+  // TODO - The Filters are first converted Intermediate sources filters expression and then these
+  // expressions are again converted back to CarbonExpression. Instead of two step process of
+  // evaluating the filters it can be merged into a single one.
+  def selectFilters(filters: Seq[Expression],
+      attrList: java.util.HashSet[AttributeReferenceWrapper],
+      aliasMap: CarbonAliasDecoderRelation): Unit = {
+    def translate(expr: Expression, or: Boolean = false): Option[sources.Filter] = {
+      expr match {
+        case or@ Or(left, right) =>
+
+          val leftFilter = translate(left, or = true)
+          val rightFilter = translate(right, or = true)
+          if (leftFilter.isDefined && rightFilter.isDefined) {
+            Some( sources.Or(leftFilter.get, rightFilter.get))
+          } else {
+            or.collect {
+              case attr: AttributeReference =>
+                attrList.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
+            }
+            None
+          }
+
+        case And(left, right) =>
+          val leftFilter = translate(left, or)
+          val rightFilter = translate(right, or)
+          if (or) {
+            if (leftFilter.isDefined && rightFilter.isDefined) {
+              (leftFilter ++ rightFilter).reduceOption(sources.And)
+            } else {
+              None
+            }
+          } else {
+            (leftFilter ++ rightFilter).reduceOption(sources.And)
+          }
+
+        case EqualTo(a: Attribute, Literal(v, t)) =>
+          Some(sources.EqualTo(a.name, v))
+        case EqualTo(l@Literal(v, t), a: Attribute) =>
+          Some(sources.EqualTo(a.name, v))
+        case Not(EqualTo(a: Attribute, Literal(v, t))) =>
+          Some(sources.Not(sources.EqualTo(a.name, v)))
+        case Not(EqualTo(Literal(v, t), a: Attribute)) =>
+          Some(sources.Not(sources.EqualTo(a.name, v)))
+        case IsNotNull(a: Attribute) =>
+          Some(sources.IsNotNull(a.name))
+        case IsNull(a: Attribute) =>
+          Some(sources.IsNull(a.name))
+        case Not(In(a: Attribute, list)) if !list.exists(!_.isInstanceOf[Literal]) =>
+          val hSet = list.map(e => e.eval(EmptyRow))
+          Some(sources.Not(sources.In(a.name, hSet.toArray)))
+        case In(a: Attribute, list) if !list.exists(!_.isInstanceOf[Literal]) =>
+          val hSet = list.map(e => e.eval(EmptyRow))
+          Some(sources.In(a.name, hSet.toArray))
+        case GreaterThan(a: Attribute, Literal(v, t)) =>
+          Some(sources.GreaterThan(a.name, v))
+        case GreaterThan(Literal(v, t), a: Attribute) =>
+          Some(sources.LessThan(a.name, v))
+        case LessThan(a: Attribute, Literal(v, t)) =>
+          Some(sources.LessThan(a.name, v))
+        case LessThan(Literal(v, t), a: Attribute) =>
+          Some(sources.GreaterThan(a.name, v))
+        case GreaterThanOrEqual(a: Attribute, Literal(v, t)) =>
+          Some(sources.GreaterThanOrEqual(a.name, v))
+        case GreaterThanOrEqual(Literal(v, t), a: Attribute) =>
+          Some(sources.LessThanOrEqual(a.name, v))
+        case LessThanOrEqual(a: Attribute, Literal(v, t)) =>
+          Some(sources.LessThanOrEqual(a.name, v))
+        case LessThanOrEqual(Literal(v, t), a: Attribute) =>
+          Some(sources.GreaterThanOrEqual(a.name, v))
+
+        case others =>
+          if (!or) {
+            others.collect {
+              case attr: AttributeReference =>
+                attrList.add(AttributeReferenceWrapper(aliasMap.getOrElse(attr, attr)))
+            }
+          }
+          None
+      }
+    }
+    filters.flatMap(translate(_, false)).toArray
+  }
+
+  def processExpression(exprs: Seq[Expression],
+      attributesNeedToDecode: java.util.HashSet[AttributeReference],
+      unprocessedExprs: ArrayBuffer[Expression],
+      carbonTable: CarbonTable): Option[CarbonExpression] = {
+    def transformExpression(expr: Expression, or: Boolean = false): Option[CarbonExpression] = {
+      expr match {
+        case orFilter@ Or(left, right) =>
+          val leftFilter = transformExpression(left, or = true)
+          val rightFilter = transformExpression(right, or = true)
+          if (leftFilter.isDefined && rightFilter.isDefined) {
+            Some(new OrExpression(leftFilter.get, rightFilter.get))
+          } else {
+            if (!or) {
+              orFilter.collect {
+                case attr: AttributeReference => attributesNeedToDecode.add(attr)
+              }
+              unprocessedExprs += orFilter
+            }
+            None
+          }
+
+        case And(left, right) =>
+          val leftFilter = transformExpression(left, or)
+          val rightFilter = transformExpression(right, or)
+          if (or) {
+            if (leftFilter.isDefined && rightFilter.isDefined) {
+              (leftFilter ++ rightFilter).reduceOption(new AndExpression(_, _))
+            } else {
+              None
+            }
+          } else {
+            (leftFilter ++ rightFilter).reduceOption(new AndExpression(_, _))
+          }
+
+
+        case EqualTo(a: Attribute, l@Literal(v, t)) =>
+          Some(
+            new EqualToExpression(
+              transformExpression(a).get,
+              transformExpression(l).get
+            )
+          )
+        case EqualTo(l@Literal(v, t), a: Attribute) =>
+          Some(
+            new EqualToExpression(
+              transformExpression(a).get,
+              transformExpression(l).get
+            )
+          )
+
+        case Not(EqualTo(a: Attribute, l@Literal(v, t))) =>
+          Some(
+            new NotEqualsExpression(
+              transformExpression(a).get,
+              transformExpression(l).get
+            )
+          )
+        case Not(EqualTo(l@Literal(v, t), a: Attribute)) =>
+          Some(
+            new NotEqualsExpression(
+              transformExpression(a).get,
+              transformExpression(l).get
+            )
+          )
+        case IsNotNull(child: Attribute) =>
+          Some(new NotEqualsExpression(transformExpression(child).get,
+            transformExpression(Literal(null)).get, true))
+        case IsNull(child: Attribute) =>
+          Some(new EqualToExpression(transformExpression(child).get,
+            transformExpression(Literal(null)).get, true))
+        case Not(In(a: Attribute, list))
+          if !list.exists(!_.isInstanceOf[Literal]) =>
+          if (list.exists(x => isNullLiteral(x.asInstanceOf[Literal]))) {
+            Some(new FalseExpression(transformExpression(a).get))
+          } else {
+            Some(new NotInExpression(transformExpression(a).get,
+              new ListExpression(convertToJavaList(list.map(transformExpression(_).get)))))
+          }
+        case In(a: Attribute, list) if !list.exists(!_.isInstanceOf[Literal]) =>
+          Some(new InExpression(transformExpression(a).get,
+            new ListExpression(convertToJavaList(list.map(transformExpression(_).get)))))
+
+        case GreaterThan(a: Attribute, l@Literal(v, t)) =>
+          Some(new GreaterThanExpression(transformExpression(a).get, transformExpression(l).get))
+        case GreaterThan(l@Literal(v, t), a: Attribute) =>
+          Some(new LessThanExpression(transformExpression(a).get, transformExpression(l).get))
+
+        case LessThan(a: Attribute, l@Literal(v, t)) =>
+          Some(new LessThanExpression(transformExpression(a).get, transformExpression(l).get))
+        case LessThan(l@Literal(v, t), a: Attribute) =>
+          Some(new GreaterThanExpression(transformExpression(a).get, transformExpression(l).get))
+
+        case GreaterThanOrEqual(a: Attribute, l@Literal(v, t)) =>
+          Some(new GreaterThanEqualToExpression(transformExpression(a).get,
+            transformExpression(l).get))
+        case GreaterThanOrEqual(l@Literal(v, t), a: Attribute) =>
+          Some(new LessThanEqualToExpression(transformExpression(a).get,
+            transformExpression(l).get))
+
+        case LessThanOrEqual(a: Attribute, l@Literal(v, t)) =>
+          Some(new LessThanEqualToExpression(transformExpression(a).get,
+            transformExpression(l).get))
+        case LessThanOrEqual(l@Literal(v, t), a: Attribute) =>
+          Some(new GreaterThanEqualToExpression(transformExpression(a).get,
+            transformExpression(l).get))
+
+        case AttributeReference(name, dataType, _, _) =>
+          Some(new CarbonColumnExpression(name,
+            CarbonScalaUtil.convertSparkToCarbonDataType(
+              getActualCarbonDataType(name, carbonTable))))
+        case Literal(name, dataType) => Some(new
+            CarbonLiteralExpression(name, CarbonScalaUtil.convertSparkToCarbonDataType(dataType)))
+        case others =>
+          if (!or) {
+            others.collect {
+              case attr: AttributeReference => attributesNeedToDecode.add(attr)
+            }
+            unprocessedExprs += others
+          }
+          None
+      }
+    }
+    exprs.flatMap(transformExpression(_, false)).reduceOption(new AndExpression(_, _))
+  }
+  private def isNullLiteral(exp: Expression): Boolean = {
+    if (null != exp
+        &&  exp.isInstanceOf[Literal]
+        && (exp.asInstanceOf[Literal].dataType == org.apache.spark.sql.types.DataTypes.NullType)
+        || (exp.asInstanceOf[Literal].value == null)) {
+      true
+    } else {
+      false
+    }
+  }
+  private def getActualCarbonDataType(column: String, carbonTable: CarbonTable) = {
+    var carbonColumn: CarbonColumn =
+      carbonTable.getDimensionByName(carbonTable.getFactTableName, column)
+    val dataType = if (carbonColumn != null) {
+      carbonColumn.getDataType
+    } else {
+      carbonColumn = carbonTable.getMeasureByName(carbonTable.getFactTableName, column)
+      carbonColumn.getDataType match {
+        case DataType.INT => DataType.LONG
+        case DataType.LONG => DataType.LONG
+        case DataType.DECIMAL => DataType.DECIMAL
+        case _ => DataType.DOUBLE
+      }
+    }
+    CarbonScalaUtil.convertCarbonToSparkDataType(dataType)
+  }
+
+  // Convert scala list to java list, Cannot use scalaList.asJava as while deserializing it is
+  // not able find the classes inside scala list and gives ClassNotFoundException.
+  private def convertToJavaList(
+      scalaList: Seq[CarbonExpression]): java.util.List[CarbonExpression] = {
+    val javaList = new java.util.ArrayList[CarbonExpression]()
+    scalaList.foreach(javaList.add)
+    javaList
+  }
+}



Mime
View raw message