carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ravipes...@apache.org
Subject carbondata git commit: [CARBONDATA-1795] Fix code issues of examples
Date Wed, 22 Nov 2017 17:21:25 GMT
Repository: carbondata
Updated Branches:
  refs/heads/master 9a69d638b -> 06204d40c


[CARBONDATA-1795] Fix code issues of examples

Fix code issue of examples:
CarbonDataFrameExample,CarbonSortColumnsExample,HadoopFileExample

This closes #1551


Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/06204d40
Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/06204d40
Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/06204d40

Branch: refs/heads/master
Commit: 06204d40c7339d994638aaa9bb95a145b823b2da
Parents: 9a69d63
Author: chenliang613 <chenliang613@huawei.com>
Authored: Wed Nov 22 19:29:41 2017 +0800
Committer: Ravindra Pesala <ravi.pesala@gmail.com>
Committed: Wed Nov 22 22:50:55 2017 +0530

----------------------------------------------------------------------
 .../apache/carbondata/examples/CarbonDataFrameExample.scala | 1 +
 .../carbondata/examples/CarbonSortColumnsExample.scala      | 5 +++--
 .../org/apache/carbondata/examples/HadoopFileExample.scala  | 9 ++++++---
 .../apache/spark/sql/CarbonDatasourceHadoopRelation.scala   | 3 ++-
 4 files changed, 12 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/carbondata/blob/06204d40/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
index 2450b49..fe15659 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonDataFrameExample.scala
@@ -73,6 +73,7 @@ object CarbonDataFrameExample {
     val carbondf = spark.read
       .format("carbondata")
       .schema(customSchema)
+      // .option("dbname", "db_name") the system will use "default" as dbname if not set
this option
       .option("tableName", "carbon_table")
       .load()
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/06204d40/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSortColumnsExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSortColumnsExample.scala
b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSortColumnsExample.scala
index 2f6322c..cea6341 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSortColumnsExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSortColumnsExample.scala
@@ -46,9 +46,9 @@ object CarbonSortColumnsExample {
       .config("spark.driver.host", "localhost")
       .getOrCreateCarbonSession(storeLocation)
 
-    spark.sparkContext.setLogLevel("WARN")
+    spark.sparkContext.setLogLevel("ERROR")
 
-    spark.sql("DROP TABLE IF EXISTS sort_columns_table")
+    spark.sql("DROP TABLE IF EXISTS no_sort_columns_table")
 
     // Create table with no sort columns
     spark.sql(
@@ -73,6 +73,7 @@ object CarbonSortColumnsExample {
     // Create table with sort columns
     // you can specify any columns to sort columns for building MDX index, remark: currently
     // sort columns don't support "FLOAT, DOUBLE, DECIMAL"
+    spark.sql("DROP TABLE IF EXISTS sort_columns_table")
     spark.sql(
       s"""
          | CREATE TABLE sort_columns_table(

http://git-wip-us.apache.org/repos/asf/carbondata/blob/06204d40/examples/spark2/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala
b/examples/spark2/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala
index d14469a..d75abc2 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala
@@ -19,7 +19,8 @@ package org.apache.carbondata.examples
 
 import org.apache.hadoop.conf.Configuration
 
-import org.apache.carbondata.hadoop.{CarbonInputFormat, CarbonProjection}
+import org.apache.carbondata.hadoop.api.CarbonTableInputFormat
+import org.apache.carbondata.hadoop.CarbonProjection
 
 // scalastyle:off println
 object HadoopFileExample {
@@ -33,11 +34,13 @@ object HadoopFileExample {
     projection.addColumn("c1")  // column c1
     projection.addColumn("c3")  // column c3
     val conf = new Configuration()
-    CarbonInputFormat.setColumnProjection(conf, projection)
+    CarbonTableInputFormat.setColumnProjection(conf, projection)
+    CarbonTableInputFormat.setDatabaseName(conf, "default")
+    CarbonTableInputFormat.setTableName(conf, "carbon1")
 
     val sc = spark.sparkContext
     val input = sc.newAPIHadoopFile(s"${ExampleUtils.storeLocation}/default/carbon1",
-      classOf[CarbonInputFormat[Array[Object]]],
+      classOf[CarbonTableInputFormat[Array[Object]]],
       classOf[Void],
       classOf[Array[Object]],
       conf)

http://git-wip-us.apache.org/repos/asf/carbondata/blob/06204d40/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
index c800055..9a48a7c 100644
--- a/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
+++ b/integration/spark2/src/main/scala/org/apache/spark/sql/CarbonDatasourceHadoopRelation.scala
@@ -46,7 +46,8 @@ case class CarbonDatasourceHadoopRelation(
   extends BaseRelation with InsertableRelation {
 
   lazy val identifier: AbsoluteTableIdentifier = AbsoluteTableIdentifier.from(paths.head,
-    parameters("dbname"), parameters("tablename"))
+    parameters.getOrElse("dbname", GetDB.getDatabaseName(None, sparkSession)),
+    parameters("tablename"))
   lazy val databaseName: String = carbonTable.getDatabaseName
   lazy val tableName: String = carbonTable.getTableName
   CarbonSession.updateSessionInfoToCurrentThread(sparkSession)


Mime
View raw message