carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [1/2] incubator-carbondata git commit: add example:directly write dataframe to carbonfile without temp CSV
Date Mon, 26 Dec 2016 02:47:12 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master a0e66461b -> 2ee23acf4


add example:directly write dataframe to carbonfile without temp CSV


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/9bbc937a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/9bbc937a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/9bbc937a

Branch: refs/heads/master
Commit: 9bbc937adaadb88c4395b295916036724491bb8f
Parents: a0e6646
Author: chenliang613 <chenliang613@apache.org>
Authored: Sun Dec 25 12:04:26 2016 +0800
Committer: jackylk <jacky.likun@huawei.com>
Committed: Mon Dec 26 10:46:19 2016 +0800

----------------------------------------------------------------------
 .../carbondata/examples/AllDictionaryExample.scala     |  2 +-
 .../apache/carbondata/examples/HadoopFileExample.scala |  2 +-
 .../org/apache/carbondata/examples/PerfTest.scala      |  6 ++----
 .../apache/carbondata/examples/util/ExampleUtils.scala | 13 ++++++++++++-
 4 files changed, 16 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9bbc937a/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
b/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
index 9fecadb..4dcc868 100644
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
+++ b/examples/spark/src/main/scala/org/apache/carbondata/examples/AllDictionaryExample.scala
@@ -23,7 +23,7 @@ import org.apache.carbondata.examples.util.{AllDictionaryUtil, ExampleUtils}
 object AllDictionaryExample {
 
   def main(args: Array[String]) {
-    val cc = ExampleUtils.createCarbonContext("CarbonExample")
+    val cc = ExampleUtils.createCarbonContext("AllDictionaryExample")
     val testData = ExampleUtils.currentPath + "/src/main/resources/data.csv"
     val csvHeader = "ID,date,country,name,phonetype,serialname,salary"
     val dictCol = "|date|country|name|phonetype|serialname|"

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9bbc937a/examples/spark/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala
b/examples/spark/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala
index 329b3c9..292a3b5 100644
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala
+++ b/examples/spark/src/main/scala/org/apache/carbondata/examples/HadoopFileExample.scala
@@ -24,7 +24,7 @@ import org.apache.carbondata.hadoop.CarbonInputFormat
 object HadoopFileExample {
 
   def main(args: Array[String]): Unit = {
-    val cc = ExampleUtils.createCarbonContext("DataFrameAPIExample")
+    val cc = ExampleUtils.createCarbonContext("HadoopFileExample")
     ExampleUtils.writeSampleCarbonFile(cc, "carbon1")
 
     val sc = cc.sparkContext

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9bbc937a/examples/spark/src/main/scala/org/apache/carbondata/examples/PerfTest.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/PerfTest.scala b/examples/spark/src/main/scala/org/apache/carbondata/examples/PerfTest.scala
index b1f6b24..824730f 100644
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/PerfTest.scala
+++ b/examples/spark/src/main/scala/org/apache/carbondata/examples/PerfTest.scala
@@ -145,15 +145,13 @@ class QueryRunner(sqlContext: SQLContext, dataFrame: DataFrame, datasources:
Seq
 
   def shutDown(): Unit = {
     // drop all tables and temp files
-    datasources.foreach { datasource =>
-      datasource match {
-        case "parquet" | "orc" =>
+    datasources.foreach {
+        case datasource @ ("parquet" | "orc") =>
           val f = new File(PerfTest.savePath(datasource))
           if (f.exists()) f.delete()
         case "carbon" =>
           sqlContext.sql(s"DROP TABLE IF EXISTS ${PerfTest.makeTableName("carbon")}")
         case _ => sys.error("unsupported data source")
-      }
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9bbc937a/examples/spark/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
----------------------------------------------------------------------
diff --git a/examples/spark/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
b/examples/spark/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
index 653d78b..7649ac3 100644
--- a/examples/spark/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
+++ b/examples/spark/src/main/scala/org/apache/carbondata/examples/util/ExampleUtils.scala
@@ -81,7 +81,7 @@ object ExampleUtils {
         .map(x => ("a", "b", x))
         .toDF("c1", "c2", "c3")
 
-    // save dataframe to carbon file
+    // save dataframe to carbon file:(df->csv->carbon file)
     df.write
         .format("carbondata")
         .option("tableName", tableName)
@@ -89,6 +89,17 @@ object ExampleUtils {
         .option("useKettle", "false")
         .mode(mode)
         .save()
+
+    // save dataframe directl to carbon file without tempCSV
+    df.write
+      .format("carbondata")
+      .option("tableName", tableName)
+      .option("compress", "true")
+      .option("useKettle", "false")
+      .option("tempCSV", "false")
+      .mode(mode)
+      .save()
+
   }
 }
 // scalastyle:on println


Mime
View raw message