carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [1/2] incubator-carbondata git commit: [CARBONDATA-935] Define PartitionInfo model
Date Thu, 04 May 2017 04:13:51 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/12-dev 62d1cb37b -> 558366ae8


[CARBONDATA-935] Define PartitionInfo model

fix javastyle error

modify partitioninfo and singlePartition

modify schema.thrift to add partitioningList

modify definition

variable style fix, remove partitionName and some comments

change import order

change comments

change partition info and create partition statistic

add get method in carbontable

remove partition in schema


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/620f2927
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/620f2927
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/620f2927

Branch: refs/heads/12-dev
Commit: 620f2927288e40d66e0b7267e85d8176b7fe2cb7
Parents: 62d1cb3
Author: lucao <whucaolu@gmail.com>
Authored: Tue Apr 18 14:52:18 2017 +0800
Committer: jackylk <jacky.likun@huawei.com>
Committed: Thu May 4 12:12:37 2017 +0800

----------------------------------------------------------------------
 .../core/metadata/schema/PartitionInfo.java     | 98 ++++++++++++++++++++
 .../schema/partition/AbstractPartition.java     | 42 +++++++++
 .../schema/partition/HashPartition.java         | 34 +++++++
 .../schema/partition/ListPartition.java         | 36 +++++++
 .../schema/partition/PartitionType.java         | 27 ++++++
 .../schema/partition/RangePartition.java        | 40 ++++++++
 .../core/metadata/schema/table/CarbonTable.java | 20 ++++
 .../core/metadata/schema/table/TableSchema.java | 14 +++
 .../core/stats/PartitionStatistic.java          | 59 ++++++++++++
 format/src/main/thrift/schema.thrift            | 18 ++++
 10 files changed, 388 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/620f2927/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
new file mode 100644
index 0000000..86ef3c5
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/PartitionInfo.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.metadata.schema;
+
+import java.io.Serializable;
+import java.util.List;
+
+import org.apache.carbondata.core.metadata.schema.partition.PartitionType;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+
+/**
+ * Partition Information of carbon partition table
+ */
+public class PartitionInfo implements Serializable {
+
+  /**
+   * Partition columns
+   */
+  private List<ColumnSchema> columnSchemaList;
+
+  /**
+   * partition type
+   */
+  private PartitionType partitionType;
+
+  /**
+   * Range Partition definition
+   */
+  private List<String> rangeInfo;
+
+  /**
+   * List Partition definition
+   */
+  private List<List<String>> listInfo;
+
+  /**
+   * Hash Partition numbers
+   */
+  private int hashNumber;
+
+  /**
+   * For range partition table
+   * @param columnSchemaList
+   * @param partitionType
+   */
+  public PartitionInfo(List<ColumnSchema> columnSchemaList, PartitionType partitionType)
{
+    this.columnSchemaList = columnSchemaList;
+    this.partitionType = partitionType;
+  }
+
+  public List<ColumnSchema> getColumnSchemaList() {
+    return columnSchemaList;
+  }
+
+  public PartitionType getPartitionType() {
+    return partitionType;
+  }
+
+  public void setHashNumber(int hashNumber) {
+    this.hashNumber = hashNumber;
+  }
+
+  public int getHashNumber() {
+    return hashNumber;
+  }
+
+  public void setRangeInfo(List<String> rangeInfo) {
+    this.rangeInfo = rangeInfo;
+  }
+
+  public List<String> getRangeInfo() {
+    return rangeInfo;
+  }
+
+  public void setListInfo(List<List<String>> listInfo) {
+    this.listInfo = listInfo;
+  }
+
+  public List<List<String>> getListInfo() {
+    return listInfo;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/620f2927/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/AbstractPartition.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/AbstractPartition.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/AbstractPartition.java
new file mode 100644
index 0000000..1d6337c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/AbstractPartition.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.metadata.schema.partition;
+
+public abstract class AbstractPartition {
+
+  /**
+   * Partition unique identifier
+   */
+  protected int partitionId;
+
+  /**
+   * Total row count of this partition
+   */
+  protected int rowCount;
+
+  public int getPartitionId() {
+    return partitionId;
+  }
+
+  public void setRowCount(int count) {
+    this.rowCount = count;
+  }
+
+  public int getRowCount() {
+    return rowCount;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/620f2927/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/HashPartition.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/HashPartition.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/HashPartition.java
new file mode 100644
index 0000000..6b43525
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/HashPartition.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.metadata.schema.partition;
+
+public class HashPartition extends AbstractPartition {
+
+  /**
+   * hash value for hash partition table
+   */
+  private int hashValue;
+
+  public HashPartition(int id, int hashValue) {
+    this.partitionId = id;
+    this.hashValue = hashValue;
+  }
+
+  public int getHashValue() {
+    return hashValue;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/620f2927/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/ListPartition.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/ListPartition.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/ListPartition.java
new file mode 100644
index 0000000..11a396f
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/ListPartition.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.metadata.schema.partition;
+
+import java.util.List;
+
+public class ListPartition extends AbstractPartition {
+
+  /**
+   * value list for list partition table
+   */
+  private List<String> listInfo;
+
+  public ListPartition(int id, List<String> listInfo) {
+    this.partitionId = id;
+    this.listInfo = listInfo;
+  }
+
+  public List<String> getListInfo() {
+    return listInfo;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/620f2927/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/PartitionType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/PartitionType.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/PartitionType.java
new file mode 100644
index 0000000..9a4cf4a
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/PartitionType.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.metadata.schema.partition;
+
+/**
+ * Partition type supported in carbon
+ */
+public enum PartitionType {
+  RANGE,
+  RANGE_INTERVAL,
+  LIST,
+  HASH
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/620f2927/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/RangePartition.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/RangePartition.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/RangePartition.java
new file mode 100644
index 0000000..af2b05d
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/partition/RangePartition.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.metadata.schema.partition;
+
+public class RangePartition extends AbstractPartition {
+  /**
+   * boundary value for range partition table
+   */
+  private String lowerBoundary;
+
+  private String upperBoundary;
+
+  public RangePartition(int id, String lowerBoundary, String upperBoundary) {
+    this.partitionId = id;
+    this.lowerBoundary = lowerBoundary;
+    this.upperBoundary = upperBoundary;
+  }
+
+  public String getLowerBoundary() {
+    return lowerBoundary;
+  }
+
+  public String getUpperBoundary() {
+    return upperBoundary;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/620f2927/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
index fae9960..ce5cbe4 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java
@@ -27,11 +27,13 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
 import org.apache.carbondata.core.metadata.CarbonTableIdentifier;
 import org.apache.carbondata.core.metadata.encoder.Encoding;
 import org.apache.carbondata.core.metadata.schema.BucketingInfo;
+import org.apache.carbondata.core.metadata.schema.PartitionInfo;
 import org.apache.carbondata.core.metadata.schema.table.column.CarbonColumn;
 import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
 import org.apache.carbondata.core.metadata.schema.table.column.CarbonImplicitDimension;
 import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
 import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.stats.PartitionStatistic;
 
 /**
  * Mapping class for Carbon actual table
@@ -92,6 +94,15 @@ public class CarbonTable implements Serializable {
   private Map<String, BucketingInfo> tableBucketMap;
 
   /**
+   * table partition info
+   */
+  private Map<String, PartitionInfo> tablePartitionMap;
+
+  /**
+   * statistic information of partition table
+   */
+  private PartitionStatistic partitionStatistic;
+  /**
    * tableUniqueName
    */
   private String tableUniqueName;
@@ -131,6 +142,7 @@ public class CarbonTable implements Serializable {
     this.tableImplicitDimensionsMap = new HashMap<String, List<CarbonDimension>>();
     this.tableMeasuresMap = new HashMap<String, List<CarbonMeasure>>();
     this.tableBucketMap = new HashMap<>();
+    this.tablePartitionMap = new HashMap<>();
     this.aggregateTablesName = new ArrayList<String>();
     this.createOrderColumn = new HashMap<String, List<CarbonColumn>>();
     this.tablePrimitiveDimensionsMap = new HashMap<String, List<CarbonDimension>>();
@@ -566,6 +578,14 @@ public class CarbonTable implements Serializable {
     return tableBucketMap.get(tableName);
   }
 
+  public PartitionInfo getPartitionInfo(String tableName) {
+    return tablePartitionMap.get(tableName);
+  }
+
+  public PartitionStatistic getPartitionStatistic() {
+    return partitionStatistic;
+  }
+
   /**
    * @return absolute table identifier
    */

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/620f2927/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
index 179f4d4..ad2730b 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/TableSchema.java
@@ -23,6 +23,7 @@ import java.util.Map;
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.metadata.schema.BucketingInfo;
+import org.apache.carbondata.core.metadata.schema.PartitionInfo;
 import org.apache.carbondata.core.metadata.schema.SchemaEvolution;
 import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
 
@@ -66,6 +67,11 @@ public class TableSchema implements Serializable {
    */
   private BucketingInfo bucketingInfo;
 
+  /**
+   * Information about partition type, partition column, numbers
+   */
+  private PartitionInfo partitionInfo;
+
   public TableSchema() {
     this.listOfColumns = new ArrayList<ColumnSchema>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
   }
@@ -184,4 +190,12 @@ public class TableSchema implements Serializable {
   public void setBucketingInfo(BucketingInfo bucketingInfo) {
     this.bucketingInfo = bucketingInfo;
   }
+
+  public PartitionInfo getPartitionInfo() {
+    return partitionInfo;
+  }
+
+  public void setpartitionInfo(PartitionInfo partitionInfo) {
+    this.partitionInfo = partitionInfo;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/620f2927/core/src/main/java/org/apache/carbondata/core/stats/PartitionStatistic.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/stats/PartitionStatistic.java b/core/src/main/java/org/apache/carbondata/core/stats/PartitionStatistic.java
new file mode 100644
index 0000000..44f62b1
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/stats/PartitionStatistic.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.stats;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.carbondata.core.metadata.schema.partition.AbstractPartition;
+
+public class PartitionStatistic implements Serializable {
+
+  /**
+   * total count of partitions
+   */
+  private int numberOfPartitions;
+
+  /**
+   * partition id, increase only
+   */
+  private int partitionIndex;
+
+  private Map<Integer, AbstractPartition> partitionMap;
+
+  public PartitionStatistic() {
+    this.partitionIndex = 0;
+    this.numberOfPartitions = 0;
+    this.partitionMap = new HashMap<>();
+  }
+
+  public void addNewPartition(int id, AbstractPartition partition) {
+    partitionMap.put(id, partition);
+    partitionIndex ++;
+    numberOfPartitions ++;
+  }
+
+  public void deletePartition(int id) {
+    partitionMap.remove(id);
+    numberOfPartitions --;
+  }
+
+  public int getNumberOfPartitions() {
+    return numberOfPartitions;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/620f2927/format/src/main/thrift/schema.thrift
----------------------------------------------------------------------
diff --git a/format/src/main/thrift/schema.thrift b/format/src/main/thrift/schema.thrift
index fba70ea..695333c 100644
--- a/format/src/main/thrift/schema.thrift
+++ b/format/src/main/thrift/schema.thrift
@@ -49,6 +49,12 @@ enum Encoding{
 	DIRECT_DICTIONARY = 5; // Identifies that a column is direct dictionary encoded
 }
 
+enum PartitionType{
+  RANGE = 0;
+  RANGE_INTERVAL = 1;
+  LIST = 2;
+  HASH = 3;
+}
 
 /**
  * Description of a Column for both dimension and measure
@@ -121,6 +127,17 @@ struct SchemaEvolution{
 }
 
 /**
+ * Partition information of table
+ */
+struct PartitionInfo{
+    1: required list<ColumnSchema> partition_columns;
+    2: required PartitionType partition_type;
+    3: optional i32 hash_number;  // number of partitions defined in hash partition table
+    4: optional list<list<string>> list_info; // value list of list partition
table
+    5: optional list<string> range_info;  // range value list of range partition table
+}
+
+/**
  * Bucketing information of fields on table
  */
 struct BucketingInfo{
@@ -137,6 +154,7 @@ struct TableSchema{
 	3: required SchemaEvolution schema_evolution; // History of schema evolution of this table
   4: optional map<string,string> tableProperties; // Table properties configured by
the user
   5: optional BucketingInfo bucketingInfo; // Bucketing information
+  6: optional PartitionInfo partitionInfo; // Partition information
 }
 
 struct TableInfo{


Mime
View raw message