kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a...@apache.org
Subject [kudu] branch master updated: KUDU-2823 [java client] Support setting dimension for the newly created tablet
Date Mon, 22 Jul 2019 19:11:09 GMT
This is an automated email from the ASF dual-hosted git repository.

adar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git


The following commit(s) were added to refs/heads/master by this push:
     new c6fa433  KUDU-2823 [java client] Support setting dimension for the newly created
tablet
c6fa433 is described below

commit c6fa43362f79b4adc1b489d04d5b0820b44b68ff
Author: oclarms <oclarms@gmail.com>
AuthorDate: Wed Jul 17 17:38:13 2019 +0800

    KUDU-2823 [java client] Support setting dimension for the newly created tablet
    
    Change-Id: I5fc7797011df578000544147938bd15a6027f4c8
    Reviewed-on: http://gerrit.cloudera.org:8080/13875
    Tested-by: Kudu Jenkins
    Reviewed-by: Adar Dembo <adar@cloudera.com>
---
 .../org/apache/kudu/client/AlterTableOptions.java  | 40 ++++++++++++++++++++++
 .../org/apache/kudu/client/AsyncKuduClient.java    |  3 ++
 .../org/apache/kudu/client/CreateTableOptions.java | 19 ++++++++++
 .../java/org/apache/kudu/client/LocatedTablet.java |  4 +++
 .../java/org/apache/kudu/client/TestKuduTable.java | 33 ++++++++++++++++++
 src/kudu/master/catalog_manager.cc                 |  7 ++++
 src/kudu/master/master.proto                       |  6 +++-
 7 files changed, 111 insertions(+), 1 deletion(-)

diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java
b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java
index b4fbbab..8071257 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AlterTableOptions.java
@@ -283,6 +283,43 @@ public class AlterTableOptions {
                                              PartialRow upperBound,
                                              RangePartitionBound lowerBoundType,
                                              RangePartitionBound upperBoundType) {
+    return addRangePartition(lowerBound, upperBound, null, lowerBoundType, upperBoundType);
+  }
+
+  /**
+   * Add a range partition to the table with dimension label.
+   *
+   * If either row is empty, then that end of the range will be unbounded. If a range column
is
+   * missing a value, the logical minimum value for that column type will be used as the
default.
+   *
+   * Multiple range partitions may be added as part of a single alter table transaction by
calling
+   * this method multiple times. Added range partitions must not overlap with each
+   * other or any existing range partitions (unless the existing range partitions are dropped
as
+   * part of the alter transaction first). The lower bound must be less than the upper bound.
+   *
+   * This client will immediately be able to write and scan the new tablets when the alter
table
+   * operation returns success, however other existing clients may have to wait for a timeout
period
+   * to elapse before the tablets become visible. This period is configured by the master's
+   * 'table_locations_ttl_ms' flag, and defaults to 5 minutes.
+   *
+   * By default, the master will try to place newly created tablet replicas on tablet
+   * servers with a small number of tablet replicas. If the dimension label is provided,
+   * newly created replicas will be evenly distributed in the cluster based on the dimension
+   * label. In other words, the master will try to place newly created tablet replicas on
+   * tablet servers with a small number of tablet replicas belonging to this dimension label.
+   *
+   * @param lowerBound lower bound, may be empty but not null
+   * @param upperBound upper bound, may be empty but not null
+   * @param dimensionLabel the dimension label for the tablet to be created
+   * @param lowerBoundType the type of the lower bound, either inclusive or exclusive
+   * @param upperBoundType the type of the upper bound, either inclusive or exclusive
+   * @return this instance
+   */
+  public AlterTableOptions addRangePartition(PartialRow lowerBound,
+                                             PartialRow upperBound,
+                                             String dimensionLabel,
+                                             RangePartitionBound lowerBoundType,
+                                             RangePartitionBound upperBoundType) {
     Preconditions.checkNotNull(lowerBound);
     Preconditions.checkNotNull(upperBound);
     Preconditions.checkArgument(lowerBound.getSchema().equals(upperBound.getSchema()));
@@ -294,6 +331,9 @@ public class AlterTableOptions {
     builder.setRangeBounds(
         new Operation.OperationsEncoder()
             .encodeLowerAndUpperBounds(lowerBound, upperBound, lowerBoundType, upperBoundType));
+    if (dimensionLabel != null) {
+      builder.setDimensionLabel(dimensionLabel);
+    }
     step.setAddRangePartition(builder);
     if (!pb.hasSchema()) {
       pb.setSchema(ProtobufHelper.schemaToPb(lowerBound.getSchema(),
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
index 1de7306..d908692 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
@@ -2303,6 +2303,9 @@ public class AsyncKuduClient implements AutoCloseable {
         Master.TabletLocationsPB.ReplicaPB.Builder builder = Master.TabletLocationsPB.ReplicaPB.newBuilder();
         builder.setRole(replica.getRole());
         builder.setTsInfo(tsInfo);
+        if (replica.hasDimensionLabel()) {
+          builder.setDimensionLabel(replica.getDimensionLabel());
+        }
         replicas.add(builder.build());
       }
 
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java
b/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java
index 1790037..9372ad9 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/CreateTableOptions.java
@@ -192,6 +192,25 @@ public class CreateTableOptions {
   }
 
   /**
+   * Sets the dimension label for all tablets created at table creation time.
+   *
+   * By default, the master will try to place newly created tablet replicas on tablet
+   * servers with a small number of tablet replicas. If the dimension label is provided,
+   * newly created replicas will be evenly distributed in the cluster based on the dimension
+   * label. In other words, the master will try to place newly created tablet replicas on
+   * tablet servers with a small number of tablet replicas belonging to this dimension label.
+   *
+   * @param dimensionLabel the dimension label for the tablet to be created.
+   * @return this instance
+   */
+  public CreateTableOptions setDimensionLabel(String dimensionLabel) {
+    Preconditions.checkArgument(dimensionLabel != null,
+        "dimension label must not be null");
+    pb.setDimensionLabel(dimensionLabel);
+    return this;
+  }
+
+  /**
    * Sets the table's extra configuration properties.
    *
    * If the value of the kv pair is empty, the property will be ignored.
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/LocatedTablet.java b/java/kudu-client/src/main/java/org/apache/kudu/client/LocatedTablet.java
index ae31903..837156e 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/LocatedTablet.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/LocatedTablet.java
@@ -137,6 +137,10 @@ public class LocatedTablet {
       return pb.getRole().toString();
     }
 
+    public String getDimensionLabel() {
+      return pb.hasDimensionLabel() ? pb.getDimensionLabel() : null;
+    }
+
     @Override
     public String toString() {
       return pb.toString();
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduTable.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduTable.java
index d1332e3..064b3cc 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduTable.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduTable.java
@@ -30,7 +30,9 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
@@ -746,4 +748,35 @@ public class TestKuduTable {
     assertEquals("wrong value comment post alter", "",
         table2.getSchema().getColumn("value").getComment());
   }
+
+  @Test(timeout = 100000)
+  public void testDimensionLabel() throws Exception {
+    // Create a table with dimension label.
+    KuduTable table = client.createTable(tableName, basicSchema,
+        getBasicTableOptionsWithNonCoveredRange().setDimensionLabel("labelA"));
+
+    // Add a range partition to the table with dimension label.
+    AlterTableOptions ato = new AlterTableOptions();
+    PartialRow bLowerBound = BASIC_SCHEMA.newPartialRow();
+    bLowerBound.addInt("key", 300);
+    PartialRow bUpperBound = BASIC_SCHEMA.newPartialRow();
+    bUpperBound.addInt("key", 400);
+    ato.addRangePartition(bLowerBound, bUpperBound, "labelB",
+                          RangePartitionBound.INCLUSIVE_BOUND,
+                          RangePartitionBound.EXCLUSIVE_BOUND);
+    client.alterTable(tableName, ato);
+
+    Map<String, Integer> dimensionMap = new HashMap<>();
+    for (LocatedTablet tablet : table.getTabletsLocations(DEFAULT_SLEEP)) {
+      for (LocatedTablet.Replica replica : tablet.getReplicas()) {
+        Integer number = dimensionMap.get(replica.getDimensionLabel());
+        if (number == null) {
+          number = 0;
+        }
+        dimensionMap.put(replica.getDimensionLabel(), number + 1);
+      }
+    }
+    assertEquals(9, dimensionMap.get("labelA").intValue());
+    assertEquals(3, dimensionMap.get("labelB").intValue());
+  }
 }
diff --git a/src/kudu/master/catalog_manager.cc b/src/kudu/master/catalog_manager.cc
index 1ea7408..46ebf25 100644
--- a/src/kudu/master/catalog_manager.cc
+++ b/src/kudu/master/catalog_manager.cc
@@ -4763,6 +4763,10 @@ Status CatalogManager::BuildLocationsForTablet(
     };
 
     auto role = GetParticipantRole(peer, cstate);
+    optional<string> dimension = none;
+    if (l_tablet.data().pb.has_dimension_label()) {
+      dimension = l_tablet.data().pb.dimension_label();
+    }
     if (ts_infos_dict) {
       int idx = *ComputeIfAbsent(
           &ts_infos_dict->uuid_to_idx, peer.permanent_uuid(),
@@ -4776,6 +4780,9 @@ Status CatalogManager::BuildLocationsForTablet(
       auto* interned_replica_pb = locs_pb->add_interned_replicas();
       interned_replica_pb->set_ts_info_idx(idx);
       interned_replica_pb->set_role(role);
+      if (dimension) {
+        interned_replica_pb->set_dimension_label(*dimension);
+      }
     } else {
       TabletLocationsPB_ReplicaPB* replica_pb = locs_pb->add_replicas();
       replica_pb->set_allocated_ts_info(make_tsinfo_pb().release());
diff --git a/src/kudu/master/master.proto b/src/kudu/master/master.proto
index c74f3f1..448e074 100644
--- a/src/kudu/master/master.proto
+++ b/src/kudu/master/master.proto
@@ -368,16 +368,20 @@ message TSHeartbeatResponsePB {
 //////////////////////////////
 
 message TabletLocationsPB {
-  // DEPRECATED: new clients should prefer the 'Interned' type below.
+  // DEPRECATED.
+  // TODO: new clients should prefer the 'Interned' type below.
+  // Remove 'ReplicaPB' when we stop using it internally.
   message ReplicaPB {
     required TSInfoPB ts_info = 1;
     required consensus.RaftPeerPB.Role role = 2;
+    optional string dimension_label = 3;
   }
 
   message InternedReplicaPB {
     // Index into the 'ts_infos' list in the top-level RPC response.
     required uint32 ts_info_idx = 1;
     required consensus.RaftPeerPB.Role role = 2;
+    optional string dimension_label = 3;
   }
 
   required bytes tablet_id = 1;


Mime
View raw message