kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject [2/3] kudu git commit: [java-client] clear non-covered entries from meta cache on table open
Date Thu, 04 May 2017 01:06:37 GMT
[java-client] clear non-covered entries from meta cache on table open

Clearing non-covered range entries from the meta cache on table open
makes it easier to share client instances among different contexts
without having to worry about polluting the non-covering range cache.

Change-Id: I2d18f23045135a2276657da6808d65294be77653
Reviewed-on: http://gerrit.cloudera.org:8080/6719
Tested-by: Kudu Jenkins
Reviewed-by: Todd Lipcon <todd@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/d07ecd6d
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/d07ecd6d
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/d07ecd6d

Branch: refs/heads/master
Commit: d07ecd6ded01201c912d2e336611a6a941f48d98
Parents: c6af718
Author: Dan Burkert <danburkert@apache.org>
Authored: Fri Apr 21 17:43:42 2017 -0700
Committer: Todd Lipcon <todd@apache.org>
Committed: Thu May 4 00:10:28 2017 +0000

----------------------------------------------------------------------
 .../org/apache/kudu/client/AsyncKuduClient.java | 17 +++++++-
 .../java/org/apache/kudu/client/KuduClient.java |  8 +++-
 .../apache/kudu/client/TableLocationsCache.java | 16 ++++++++
 .../org/apache/kudu/client/TestKuduClient.java  | 42 ++++++++++++++++++++
 4 files changed, 79 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/d07ecd6d/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
index fbc181a..c2126a4 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/AsyncKuduClient.java
@@ -419,8 +419,13 @@ public class AsyncKuduClient implements AutoCloseable {
   }
 
   /**
-   * Open the table with the given name. If the table was just created, the Deferred will
only get
-   * called back when all the tablets have been successfully created.
+   * Open the table with the given name. If the table was just created, the
+   * Deferred will only get called back when all the tablets have been
+   * successfully created.
+   *
+   * New range partitions created by other clients will immediately be available
+   * after opening the table.
+   *
    * @param name table to open
    * @return a KuduTable if the table exists, else a MasterErrorException
    */
@@ -466,6 +471,14 @@ public class AsyncKuduClient implements AutoCloseable {
         // return a different, non-triggered Deferred.
         Deferred<KuduTable> d = fakeRpc.getDeferred();
         if (response.isCreateTableDone()) {
+          // When opening a table, clear the existing cached non-covered range entries.
+          // This avoids surprises where a new table instance won't be able to see the
+          // current range partitions of a table for up to the ttl.
+          TableLocationsCache cache = tableLocations.get(response.getTableId());
+          if (cache != null) {
+            cache.clearNonCoveredRangeEntries();
+          }
+
           LOG.debug("Opened table {}", name);
           fakeRpc.callback(table);
         } else {

http://git-wip-us.apache.org/repos/asf/kudu/blob/d07ecd6d/java/kudu-client/src/main/java/org/apache/kudu/client/KuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduClient.java b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduClient.java
index fc2e4da..b6ecb68 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/KuduClient.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/KuduClient.java
@@ -175,8 +175,12 @@ public class KuduClient implements AutoCloseable {
   }
 
   /**
-   * Open the table with the given name. If the table was just created, this method will
block until
-   * all its tablets have also been created.
+   * Open the table with the given name. If the table was just created, this
+   * method will block until all its tablets have also been created.
+   *
+   * New range partitions created by other clients will immediately be available
+   * after opening the table.
+   *
    * @param name table to open
    * @return a KuduTable if the table exists
    * @throws KuduException if anything went wrong

http://git-wip-us.apache.org/repos/asf/kudu/blob/d07ecd6d/java/kudu-client/src/main/java/org/apache/kudu/client/TableLocationsCache.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/main/java/org/apache/kudu/client/TableLocationsCache.java
b/java/kudu-client/src/main/java/org/apache/kudu/client/TableLocationsCache.java
index 12400d4..7b60579 100644
--- a/java/kudu-client/src/main/java/org/apache/kudu/client/TableLocationsCache.java
+++ b/java/kudu-client/src/main/java/org/apache/kudu/client/TableLocationsCache.java
@@ -19,6 +19,7 @@ package org.apache.kudu.client;
 
 import java.util.ArrayList;
 import java.util.Comparator;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.NavigableMap;
@@ -211,6 +212,21 @@ class TableLocationsCache {
     }
   }
 
+  /**
+   * Clears all non-covered range entries from the cache.
+   */
+  public void clearNonCoveredRangeEntries() {
+    rwl.writeLock().lock();
+    try {
+      Iterator<Map.Entry<byte[], Entry>> it = entries.entrySet().iterator();
+      while (it.hasNext()) {
+        if (it.next().getValue().isNonCoveredRange()) it.remove();
+      }
+    } finally {
+      rwl.writeLock().unlock();
+    }
+  }
+
   @Override
   public String toString() {
     return entries.values().toString();

http://git-wip-us.apache.org/repos/asf/kudu/blob/d07ecd6d/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
index 6d3be3e..42f0f9b 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestKuduClient.java
@@ -853,5 +853,47 @@ public class TestKuduClient extends BaseKuduTest {
     syncClient.createTable(tableName, basicSchema, new CreateTableOptions());
   }
 
+  @Test(timeout = 100000)
+  public void testOpenTableClearsNonCoveringRangePartitions() throws KuduException {
+    CreateTableOptions options = createTableOptions();
+    PartialRow lower = basicSchema.newPartialRow();
+    PartialRow upper = basicSchema.newPartialRow();
+    lower.addInt("key", 0);
+    upper.addInt("key", 1);
+    options.addRangePartition(lower, upper);
+
+    syncClient.createTable(tableName, basicSchema, options);
+    KuduTable table = syncClient.openTable(tableName);
+
+    // Count the number of tablets.
+    KuduScanToken.KuduScanTokenBuilder tokenBuilder = syncClient.newScanTokenBuilder(table);
+    List<KuduScanToken> tokens = tokenBuilder.build();
+    assertEquals(1, tokens.size());
 
+    // Add a range partition with a separate client. The new client is necessary
+    // in order to avoid clearing the meta cache as part of the alter operation.
+    try (KuduClient alterClient = new KuduClient.KuduClientBuilder(masterAddresses)
+                                                .defaultAdminOperationTimeoutMs(DEFAULT_SLEEP)
+                                                .build()) {
+      AlterTableOptions alter = new AlterTableOptions();
+      lower = basicSchema.newPartialRow();
+      upper = basicSchema.newPartialRow();
+      lower.addInt("key", 1);
+      alter.addRangePartition(lower, upper);
+      alterClient.alterTable(tableName, alter);
+      assertTrue(syncClient.isAlterTableDone(tableName));
+    }
+
+    // Count the number of tablets.  The result should still be the same, since
+    // the new tablet is still cached as a non-covered range.
+    tokenBuilder = syncClient.newScanTokenBuilder(table);
+    tokens = tokenBuilder.build();
+    assertEquals(1, tokens.size());
+
+    // Reopen the table and count the tablets again. The new tablet should now show up.
+    table = syncClient.openTable(tableName);
+    tokenBuilder = syncClient.newScanTokenBuilder(table);
+    tokens = tokenBuilder.build();
+    assertEquals(2, tokens.size());
+  }
 }


Mime
View raw message