kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mpe...@apache.org
Subject kudu git commit: KUDU-2033 (part 1). Add write and stop/start in a loop to TestLeaderFailover
Date Tue, 08 Aug 2017 18:34:30 GMT
Repository: kudu
Updated Branches:
  refs/heads/master d1f53cc32 -> 4a0417018


KUDU-2033 (part 1). Add write and stop/start in a loop to TestLeaderFailover

TestLeaderFailover is made more robust by adding a write loop in which
the tablet leader is killed and restarted. The main idea is to check
that the writes complete and can be read after the leader is killed.
Done in a loop so we can be sure that we stay stable through multiple
stops and restarts of the leader tablet.

Change-Id: Ie93a15084a4c4c0748dc74c005b8313f443a5ba9
Reviewed-on: http://gerrit.cloudera.org:8080/7456
Reviewed-by: Mike Percy <mpercy@apache.org>
Tested-by: Mike Percy <mpercy@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/4a041701
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/4a041701
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/4a041701

Branch: refs/heads/master
Commit: 4a0417018b2833ee9f308a294b319d8c7570990f
Parents: d1f53cc
Author: efancher <efan@cloudera.com>
Authored: Tue Jul 18 14:06:16 2017 -0700
Committer: Mike Percy <mpercy@apache.org>
Committed: Tue Aug 8 18:34:11 2017 +0000

----------------------------------------------------------------------
 .../org/apache/kudu/client/BaseKuduTest.java    |  2 +-
 .../kudu/client/TestMultipleLeaderFailover.java | 95 ++++++++++++++++++++
 .../org/apache/kudu/util/AssertHelpers.java     | 41 +++++++++
 3 files changed, 137 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/4a041701/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java b/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java
index 1caa21c..a0a5832 100644
--- a/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/BaseKuduTest.java
@@ -382,7 +382,7 @@ public class BaseKuduTest {
    * @return the RPC port of the given tablet's leader tablet server.
    * @throws Exception
    */
-  private static int findLeaderTabletServerPort(LocatedTablet tablet)
+  protected static int findLeaderTabletServerPort(LocatedTablet tablet)
       throws Exception {
     LocatedTablet.Replica leader = null;
     DeadlineTracker deadlineTracker = new DeadlineTracker();

http://git-wip-us.apache.org/repos/asf/kudu/blob/4a041701/java/kudu-client/src/test/java/org/apache/kudu/client/TestMultipleLeaderFailover.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/client/TestMultipleLeaderFailover.java
b/java/kudu-client/src/test/java/org/apache/kudu/client/TestMultipleLeaderFailover.java
new file mode 100644
index 0000000..526fdb3
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/client/TestMultipleLeaderFailover.java
@@ -0,0 +1,95 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.kudu.client;
+
+import static org.apache.kudu.util.AssertHelpers.assertEventuallyTrue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.util.List;
+import org.apache.kudu.util.AssertHelpers.BooleanExpression;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestMultipleLeaderFailover extends BaseKuduTest {
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    BaseKuduTest.setUpBeforeClass();
+  }
+
+  private void waitUntilRowCount(final KuduTable table, final int rowCount, long timeoutMs)
+      throws Exception {
+    assertEventuallyTrue(String.format("Read count should be %s", rowCount),
+        new BooleanExpression() {
+          @Override
+          public boolean get() throws Exception {
+            AsyncKuduScanner scanner = client.newScannerBuilder(table).build();
+            int read_count = countRowsInScan(scanner);
+            return read_count == rowCount;
+          }
+        }, timeoutMs);
+  }
+
+  /**
+   * This test writes 3 rows. Then in a loop, it kills the leader, then tries to write inner_row
+   * rows, and finally restarts the tablet server it killed. Verifying with a read as it
goes.
+   * Finally it counts to make sure we have total_rows_to_insert of them.
+   */
+  @Test(timeout = 100000)
+  public void testMultipleFailover() throws Exception {
+    KuduTable table;
+    CreateTableOptions builder = getBasicCreateTableOptions();
+    String tableName =
+        TestMultipleLeaderFailover.class.getName() + "-" + System.currentTimeMillis();
+    createTable(tableName, basicSchema, builder);
+
+    table = openTable(tableName);
+    KuduSession session = syncClient.newSession();
+    final int ROWS_PER_ITERATION = 3;
+    final int NUM_ITERATIONS = 10;
+    final int TOTAL_ROWS_TO_INSERT = ROWS_PER_ITERATION + NUM_ITERATIONS * ROWS_PER_ITERATION;
+
+    for (int i = 0; i < ROWS_PER_ITERATION; i++) {
+      session.apply(createBasicSchemaInsert(table, i));
+    }
+
+    waitUntilRowCount(table, ROWS_PER_ITERATION, DEFAULT_SLEEP);
+
+    int currentRows = ROWS_PER_ITERATION;
+    for (int i = 0; i < NUM_ITERATIONS; i++) {
+      List<LocatedTablet> tablets = table.getTabletsLocations(DEFAULT_SLEEP);
+      assertEquals(1, tablets.size());
+      int leaderPort = findLeaderTabletServerPort(tablets.get(0));
+      miniCluster.killTabletServerOnPort(leaderPort);
+
+      for (int j = 0; j < ROWS_PER_ITERATION; j++) {
+        OperationResponse resp = session.apply(createBasicSchemaInsert(table, currentRows));
+        if (resp.hasRowError()) {
+          fail("Encountered a row error " + resp.getRowError());
+        }
+        currentRows++;
+      }
+
+      miniCluster.restartDeadTabletServerOnPort(leaderPort);
+      // Read your writes hasn't been enabled, so we need to use a helper function to poll.
+      waitUntilRowCount(table, currentRows, DEFAULT_SLEEP);
+
+    }
+    waitUntilRowCount(table, TOTAL_ROWS_TO_INSERT, DEFAULT_SLEEP);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kudu/blob/4a041701/java/kudu-client/src/test/java/org/apache/kudu/util/AssertHelpers.java
----------------------------------------------------------------------
diff --git a/java/kudu-client/src/test/java/org/apache/kudu/util/AssertHelpers.java b/java/kudu-client/src/test/java/org/apache/kudu/util/AssertHelpers.java
new file mode 100644
index 0000000..ff7d67e
--- /dev/null
+++ b/java/kudu-client/src/test/java/org/apache/kudu/util/AssertHelpers.java
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.kudu.util;
+
+import static org.junit.Assert.assertTrue;
+
+public class AssertHelpers {
+  public interface BooleanExpression {
+    boolean get() throws Exception;
+  }
+
+  // A looping check. It's mainly useful for scanners, since writes may take a little time
to show
+  // up.
+  public static void assertEventuallyTrue(String description, BooleanExpression expression,
+                                          long timeoutMillis) throws Exception {
+    long deadlineNanos = System.nanoTime() + timeoutMillis * 1000000;
+    boolean success;
+
+    do {
+      success = expression.get();
+      if (success) break;
+      Thread.sleep(50); // Sleep for 50ms
+    } while (System.nanoTime() < deadlineNanos);
+
+    assertTrue(description, success);
+  }
+}


Mime
View raw message