kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From granthe...@apache.org
Subject [kudu] 02/04: [backup] Ensure the last metadata is restored
Date Thu, 02 May 2019 12:45:53 GMT
This is an automated email from the ASF dual-hosted git repository.

granthenke pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git

commit 9b8be6ab480d28c19e867762b4f3049d4ff48512
Author: Grant Henke <granthenke@apache.org>
AuthorDate: Fri Apr 26 12:37:36 2019 -0500

    [backup] Ensure the last metadata is restored
    
    When creating a table, ensure the last metadata is
    restored instead of the first.
    
    This is the first step to allow table alterations. Follow on
    patches will take care of metadata validation and
    alteration handling.
    
    Change-Id: I08dce740d6e5063708cce7f690718f895348fc54
    Reviewed-on: http://gerrit.cloudera.org:8080/13132
    Tested-by: Kudu Jenkins
    Reviewed-by: Adar Dembo <adar@cloudera.com>
    Reviewed-by: Mike Percy <mpercy@apache.org>
---
 .../src/main/scala/org/apache/kudu/backup/KuduRestore.scala  | 12 +++++++++---
 .../test/scala/org/apache/kudu/backup/TestKuduBackup.scala   |  3 ++-
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduRestore.scala b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduRestore.scala
index c4b5f8b..eb2a03e 100644
--- a/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduRestore.scala
+++ b/java/kudu-backup/src/main/scala/org/apache/kudu/backup/KuduRestore.scala
@@ -48,6 +48,7 @@ object KuduRestore {
     // TODO (KUDU-2787): Handle single table failures.
     options.tables.foreach { tableName =>
       val graph = io.readBackupGraph(tableName).filterByTime(options.timestampMs)
+      val lastMetadata = graph.restorePath.backups.last.metadata
       graph.restorePath.backups.foreach { backup =>
         log.info(s"Restoring table $tableName from path: ${backup.path}")
         val isFullRestore = backup.metadata.getFromMs == 0
@@ -59,16 +60,21 @@ object KuduRestore {
         if (isFullRestore) {
           if (options.createTables) {
             log.info(s"Creating restore table $restoreName")
-            createTableRangePartitionByRangePartition(restoreName, backup.metadata, context)
+            // We use the last schema in the restore path when creating the table to
+            // ensure the table is created in its final state.
+            createTableRangePartitionByRangePartition(restoreName, lastMetadata, context)
           }
         }
+
+        val backupSchema = io.dataSchema(TableMetadata.getKuduSchema(backup.metadata))
+        val rowActionCol = backupSchema.fields.last.name
+
         val table = context.syncClient.openTable(restoreName)
         val restoreSchema = io.dataSchema(table.getSchema)
-        val rowActionCol = restoreSchema.fields.last.name
 
         var data = session.sqlContext.read
           .format(backup.metadata.getDataFormat)
-          .schema(restoreSchema)
+          .schema(backupSchema)
           .load(backup.path.toString)
           // Default the the row action column with a value of "UPSERT" so that the
           // rows from a full backup, which don't have a row action, are upserted.
diff --git a/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala b/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
index 7533251..c2b5356 100644
--- a/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
+++ b/java/kudu-backup/src/test/scala/org/apache/kudu/backup/TestKuduBackup.scala
@@ -126,7 +126,8 @@ class TestKuduBackup extends KuduTestSuite {
   def testForceIncrementalBackup() {
     insertRows(table, 100) // Insert data into the default test table.
     // Set beforeMs so we can force an incremental at this time.
-    val beforeMs = System.currentTimeMillis() + 1
+    val beforeMs = System.currentTimeMillis()
+    Thread.sleep(1) // Ensure the next insert is after beforeMs.
     insertRows(table, 100, 100) // Insert more data.
 
     // Force an incremental backup without a full backup.


Mime
View raw message