kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a...@apache.org
Subject [2/3] kudu git commit: fs: allow format with user-specified uuid
Date Tue, 16 Aug 2016 01:38:19 GMT
fs: allow format with user-specified uuid

The CLI tool is going to use this for the "handling permanent failure"
workflow.

Change-Id: Ib4189d6150a263b7dde304dd19a449d3c0ab6c8c
Reviewed-on: http://gerrit.cloudera.org:8080/3968
Reviewed-by: Todd Lipcon <todd@apache.org>
Tested-by: Adar Dembo <adar@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/1bce7312
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/1bce7312
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/1bce7312

Branch: refs/heads/master
Commit: 1bce73126799cb046755c0555bbe5be7622c9ace
Parents: aa94185
Author: Adar Dembo <adar@cloudera.com>
Authored: Thu Aug 11 21:14:36 2016 -0700
Committer: Adar Dembo <adar@cloudera.com>
Committed: Tue Aug 16 01:32:58 2016 +0000

----------------------------------------------------------------------
 src/kudu/fs/fs_manager-test.cc      | 25 ++++++++++++++--
 src/kudu/fs/fs_manager.cc           | 17 ++++++++---
 src/kudu/fs/fs_manager.h            | 11 +++++--
 src/kudu/util/CMakeLists.txt        |  1 +
 src/kudu/util/oid_generator-test.cc | 50 ++++++++++++++++++++++++++++++++
 src/kudu/util/oid_generator.cc      | 37 +++++++++++++++++++----
 src/kudu/util/oid_generator.h       | 12 ++++++++
 7 files changed, 139 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/1bce7312/src/kudu/fs/fs_manager-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/fs_manager-test.cc b/src/kudu/fs/fs_manager-test.cc
index 922e617..21bb460 100644
--- a/src/kudu/fs/fs_manager-test.cc
+++ b/src/kudu/fs/fs_manager-test.cc
@@ -21,12 +21,15 @@
 
 #include "kudu/fs/block_manager.h"
 #include "kudu/fs/fs_manager.h"
+#include "kudu/gutil/strings/substitute.h"
 #include "kudu/gutil/strings/util.h"
 #include "kudu/util/metrics.h"
+#include "kudu/util/oid_generator.h"
 #include "kudu/util/test_macros.h"
 #include "kudu/util/test_util.h"
 
 using std::shared_ptr;
+using strings::Substitute;
 
 namespace kudu {
 
@@ -155,7 +158,7 @@ TEST_F(FsManagerTestBase, TestCannotUseNonEmptyFsRoot) {
 }
 
 TEST_F(FsManagerTestBase, TestEmptyWALPath) {
-  ReinitFsManager("", vector<string>());
+  ReinitFsManager("", {});
   Status s = fs_manager()->CreateInitialFileSystemLayout();
   ASSERT_TRUE(s.IsIOError());
   ASSERT_STR_CONTAINS(s.ToString(), "directory (fs_wal_dir) not provided");
@@ -165,7 +168,7 @@ TEST_F(FsManagerTestBase, TestOnlyWALPath) {
   string path = GetTestPath("new_fs_root");
   ASSERT_OK(env_->CreateDir(path));
 
-  ReinitFsManager(path, vector<string>());
+  ReinitFsManager(path, {});
   ASSERT_OK(fs_manager()->CreateInitialFileSystemLayout());
   ASSERT_TRUE(HasPrefixString(fs_manager()->GetWalsRootDir(), path));
   ASSERT_TRUE(HasPrefixString(fs_manager()->GetConsensusMetadataDir(), path));
@@ -175,4 +178,22 @@ TEST_F(FsManagerTestBase, TestOnlyWALPath) {
   ASSERT_TRUE(HasPrefixString(data_dirs[0], path));
 }
 
+TEST_F(FsManagerTestBase, TestFormatWithSpecificUUID) {
+  string path = GetTestPath("new_fs_root");
+  ReinitFsManager(path, {});
+
+  // Use an invalid uuid at first.
+  string uuid = "not_a_valid_uuid";
+  Status s = fs_manager()->CreateInitialFileSystemLayout(uuid);
+  ASSERT_TRUE(s.IsInvalidArgument());
+  ASSERT_STR_CONTAINS(s.ToString(), Substitute("invalid uuid $0", uuid));
+
+  // Now use a valid one.
+  ObjectIdGenerator oid_generator;
+  uuid = oid_generator.Next();
+  ASSERT_OK(fs_manager()->CreateInitialFileSystemLayout(uuid));
+  ASSERT_OK(fs_manager()->Open());
+  ASSERT_EQ(uuid, fs_manager()->uuid());
+}
+
 } // namespace kudu

http://git-wip-us.apache.org/repos/asf/kudu/blob/1bce7312/src/kudu/fs/fs_manager.cc
----------------------------------------------------------------------
diff --git a/src/kudu/fs/fs_manager.cc b/src/kudu/fs/fs_manager.cc
index 117389f..86e5d4f 100644
--- a/src/kudu/fs/fs_manager.cc
+++ b/src/kudu/fs/fs_manager.cc
@@ -22,6 +22,7 @@
 #include <map>
 #include <unordered_set>
 
+#include <boost/optional.hpp>
 #include <glog/logging.h>
 #include <glog/stl_logging.h>
 #include <google/protobuf/message.h>
@@ -244,7 +245,7 @@ Status FsManager::Open() {
   return Status::OK();
 }
 
-Status FsManager::CreateInitialFileSystemLayout() {
+Status FsManager::CreateInitialFileSystemLayout(boost::optional<string> uuid) {
   CHECK(!read_only_);
 
   RETURN_NOT_OK(Init());
@@ -271,7 +272,7 @@ Status FsManager::CreateInitialFileSystemLayout() {
   ElementDeleter d(&delete_on_failure);
 
   InstanceMetadataPB metadata;
-  CreateInstanceMetadata(&metadata);
+  RETURN_NOT_OK(CreateInstanceMetadata(std::move(uuid), &metadata));
   unordered_set<string> to_sync;
   for (const string& root : canonicalized_all_fs_roots_) {
     bool created;
@@ -319,9 +320,16 @@ Status FsManager::CreateInitialFileSystemLayout() {
   return Status::OK();
 }
 
-void FsManager::CreateInstanceMetadata(InstanceMetadataPB* metadata) {
+Status FsManager::CreateInstanceMetadata(boost::optional<string> uuid,
+                                         InstanceMetadataPB* metadata) {
   ObjectIdGenerator oid_generator;
-  metadata->set_uuid(oid_generator.Next());
+  if (uuid) {
+    string canonicalized_uuid;
+    RETURN_NOT_OK(oid_generator.Canonicalize(uuid.get(), &canonicalized_uuid));
+    metadata->set_uuid(canonicalized_uuid);
+  } else {
+    metadata->set_uuid(oid_generator.Next());
+  }
 
   string time_str;
   StringAppendStrftime(&time_str, "%Y-%m-%d %H:%M:%S", time(nullptr), false);
@@ -330,6 +338,7 @@ void FsManager::CreateInstanceMetadata(InstanceMetadataPB* metadata) {
     hostname = "<unknown host>";
   }
   metadata->set_format_stamp(Substitute("Formatted at $0 on $1", time_str, hostname));
+  return Status::OK();
 }
 
 Status FsManager::WriteInstanceMetadata(const InstanceMetadataPB& metadata,

http://git-wip-us.apache.org/repos/asf/kudu/blob/1bce7312/src/kudu/fs/fs_manager.h
----------------------------------------------------------------------
diff --git a/src/kudu/fs/fs_manager.h b/src/kudu/fs/fs_manager.h
index fab0436..3c4420a 100644
--- a/src/kudu/fs/fs_manager.h
+++ b/src/kudu/fs/fs_manager.h
@@ -18,6 +18,8 @@
 #ifndef KUDU_FS_FS_MANAGER_H
 #define KUDU_FS_FS_MANAGER_H
 
+#include <boost/none.hpp>
+#include <boost/optional/optional.hpp>
 #include <gtest/gtest_prod.h>
 #include <iosfwd>
 #include <memory>
@@ -107,10 +109,12 @@ class FsManager {
   // the on-disk structures.
   Status Open();
 
-  // Create the initial filesystem layout.
+  // Create the initial filesystem layout. If 'uuid' is provided, uses it as
+  // uuid of the filesystem. Otherwise generates one at random.
   //
   // Returns an error if the file system is already initialized.
-  Status CreateInitialFileSystemLayout();
+  Status CreateInitialFileSystemLayout(
+      boost::optional<std::string> uuid = boost::none);
 
   void DumpFileSystemTree(std::ostream& out);
 
@@ -212,7 +216,8 @@ class FsManager {
   void InitBlockManager();
 
   // Create a new InstanceMetadataPB.
-  void CreateInstanceMetadata(InstanceMetadataPB* metadata);
+  Status CreateInstanceMetadata(boost::optional<std::string> uuid,
+                                InstanceMetadataPB* metadata);
 
   // Save a InstanceMetadataPB to the filesystem.
   // Does not mutate the current state of the fsmanager.

http://git-wip-us.apache.org/repos/asf/kudu/blob/1bce7312/src/kudu/util/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/src/kudu/util/CMakeLists.txt b/src/kudu/util/CMakeLists.txt
index 4a000b6..a4527b9 100644
--- a/src/kudu/util/CMakeLists.txt
+++ b/src/kudu/util/CMakeLists.txt
@@ -309,6 +309,7 @@ ADD_KUDU_TEST(mt-threadlocal-test RUN_SERIAL true)
 ADD_KUDU_TEST(net/dns_resolver-test)
 ADD_KUDU_TEST(net/net_util-test)
 ADD_KUDU_TEST(object_pool-test)
+ADD_KUDU_TEST(oid_generator-test)
 ADD_KUDU_TEST(once-test)
 ADD_KUDU_TEST(os-util-test)
 ADD_KUDU_TEST(path_util-test)

http://git-wip-us.apache.org/repos/asf/kudu/blob/1bce7312/src/kudu/util/oid_generator-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/oid_generator-test.cc b/src/kudu/util/oid_generator-test.cc
new file mode 100644
index 0000000..a38b496
--- /dev/null
+++ b/src/kudu/util/oid_generator-test.cc
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "kudu/util/oid_generator.h"
+
+#include <gtest/gtest.h>
+#include <string>
+
+#include "kudu/util/test_util.h"
+
+using std::string;
+
+namespace kudu {
+
+TEST(ObjectIdGeneratorTest, TestCanoicalizeUuid) {
+  ObjectIdGenerator gen;
+  const string kExpectedCanonicalized = "0123456789abcdef0123456789abcdef";
+  string canonicalized;
+  Status s = gen.Canonicalize("not_a_uuid", &canonicalized);
+  {
+    SCOPED_TRACE(s.ToString());
+    ASSERT_TRUE(s.IsInvalidArgument());
+    ASSERT_STR_CONTAINS(s.ToString(), "invalid uuid");
+  }
+  ASSERT_OK(gen.Canonicalize(
+      "01234567-89ab-cdef-0123-456789abcdef", &canonicalized));
+  ASSERT_EQ(kExpectedCanonicalized, canonicalized);
+  ASSERT_OK(gen.Canonicalize(
+      "0123456789abcdef0123456789abcdef", &canonicalized));
+  ASSERT_EQ(kExpectedCanonicalized, canonicalized);
+  ASSERT_OK(gen.Canonicalize(
+      "0123456789AbCdEf0123456789aBcDeF", &canonicalized));
+  ASSERT_EQ(kExpectedCanonicalized, canonicalized);
+}
+
+} // namespace kudu

http://git-wip-us.apache.org/repos/asf/kudu/blob/1bce7312/src/kudu/util/oid_generator.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/oid_generator.cc b/src/kudu/util/oid_generator.cc
index da7acd8..580463c 100644
--- a/src/kudu/util/oid_generator.cc
+++ b/src/kudu/util/oid_generator.cc
@@ -15,21 +15,48 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include "kudu/util/oid_generator.h"
+
+#include <boost/uuid/uuid_generators.hpp>
+#include <exception>
 #include <mutex>
 #include <string>
 
 #include "kudu/gutil/stringprintf.h"
-#include "kudu/util/oid_generator.h"
+#include "kudu/gutil/strings/substitute.h"
+#include "kudu/util/status.h"
+
+using strings::Substitute;
 
 namespace kudu {
 
-string ObjectIdGenerator::Next() {
-  std::lock_guard<LockType> l(oid_lock_);
-  boost::uuids::uuid oid = oid_generator_();
-  const uint8_t *uuid = oid.data;
+namespace {
+
+string ConvertUuidToString(const boost::uuids::uuid& to_convert) {
+  const uint8_t* uuid = to_convert.data;
   return StringPrintf("%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x",
                uuid[0], uuid[1], uuid[2], uuid[3], uuid[4], uuid[5], uuid[6], uuid[7],
                uuid[8], uuid[9], uuid[10], uuid[11], uuid[12], uuid[13], uuid[14], uuid[15]);
 }
 
+} // anonymous namespace
+
+string ObjectIdGenerator::Next() {
+  std::lock_guard<LockType> l(oid_lock_);
+  boost::uuids::uuid uuid = oid_generator_();
+  return ConvertUuidToString(uuid);
+}
+
+Status ObjectIdGenerator::Canonicalize(const string& input,
+                                       string* output) const {
+  try {
+    boost::uuids::uuid uuid = oid_validator_(input);
+    *output = ConvertUuidToString(uuid);
+    return Status::OK();
+  } catch (std::exception& e) {
+    return Status::InvalidArgument(Substitute("invalid uuid $0: $1",
+                                              input, e.what()));
+  }
+}
+
 } // namespace kudu

http://git-wip-us.apache.org/repos/asf/kudu/blob/1bce7312/src/kudu/util/oid_generator.h
----------------------------------------------------------------------
diff --git a/src/kudu/util/oid_generator.h b/src/kudu/util/oid_generator.h
index 85a7412..7acccc9 100644
--- a/src/kudu/util/oid_generator.h
+++ b/src/kudu/util/oid_generator.h
@@ -23,6 +23,7 @@
 
 #include "kudu/gutil/macros.h"
 #include "kudu/util/locks.h"
+#include "kudu/util/status.h"
 
 namespace kudu {
 
@@ -33,15 +34,26 @@ class ObjectIdGenerator {
   ObjectIdGenerator() {}
   ~ObjectIdGenerator() {}
 
+  // Generates and returns a new UUID.
   std::string Next();
 
+  // Validates an existing UUID and converts it into the format used by Kudu
+  // (that is, 16 hexadecimal bytes without any dashes).
+  Status Canonicalize(const std::string& input, std::string* output) const;
+
  private:
   DISALLOW_COPY_AND_ASSIGN(ObjectIdGenerator);
 
   typedef simple_spinlock LockType;
 
+  // Protects 'oid_generator_'.
   LockType oid_lock_;
+
+  // Generates new UUIDs.
   boost::uuids::random_generator oid_generator_;
+
+  // Validates provided UUIDs.
+  boost::uuids::string_generator oid_validator_;
 };
 
 } // namespace kudu


Mime
View raw message