kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mpe...@apache.org
Subject [1/2] kudu git commit: tools: Add debug mode to pb dump tool
Date Tue, 26 Jun 2018 21:07:59 GMT
Repository: kudu
Updated Branches:
  refs/heads/master b174d6e53 -> 97210811b


tools: Add debug mode to pb dump tool

This is useful when dumping a partially-flushed file to determine the
start of the damaged entry. To remove the damaged entry, the file can be
truncated at the beginning of the offending entry. See KUDU-2260.

Change-Id: Ica2a74a2e252d140cf7704ff68037a64db19cd80
Reviewed-on: http://gerrit.cloudera.org:8080/9024
Tested-by: Kudu Jenkins
Reviewed-by: Will Berkeley <wdberkeley@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/a935f70f
Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/a935f70f
Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/a935f70f

Branch: refs/heads/master
Commit: a935f70fb5113ce5eec2604385ea2e79ce4ed725
Parents: b174d6e
Author: Mike Percy <mpercy@apache.org>
Authored: Wed Jan 3 17:04:44 2018 -0800
Committer: Mike Percy <mpercy@apache.org>
Committed: Tue Jun 26 21:07:39 2018 +0000

----------------------------------------------------------------------
 src/kudu/tools/kudu-tool-test.cc  | 19 +++++++++++++++++++
 src/kudu/tools/tool_action_pbc.cc | 10 ++++++++++
 src/kudu/util/pb_util.cc          | 31 ++++++++++++++++++++++++++++---
 src/kudu/util/pb_util.h           |  2 ++
 4 files changed, 59 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kudu/blob/a935f70f/src/kudu/tools/kudu-tool-test.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc
index 9138f26..d08e107 100644
--- a/src/kudu/tools/kudu-tool-test.cc
+++ b/src/kudu/tools/kudu-tool-test.cc
@@ -773,6 +773,25 @@ TEST_F(ToolTest, TestPbcTools) {
     ASSERT_EQ(Substitute("uuid: \"$0\"", uuid), stdout[2]);
     ASSERT_STR_MATCHES(stdout[3], "^format_stamp: \"Formatted at .*\"$");
   }
+  // Test dump --debug
+  {
+    vector<string> stdout;
+    NO_FATALS(RunActionStdoutLines(Substitute(
+        "pbc dump $0 --debug", instance_path), &stdout));
+    SCOPED_TRACE(stdout);
+    ASSERT_EQ(12, stdout.size());
+    ASSERT_EQ("File header", stdout[0]);
+    ASSERT_EQ("-------", stdout[1]);
+    ASSERT_STR_MATCHES(stdout[2], "^Protobuf container version:");
+    ASSERT_STR_MATCHES(stdout[3], "^Total container file size:");
+    ASSERT_STR_MATCHES(stdout[4], "^Entry PB type:");
+    ASSERT_EQ("Message 0", stdout[6]);
+    ASSERT_STR_MATCHES(stdout[7], "^offset:");
+    ASSERT_STR_MATCHES(stdout[8], "^length:");
+    ASSERT_EQ("-------", stdout[9]);
+    ASSERT_EQ(Substitute("uuid: \"$0\"", uuid), stdout[10]);
+    ASSERT_STR_MATCHES(stdout[11], "^format_stamp: \"Formatted at .*\"$");
+  }
   // Test dump --oneline
   {
     string stdout;

http://git-wip-us.apache.org/repos/asf/kudu/blob/a935f70f/src/kudu/tools/tool_action_pbc.cc
----------------------------------------------------------------------
diff --git a/src/kudu/tools/tool_action_pbc.cc b/src/kudu/tools/tool_action_pbc.cc
index c26a89d..ee3837d 100644
--- a/src/kudu/tools/tool_action_pbc.cc
+++ b/src/kudu/tools/tool_action_pbc.cc
@@ -58,6 +58,9 @@ TAG_FLAG(oneline, stable);
 DEFINE_bool(json, false, "print protobufs in JSON format");
 TAG_FLAG(json, stable);
 
+DEFINE_bool(debug, false, "print extra debugging information about each protobuf");
+TAG_FLAG(debug, stable);
+
 namespace kudu {
 
 using pb_util::ReadablePBContainerFile;
@@ -74,12 +77,18 @@ Status DumpPBContainerFile(const RunnerContext& context) {
     return Status::InvalidArgument("only one of --json or --oneline may be provided");
   }
 
+  if (FLAGS_debug && (FLAGS_oneline || FLAGS_json)) {
+    return Status::InvalidArgument("--debug is not compatible with --json or --oneline");
+  }
+
   const string& path = FindOrDie(context.required_args, kPathArg);
   auto format = ReadablePBContainerFile::Format::DEFAULT;
   if (FLAGS_json) {
     format = ReadablePBContainerFile::Format::JSON;
   } else if (FLAGS_oneline) {
     format = ReadablePBContainerFile::Format::ONELINE;
+  } else if (FLAGS_debug) {
+    format = ReadablePBContainerFile::Format::DEBUG;
   }
 
   Env* env = Env::Default();
@@ -220,6 +229,7 @@ unique_ptr<Mode> BuildPbcMode() {
   unique_ptr<Action> dump =
       ActionBuilder("dump", &DumpPBContainerFile)
       .Description("Dump a PBC (protobuf container) file")
+      .AddOptionalParameter("debug")
       .AddOptionalParameter("oneline")
       .AddOptionalParameter("json")
       .AddRequiredParameter({kPathArg, "path to PBC file"})

http://git-wip-us.apache.org/repos/asf/kudu/blob/a935f70f/src/kudu/util/pb_util.cc
----------------------------------------------------------------------
diff --git a/src/kudu/util/pb_util.cc b/src/kudu/util/pb_util.cc
index 1c072d6..edf1222 100644
--- a/src/kudu/util/pb_util.cc
+++ b/src/kudu/util/pb_util.cc
@@ -940,6 +940,17 @@ Status ReadablePBContainerFile::Dump(ostream* os, ReadablePBContainerFile::Forma
     return Status::NotSupported("cannot dump PBC file in JSON format if redaction is enabled");
   }
 
+  const char* const kDashes = "-------";
+
+  if (format == Format::DEBUG) {
+    *os << "File header" << endl;
+    *os << kDashes << endl;
+    *os << "Protobuf container version: " << version_ << endl;
+    *os << "Total container file size: " << *cached_file_size_ << endl;
+    *os << "Entry PB type: " << pb_type_ << endl;
+    *os << endl;
+  }
+
   // Use the embedded protobuf information from the container file to
   // create the appropriate kind of protobuf Message.
   const Message* prototype;
@@ -948,6 +959,7 @@ Status ReadablePBContainerFile::Dump(ostream* os, ReadablePBContainerFile::Forma
 
   // Dump each message in the container file.
   int count = 0;
+  uint64_t prev_offset = offset_;
   Status s;
   string buf;
   for (s = ReadNextPB(msg.get());
@@ -958,8 +970,13 @@ Status ReadablePBContainerFile::Dump(ostream* os, ReadablePBContainerFile::Forma
         *os << count << "\t" << SecureShortDebugString(*msg) << endl;
         break;
       case Format::DEFAULT:
+      case Format::DEBUG:
         *os << "Message " << count << endl;
-        *os << "-------" << endl;
+        if (format == Format::DEBUG) {
+          *os << "offset: " << prev_offset << endl;
+          *os << "length: " << (offset_ - prev_offset) << endl;
+        }
+        *os << kDashes << endl;
         *os << SecureDebugString(*msg) << endl;
         break;
       case Format::JSON:
@@ -972,10 +989,18 @@ Status ReadablePBContainerFile::Dump(ostream* os, ReadablePBContainerFile::Forma
         *os << buf << endl;
         break;
     }
-    count++;
 
+    prev_offset = offset_;
+    count++;
+  }
+  if (format == Format::DEBUG && !s.IsEndOfFile()) {
+    *os << "Message " << count << endl;
+    *os << "error: failed to parse protobuf message" << endl;
+    *os << "offset: " << prev_offset << endl;
+    *os << "remaining file length: " << (*cached_file_size_ - prev_offset) <<
endl;
+    *os << kDashes << endl;
   }
-  return s.IsEndOfFile() ? s.OK() : s;
+  return s.IsEndOfFile() ? Status::OK() : s;
 }
 
 Status ReadablePBContainerFile::Close() {

http://git-wip-us.apache.org/repos/asf/kudu/blob/a935f70f/src/kudu/util/pb_util.h
----------------------------------------------------------------------
diff --git a/src/kudu/util/pb_util.h b/src/kudu/util/pb_util.h
index 9e154c2..6c132a6 100644
--- a/src/kudu/util/pb_util.h
+++ b/src/kudu/util/pb_util.h
@@ -402,6 +402,8 @@ class ReadablePBContainerFile {
   enum class Format {
     // Print each message on multiple lines, with intervening headers.
     DEFAULT,
+    // Same as DEFAULT but includes additional metadata information.
+    DEBUG,
     // Print each message on its own line.
     ONELINE,
     // Dump in JSON.


Mime
View raw message