kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From granthe...@apache.org
Subject [kudu] 02/03: tools: avoid extra 100ms sleep at end of table scan
Date Thu, 16 Jan 2020 22:46:52 GMT
This is an automated email from the ASF dual-hosted git repository.

granthenke pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git

commit 6325c4edcc389aa2ee1d8f028f367978047b8be6
Author: Todd Lipcon <todd@apache.org>
AuthorDate: Wed Jan 15 15:23:00 2020 -0800

    tools: avoid extra 100ms sleep at end of table scan
    
    Prior to this patch, the table scan tool printed a status output every
    5 seconds by starting a separate monitor thread. This thread's main loop
    would only check for the scan completion every 100ms, so after the scans
    completed, the process could hang for up to an extra 100ms before
    exiting.
    
    This changes the printing to happen from the main thread instead, with
    the waiting based on the actual scan completion. Now the tool exits
    immediately when the scan completes.
    
    Example output:
    
    Before:
    todd@turbo:~/kudu$ time ./build/latest/bin/kudu table scan localhost sequences  | wc -l
      79
    
      real  0m0.180s
      user  0m0.041s
      sys   0m0.021s
    
    After:
      todd@turbo:~/kudu$ time ./build/latest/bin/kudu table scan localhost sequences  | wc
-l
      79
    
      real  0m0.078s
      user  0m0.055s
      sys   0m0.005s
    
    Change-Id: Ic7dcd8408b1dbd5322a2f5e8f4889f5f5660ece0
    Reviewed-on: http://gerrit.cloudera.org:8080/15040
    Reviewed-by: Bankim Bhavsar <bankim@cloudera.com>
    Reviewed-by: Grant Henke <granthenke@apache.org>
    Tested-by: Grant Henke <granthenke@apache.org>
    Reviewed-by: Alexey Serbin <aserbin@cloudera.com>
---
 src/kudu/tools/table_scanner.cc | 20 ++++++--------------
 src/kudu/tools/table_scanner.h  |  1 -
 2 files changed, 6 insertions(+), 15 deletions(-)

diff --git a/src/kudu/tools/table_scanner.cc b/src/kudu/tools/table_scanner.cc
index 4924f30..f50b93a 100644
--- a/src/kudu/tools/table_scanner.cc
+++ b/src/kudu/tools/table_scanner.cc
@@ -502,8 +502,9 @@ void TableScanner::ScanTask(const vector<KuduScanToken *>& tokens,
Status* threa
     if (out_ && FLAGS_show_values) {
       MutexLock l(output_lock_);
       for (const auto& row : batch) {
-        *out_ << row.ToString() << endl;
+        *out_ << row.ToString() << "\n";
       }
+      out_->flush();
     }
   });
 }
@@ -530,16 +531,6 @@ void TableScanner::CopyTask(const vector<KuduScanToken*>& tokens,
Status* thread
   });
 }
 
-void TableScanner::MonitorTask() {
-  MonoTime last_log_time = MonoTime::Now();
-  while (thread_pool_->num_threads() > 1) {    // Some other table scan thread is running.
-    if (MonoTime::Now() - last_log_time >= MonoDelta::FromSeconds(5)) {
-      LOG(INFO) << "Scanned count: " << total_count_.Load();
-      last_log_time = MonoTime::Now();
-    }
-    SleepFor(MonoDelta::FromMilliseconds(100));
-  }
-}
 
 void TableScanner::SetOutput(ostream* out) {
   out_ = out;
@@ -600,7 +591,7 @@ Status TableScanner::StartWork(WorkType type) {
   vector<Status> thread_statuses(FLAGS_num_threads);
 
   RETURN_NOT_OK(ThreadPoolBuilder("table_scan_pool")
-                  .set_max_threads(FLAGS_num_threads + 1)  // add extra 1 thread for MonitorTask
+                  .set_max_threads(FLAGS_num_threads)
                   .set_idle_timeout(MonoDelta::FromMilliseconds(1))
                   .Build(&thread_pool_));
 
@@ -617,8 +608,9 @@ Status TableScanner::StartWork(WorkType type) {
         boost::bind(&TableScanner::CopyTask, this, thread_tokens[i], &thread_statuses[i])));
     }
   }
-  RETURN_NOT_OK(thread_pool_->SubmitFunc(boost::bind(&TableScanner::MonitorTask, this)));
-  thread_pool_->Wait();
+  while (!thread_pool_->WaitFor(MonoDelta::FromSeconds(5))) {
+    LOG(INFO) << "Scanned count: " << total_count_.Load();
+  }
   thread_pool_->Shutdown();
 
   sw.stop();
diff --git a/src/kudu/tools/table_scanner.h b/src/kudu/tools/table_scanner.h
index b784835..ff6b81b 100644
--- a/src/kudu/tools/table_scanner.h
+++ b/src/kudu/tools/table_scanner.h
@@ -83,7 +83,6 @@ class TableScanner {
                   const std::function<void(const kudu::client::KuduScanBatch& batch)>&
cb);
   void ScanTask(const std::vector<kudu::client::KuduScanToken*>& tokens, Status*
thread_status);
   void CopyTask(const std::vector<kudu::client::KuduScanToken*>& tokens, Status*
thread_status);
-  void MonitorTask();
 
   Status AddRow(const client::sp::shared_ptr<kudu::client::KuduTable>& table,
                 const kudu::client::KuduSchema& table_schema,


Mime
View raw message