kudu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a...@apache.org
Subject [kudu] branch master updated: KUDU-2934: add MeanGauge type to metrics
Date Fri, 20 Sep 2019 04:15:39 GMT
This is an automated email from the ASF dual-hosted git repository.

adar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git


The following commit(s) were added to refs/heads/master by this push:
     new 9339f6d  KUDU-2934: add MeanGauge type to metrics
9339f6d is described below

commit 9339f6dda1c30e007bab3cfb0f37a9dcb327e683
Author: zhangyifan27 <chinazhangyifan@163.com>
AuthorDate: Wed Sep 18 15:30:28 2019 +0800

    KUDU-2934: add MeanGauge type to metrics
    
    The current merge method is not suitable for the metric
    'average_diskrowset_height', it is a mean value and we
    could not just sum up the metric values when merging
    metrics.
    
    There is another issue with the metric 'average_diskrowset_height',
    the value of the metric is double but it was used as an AtomicGauge
    metric, which is only for types that are convertible to/from int64_t.
    
    This patch added a new 'MeanGauge' type for metrics which show
    the mean value of something.
    
    Change-Id: I827d3acb5d4f47a7e28ac40cb7df392c29c82a40
    Reviewed-on: http://gerrit.cloudera.org:8080/14252
    Reviewed-by: Adar Dembo <adar@cloudera.com>
    Tested-by: Adar Dembo <adar@cloudera.com>
---
 src/kudu/tablet/compaction_policy-test.cc | 10 +++++--
 src/kudu/tablet/compaction_policy.cc      |  3 +-
 src/kudu/tablet/rowset_info.cc            |  9 +++---
 src/kudu/tablet/rowset_info.h             |  8 ++++--
 src/kudu/tablet/tablet.cc                 | 20 +++++++++----
 src/kudu/tablet/tablet_metrics.cc         |  4 ++-
 src/kudu/tablet/tablet_metrics.h          |  4 +--
 src/kudu/util/metrics-test.cc             | 33 +++++++++++++++++++++
 src/kudu/util/metrics.cc                  | 48 +++++++++++++++++++++++++++++++
 src/kudu/util/metrics.h                   | 48 +++++++++++++++++++++++++++++++
 10 files changed, 167 insertions(+), 20 deletions(-)

diff --git a/src/kudu/tablet/compaction_policy-test.cc b/src/kudu/tablet/compaction_policy-test.cc
index 34824b0..69c1bdf 100644
--- a/src/kudu/tablet/compaction_policy-test.cc
+++ b/src/kudu/tablet/compaction_policy-test.cc
@@ -445,9 +445,13 @@ double ComputeAverageRowsetHeight(
   RowSetTree tree;
   CHECK_OK(tree.Reset(rowsets));
 
-  double avg_height;
-  RowSetInfo::ComputeCdfAndCollectOrdered(tree, &avg_height, nullptr, nullptr);
-  return avg_height;
+  double rowset_total_height, rowset_total_width;
+  RowSetInfo::ComputeCdfAndCollectOrdered(tree,
+                                          &rowset_total_height,
+                                          &rowset_total_width,
+                                          nullptr,
+                                          nullptr);
+  return rowset_total_width > 0 ? rowset_total_height / rowset_total_width : 0.0;
 }
 } // anonymous namespace
 
diff --git a/src/kudu/tablet/compaction_policy.cc b/src/kudu/tablet/compaction_policy.cc
index e740ac4..0d96cfc 100644
--- a/src/kudu/tablet/compaction_policy.cc
+++ b/src/kudu/tablet/compaction_policy.cc
@@ -107,7 +107,8 @@ void BudgetedCompactionPolicy::SetupKnapsackInput(
     vector<RowSetInfo>* asc_min_key,
     vector<RowSetInfo>* asc_max_key) const {
   RowSetInfo::ComputeCdfAndCollectOrdered(tree,
-                                          /*average_height=*/nullptr,
+                                          /*rowset_total_height=*/nullptr,
+                                          /*rowset_total_width=*/nullptr,
                                           asc_min_key,
                                           asc_max_key);
 
diff --git a/src/kudu/tablet/rowset_info.cc b/src/kudu/tablet/rowset_info.cc
index 187a96f..f276125 100644
--- a/src/kudu/tablet/rowset_info.cc
+++ b/src/kudu/tablet/rowset_info.cc
@@ -253,7 +253,8 @@ void RowSetInfo::Collect(const RowSetTree& tree, vector<RowSetInfo>*
rsvec) {
 }
 
 void RowSetInfo::ComputeCdfAndCollectOrdered(const RowSetTree& tree,
-                                             double* average_height,
+                                             double* rowset_total_height,
+                                             double* rowset_total_width,
                                              vector<RowSetInfo>* info_by_min_key,
                                              vector<RowSetInfo>* info_by_max_key) {
   DCHECK((info_by_min_key && info_by_max_key) ||
@@ -352,9 +353,9 @@ void RowSetInfo::ComputeCdfAndCollectOrdered(const RowSetTree& tree,
   FinalizeCDFVector(total_width, &info_by_min_key_tmp);
   FinalizeCDFVector(total_width, &info_by_max_key_tmp);
 
-  if (average_height) {
-    *average_height = total_width > 0 ? weighted_height_sum / total_width
-                                      : 0.0;
+  if (rowset_total_height && rowset_total_width) {
+    *rowset_total_height = weighted_height_sum;
+    *rowset_total_width = total_width;
   }
 
   if (info_by_min_key && info_by_max_key) {
diff --git a/src/kudu/tablet/rowset_info.h b/src/kudu/tablet/rowset_info.h
index f133e3e..585c9cb 100644
--- a/src/kudu/tablet/rowset_info.h
+++ b/src/kudu/tablet/rowset_info.h
@@ -47,14 +47,16 @@ class RowSetInfo {
 
   // From the rowset tree 'tree', computes the keyspace cdf and collects rowset
   // information in min-key- and max-key-sorted order into 'info_by_min_key'
-  // and 'info_by_max_key', respectively. The average value of the height of the
-  // rowset tree is set into 'average_height', if it is not nullptr.
+  // and 'info_by_max_key', respectively.
+  // The total weighted height and the total width of the rowset tree is set into
+  // 'rowset_total_height' and 'rowset_total_width', if they are not nullptr.
   // If one of 'info_by_min_key' and 'info_by_max_key' is nullptr, the other
   // must be.
   // Requires holding the compact_select_lock_ for the tablet that the
   // rowsets in 'tree' references.
   static void ComputeCdfAndCollectOrdered(const RowSetTree& tree,
-                                          double* average_height,
+                                          double* rowset_total_height,
+                                          double* rowset_total_width,
                                           std::vector<RowSetInfo>* info_by_min_key,
                                           std::vector<RowSetInfo>* info_by_max_key);
 
diff --git a/src/kudu/tablet/tablet.cc b/src/kudu/tablet/tablet.cc
index 78eff66..5c3db23 100644
--- a/src/kudu/tablet/tablet.cc
+++ b/src/kudu/tablet/tablet.cc
@@ -1725,12 +1725,13 @@ void Tablet::UpdateAverageRowsetHeight() {
   scoped_refptr<TabletComponents> comps;
   GetComponents(&comps);
   std::lock_guard<std::mutex> l(compact_select_lock_);
-  double avg_height;
+  double rowset_total_height, rowset_total_width;
   RowSetInfo::ComputeCdfAndCollectOrdered(*comps->rowsets,
-                                          &avg_height,
+                                          &rowset_total_height,
+                                          &rowset_total_width,
                                           nullptr,
                                           nullptr);
-  metrics_->average_diskrowset_height->set_value(avg_height);
+  metrics_->average_diskrowset_height->set_value(rowset_total_height, rowset_total_width);
 }
 
 Status Tablet::Compact(CompactFlags flags) {
@@ -2358,9 +2359,16 @@ void Tablet::PrintRSLayout(ostream* o) {
     out << "</p>";
   }
 
-  double avg_height;
+  double rowset_total_height, rowset_total_width;
   vector<RowSetInfo> min, max;
-  RowSetInfo::ComputeCdfAndCollectOrdered(*rowsets_copy, &avg_height, &min, &max);
+  RowSetInfo::ComputeCdfAndCollectOrdered(*rowsets_copy,
+                                          &rowset_total_height,
+                                          &rowset_total_width,
+                                          &min,
+                                          &max);
+  double average_rowset_height = rowset_total_width > 0
+                               ? rowset_total_height / rowset_total_width
+                               : 0.0;
   DumpCompactionSVG(min, picked, o, /*print_xml_header=*/false);
 
   // Compaction policy ignores rowsets unavailable for compaction. This is good,
@@ -2422,7 +2430,7 @@ void Tablet::PrintRSLayout(ostream* o) {
                       HumanReadableNumBytes::ToString(size_bytes_median),
                       HumanReadableNumBytes::ToString(size_bytes_third_quartile),
                       HumanReadableNumBytes::ToString(size_bytes_max),
-                      avg_height);
+                      average_rowset_height);
     out << "</table>" << endl;
   }
 
diff --git a/src/kudu/tablet/tablet_metrics.cc b/src/kudu/tablet/tablet_metrics.cc
index 5aa4fa2..e266af1 100644
--- a/src/kudu/tablet/tablet_metrics.cc
+++ b/src/kudu/tablet/tablet_metrics.cc
@@ -269,6 +269,7 @@ namespace tablet {
 
 #define MINIT(x) x(METRIC_##x.Instantiate(entity))
 #define GINIT(x) x(METRIC_##x.Instantiate(entity, 0))
+#define MEANINIT(x) x(METRIC_##x.InstantiateMeanGauge(entity))
 TabletMetrics::TabletMetrics(const scoped_refptr<MetricEntity>& entity)
   : MINIT(rows_inserted),
     MINIT(rows_upserted),
@@ -313,10 +314,11 @@ TabletMetrics::TabletMetrics(const scoped_refptr<MetricEntity>&
entity)
     MINIT(undo_delta_block_gc_delete_duration),
     MINIT(undo_delta_block_gc_perform_duration),
     MINIT(leader_memory_pressure_rejections),
-    GINIT(average_diskrowset_height) {
+    MEANINIT(average_diskrowset_height) {
 }
 #undef MINIT
 #undef GINIT
+#undef MEANINIT
 
 void TabletMetrics::AddProbeStats(const ProbeStats* stats_array, int len,
                                   Arena* work_arena) {
diff --git a/src/kudu/tablet/tablet_metrics.h b/src/kudu/tablet/tablet_metrics.h
index c60b9aa..b17098f 100644
--- a/src/kudu/tablet/tablet_metrics.h
+++ b/src/kudu/tablet/tablet_metrics.h
@@ -33,7 +33,7 @@ struct ProbeStats;
 
 // Container for all metrics specific to a single tablet.
 struct TabletMetrics {
-  explicit TabletMetrics(const scoped_refptr<MetricEntity>& metric_entity);
+  explicit TabletMetrics(const scoped_refptr<MetricEntity>& entity);
 
   // Add a batch of probe stats to the metrics.
   //
@@ -100,7 +100,7 @@ struct TabletMetrics {
   scoped_refptr<Counter> leader_memory_pressure_rejections;
 
   // Compaction metrics.
-  scoped_refptr<AtomicGauge<double>> average_diskrowset_height;
+  scoped_refptr<MeanGauge> average_diskrowset_height;
 };
 
 } // namespace tablet
diff --git a/src/kudu/util/metrics-test.cc b/src/kudu/util/metrics-test.cc
index 7e5bbbc..55ddecf 100644
--- a/src/kudu/util/metrics-test.cc
+++ b/src/kudu/util/metrics-test.cc
@@ -168,6 +168,39 @@ TEST_F(MetricsTest, SimpleStringGaugeForMergeTest) {
             state_for_merge->unique_values());
 }
 
+METRIC_DEFINE_gauge_double(test_entity, test_mean_gauge, "Test mean Gauge",
+                           MetricUnit::kUnits, "Description of mean Gauge");
+
+TEST_F(MetricsTest, SimpleMeanGaugeTest) {
+  scoped_refptr<MeanGauge> average_usage =
+    METRIC_test_mean_gauge.InstantiateMeanGauge(entity_);
+  ASSERT_EQ(METRIC_test_mean_gauge.description(), average_usage->prototype()->description());
+  ASSERT_EQ(0, average_usage->value());
+  average_usage->set_value(10.0, 2.0);
+  ASSERT_EQ(5, average_usage->value());
+  average_usage->set_value(5.0, 2.0);
+  ASSERT_EQ(2.5, average_usage->value());
+}
+
+TEST_F(MetricsTest, SimpleMeanGaugeMergeTest) {
+  scoped_refptr<MeanGauge> average_usage =
+    METRIC_test_mean_gauge.InstantiateMeanGauge(entity_);
+  scoped_refptr<MeanGauge> average_usage_for_merge =
+    METRIC_test_mean_gauge.InstantiateMeanGauge(entity_same_attr_);
+  average_usage_for_merge->MergeFrom(average_usage);
+  ASSERT_EQ(0, average_usage->value());
+  ASSERT_EQ(0, average_usage_for_merge->value());
+  average_usage->set_value(10.0, 1.0);
+  average_usage_for_merge->set_value(2.0, 2.0);
+  ASSERT_EQ(10, average_usage->value());
+  ASSERT_EQ(1, average_usage_for_merge->value());
+  average_usage_for_merge->MergeFrom(average_usage);
+  ASSERT_EQ(10, average_usage->value());
+  ASSERT_EQ(4, average_usage_for_merge->value());
+  average_usage_for_merge->MergeFrom(average_usage_for_merge);
+  ASSERT_EQ(4, average_usage_for_merge->value());
+}
+
 METRIC_DEFINE_gauge_uint64(test_entity, test_gauge, "Test uint64 Gauge",
                            MetricUnit::kBytes, "Description of Test Gauge");
 
diff --git a/src/kudu/util/metrics.cc b/src/kudu/util/metrics.cc
index 6102ef7..fc6b538 100644
--- a/src/kudu/util/metrics.cc
+++ b/src/kudu/util/metrics.cc
@@ -727,6 +727,54 @@ void StringGauge::WriteValue(JsonWriter* writer) const {
 }
 
 //
+// MeanGauge
+//
+
+scoped_refptr<Metric> MeanGauge::snapshot() const {
+  std::lock_guard<simple_spinlock> l(lock_);
+  scoped_refptr<Metric> m
+    = new MeanGauge(down_cast<const GaugePrototype<double>*>(prototype_));
+  return m;
+}
+
+double MeanGauge::value() const {
+  std::lock_guard<simple_spinlock> l(lock_);
+  return total_count_ > 0 ? total_sum_ / total_count_
+                          : 0.0;
+}
+
+double MeanGauge::total_sum() const {
+  std::lock_guard<simple_spinlock> l(lock_);
+  return total_sum_;
+}
+
+double MeanGauge::total_count() const {
+  std::lock_guard<simple_spinlock> l(lock_);
+  return total_count_;
+}
+
+void MeanGauge::set_value(double total_sum, double total_count) {
+  std::lock_guard<simple_spinlock> l(lock_);
+  total_sum_ = total_sum;
+  total_count_ = total_count;
+}
+
+void MeanGauge::MergeFrom(const scoped_refptr<Metric>& other) {
+  if (PREDICT_FALSE(this == other.get())) {
+    return;
+  }
+
+  scoped_refptr<MeanGauge> other_ptr = down_cast<MeanGauge*>(other.get());
+  std::lock_guard<simple_spinlock> l(lock_);
+  total_sum_ += other_ptr->total_sum();
+  total_count_ += other_ptr->total_count();
+}
+
+void MeanGauge::WriteValue(JsonWriter* writer) const {
+  writer->Double(value());
+}
+
+//
 // Counter
 //
 // This implementation is optimized by using a striped counter. See LongAdder for details.
diff --git a/src/kudu/util/metrics.h b/src/kudu/util/metrics.h
index 072c556..eccca0a 100644
--- a/src/kudu/util/metrics.h
+++ b/src/kudu/util/metrics.h
@@ -340,6 +340,7 @@ class CounterPrototype;
 class Histogram;
 class HistogramPrototype;
 class HistogramSnapshotPB;
+class MeanGauge;
 class Metric;
 class MetricEntity;
 class MetricEntityPrototype;
@@ -617,6 +618,8 @@ class MetricEntity : public RefCountedThreadSafe<MetricEntity> {
   scoped_refptr<AtomicGauge<T> > FindOrCreateGauge(const GaugePrototype<T>*
proto,
                                                    const T& initial_value);
 
+  scoped_refptr<MeanGauge> FindOrCreateMeanGauge(const GaugePrototype<double>*
proto);
+
   template<typename T>
   scoped_refptr<FunctionGauge<T> > FindOrCreateFunctionGauge(const GaugePrototype<T>*
proto,
                                                              const Callback<T()>&
function);
@@ -895,6 +898,11 @@ class GaugePrototype : public MetricPrototype {
     return entity->FindOrCreateGauge(this, initial_value);
   }
 
+  scoped_refptr<MeanGauge> InstantiateMeanGauge(
+      const scoped_refptr<MetricEntity>& entity) const {
+    return entity->FindOrCreateMeanGauge(this);
+  }
+
   // Instantiate a gauge that is backed by the given callback.
   scoped_refptr<FunctionGauge<T> > InstantiateFunctionGauge(
       const scoped_refptr<MetricEntity>& entity,
@@ -967,6 +975,33 @@ class StringGauge : public Gauge {
   DISALLOW_COPY_AND_ASSIGN(StringGauge);
 };
 
+// Gauge implementation for mean that uses locks to ensure thread safety.
+class MeanGauge : public Gauge {
+ public:
+  explicit MeanGauge(const GaugePrototype<double>* proto)
+    : Gauge(proto),
+      total_sum_(0.0),
+      total_count_(0.0) {
+  }
+  scoped_refptr<Metric> snapshot() const override;
+  double value() const;
+  double total_count() const;
+  double total_sum() const;
+  void set_value(double total_sum, double total_count);
+  virtual bool IsUntouched() const override {
+    return false;
+  }
+  void MergeFrom(const scoped_refptr<Metric>& other) override;
+
+ protected:
+  virtual void WriteValue(JsonWriter* writer) const override;
+ private:
+  double total_sum_;
+  double total_count_;
+  mutable simple_spinlock lock_;  // Guards total_sum_ and total_count_
+  DISALLOW_COPY_AND_ASSIGN(MeanGauge);
+};
+
 // Lock-free implementation for types that are convertible to/from int64_t.
 template <typename T>
 class AtomicGauge : public Gauge {
@@ -1349,6 +1384,19 @@ inline scoped_refptr<AtomicGauge<T> > MetricEntity::FindOrCreateGauge(
   return m;
 }
 
+inline scoped_refptr<MeanGauge> MetricEntity::FindOrCreateMeanGauge(
+    const GaugePrototype<double>* proto) {
+  CheckInstantiation(proto);
+  std::lock_guard<simple_spinlock> l(lock_);
+  scoped_refptr<MeanGauge> m = down_cast<MeanGauge*>(
+      FindPtrOrNull(metric_map_, proto).get());
+  if (!m) {
+    m = new MeanGauge(proto);
+    InsertOrDie(&metric_map_, proto, m);
+  }
+  return m;
+}
+
 template<typename T>
 inline scoped_refptr<FunctionGauge<T> > MetricEntity::FindOrCreateFunctionGauge(
     const GaugePrototype<T>* proto,


Mime
View raw message