Improve debug counters to support buckets and histograms. PiperOrigin-RevId: 906539008
diff --git a/src/google/protobuf/BUILD.bazel b/src/google/protobuf/BUILD.bazel index 77736ac..01c049a 100644 --- a/src/google/protobuf/BUILD.bazel +++ b/src/google/protobuf/BUILD.bazel
@@ -416,6 +416,7 @@ "@abseil-cpp//absl/base:dynamic_annotations", "@abseil-cpp//absl/base:prefetch", "@abseil-cpp//absl/log:absl_log", + "@abseil-cpp//absl/numeric:bits", "@abseil-cpp//absl/strings", "@abseil-cpp//absl/strings:str_format", "@abseil-cpp//absl/types:optional",
diff --git a/src/google/protobuf/debug_counter_test.cc b/src/google/protobuf/debug_counter_test.cc index 85e451e..2fb9933 100644 --- a/src/google/protobuf/debug_counter_test.cc +++ b/src/google/protobuf/debug_counter_test.cc
@@ -125,6 +125,53 @@ HasSubstr(" Total : 5"))); } +TEST(DebugCounterTest, Histograms) { + EXPECT_EXIT( + { + static google::protobuf::internal::RealDebugCounter normal_counter( + "Normal.Counter"); + static google::protobuf::internal::RealDebugCounter log_counter("Log.Counter"); + static google::protobuf::internal::RealDebugCounter linear_counter( + "Linear.Counter"); + + normal_counter.Inc(); + normal_counter.Inc(); + normal_counter.Inc(); + for (int i = 0; i < 100; ++i) { + log_counter.IncLog(i); + } + + linear_counter.IncBucket(3); + linear_counter.IncBucket(3); + linear_counter.IncBucket(4); + linear_counter.IncBucket(7); + linear_counter.IncBucket(7); + linear_counter.IncBucket(7); + exit(0); + }, + ExitedWithCode(0), + HasSubstr( + R"( +---Histograms--- +Linear.Counter: +[ 3]:▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▎ +[ 4]:▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▋ +[ 5]: +[ 6]: +[ 7]:▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉ + +Log.Counter: +[ 0]:▉ +[ 1]:▉ +[ 2]:▉▉ +[ 3]:▉▉▉▉ +[ 4]:▉▉▉▉▉▉▉▉ +[ 5]:▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉ +[ 6]:▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉ +[ 7]:▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉▉ +)")); +} + } // namespace #include "google/protobuf/port_undef.inc"
diff --git a/src/google/protobuf/port.cc b/src/google/protobuf/port.cc index 0111ce0..68b91ba 100644 --- a/src/google/protobuf/port.cc +++ b/src/google/protobuf/port.cc
@@ -7,6 +7,8 @@ // #include "google/protobuf/port.h" +#include <array> +#include <atomic> #include <cstdint> #include <cstdio> #include <cstdlib> @@ -14,11 +16,11 @@ #include <string> #include <utility> #include <variant> -#include <vector> #include "absl/base/attributes.h" #include "absl/log/absl_log.h" #include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/strings/str_split.h" #include "absl/strings/string_view.h" @@ -63,19 +65,57 @@ static void PrintAllCounters(); static auto& CounterMap() { - using Map = std::map<absl::string_view, - std::map<std::variant<int64_t, absl::string_view>, - std::vector<const RealDebugCounter*>>>; + using Map = std::map< + absl::string_view, + std::map<std::variant<int64_t, absl::string_view>, + std::array<std::atomic<size_t>, RealDebugCounter::kNumBuckets>>>; static auto* counter_map = new Map{}; static bool dummy = std::atexit(PrintAllCounters); (void)dummy; return *counter_map; } +static std::string CreateHistogram( + absl::string_view category_name, absl::string_view subname, size_t subtotal, + absl::Span<const std::atomic<size_t>> buckets) { + std::string result; + absl::StrAppendFormat(&result, "%s.%s:\n", category_name, subname); + size_t sub = 0; + for (size_t i = 0; i < buckets.size(); ++i) { + if (sub == subtotal) { + // No more buckets to print. + break; + } + size_t v = buckets[i].load(std::memory_order_relaxed); + if (sub == 0 && v == 0) { + // Skip until the first non-zero bucket. + continue; + } + + sub += v; + double num_chars = v * 100.0 / subtotal; + static constexpr std::array kBlocks = {"▏", "▎", "▍", "▋", "▊", "▉"}; + std::string chars; + while (num_chars > 1) { + chars += kBlocks.back(); + --num_chars; + } + int last_char = std::round(num_chars * kBlocks.size()); + if (last_char > 0) { + chars += kBlocks[last_char - 1]; + } + + absl::StrAppendFormat(&result, "[%2d]:%s\n", i, chars); + } + return result; +} + static void PrintAllCounters() { auto& counters = CounterMap(); if (counters.empty()) return; absl::FPrintF(stderr, "Protobuf debug counters:\n"); + + std::vector<std::string> histograms; for (auto& [category_name, category_map] : counters) { // Example output: // @@ -86,14 +126,25 @@ absl::FPrintF(stderr, " %-12s:\n", category_name); size_t total = 0; for (auto& entry : category_map) { - for (auto* counter : entry.second) { - total += counter->value(); + size_t subtotal = 0; + for (auto& counter : entry.second) { + subtotal += counter.load(std::memory_order_relaxed); } + if (subtotal != entry.second[0].load(std::memory_order_relaxed)) { + histograms.push_back( + CreateHistogram(category_name, + std::holds_alternative<int64_t>(entry.first) + ? absl::StrCat(std::get<int64_t>(entry.first)) + : std::get<absl::string_view>(entry.first), + subtotal, entry.second)); + } + total += subtotal; } - for (auto& [subname, counter_vector] : category_map) { + + for (auto& [subname, counter_array] : category_map) { size_t value = 0; - for (auto* counter : counter_vector) { - value += counter->value(); + for (auto& counter : counter_array) { + value += counter.load(std::memory_order_relaxed); } if (std::holds_alternative<int64_t>(subname)) { // For integers, right align @@ -115,6 +166,13 @@ absl::FPrintF(stderr, " %-10s: %10zu\n", "Total", total); } } + + if (!histograms.empty()) { + absl::FPrintF(stderr, "---Histograms---\n"); + for (const auto& h : histograms) { + absl::FPrintF(stderr, "%s\n", h); + } + } } void RealDebugCounter::Register(absl::string_view name) { @@ -122,9 +180,9 @@ absl::StrSplit(name, '.'); int64_t as_int; if (absl::SimpleAtoi(parts.second, &as_int)) { - CounterMap()[parts.first][as_int].push_back(this); + counters_ = CounterMap()[parts.first][as_int].data(); } else { - CounterMap()[parts.first][parts.second].push_back(this); + counters_ = CounterMap()[parts.first][parts.second].data(); } }
diff --git a/src/google/protobuf/port.h b/src/google/protobuf/port.h index 94ad82b..d0e2c51 100644 --- a/src/google/protobuf/port.h +++ b/src/google/protobuf/port.h
@@ -32,6 +32,7 @@ #include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/base/dynamic_annotations.h" +#include "absl/numeric/bits.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" @@ -778,16 +779,33 @@ // PROTOBUF_DEBUG_COUNTER("Foo.Slow").Inc(); // ... // } +// +// It also supports bucket based distributions. It has two methods: +// +// PROTOBUF_DEBUG_COUNTER("Foo.Slow").IncLog(x); +// +// where `x` is a uint64_t value and it will add the value to the log-based +// bucket for it. +// +// PROTOBUF_DEBUG_COUNTER("Foo.Slow").IncBucket(x); +// +// where `x` is in the range [0,64] and increases the bucket directly. class PROTOBUF_EXPORT RealDebugCounter { public: + static constexpr size_t kNumBuckets = 64; explicit RealDebugCounter(absl::string_view name) { Register(name); } - // Lossy increment. - void Inc() { counter_.store(value() + 1, std::memory_order_relaxed); } - size_t value() const { return counter_.load(std::memory_order_relaxed); } + void Inc() { IncBucket(0); } + void IncLog(uint64_t value) { IncBucket(absl::bit_width(value)); } + void IncBucket(size_t b) { + // clamp to prevent UB if IncBucket is called out of range. + b %= kNumBuckets; + // Lossy increment. + counters_[b].store(counters_[b].load(std::memory_order_relaxed) + 1); + } private: void Register(absl::string_view name); - std::atomic<size_t> counter_{}; + std::atomic<size_t>* counters_; }; // When the feature is not enabled, the type is a noop. @@ -795,6 +813,8 @@ public: explicit constexpr NoopDebugCounter() = default; constexpr void Inc() {} + constexpr void IncLog(uint64_t) {} + constexpr void IncBucket(size_t) {} }; // Pretty random large number that seems like a safe allocation on most systems.