Roll abseil_revision 322ae2420d..b2dcbba183
Change Log:
https://chromium.googlesource.com/external/github.com/abseil/abseil-cpp/+log/322ae2420d..b2dcbba183
Full diff:
https://chromium.googlesource.com/external/github.com/abseil/abseil-cpp/+/322ae2420d..b2dcbba183
Bug: None
Change-Id: I7f4a4235d5eb8b57bfbdb0e07039f975c76b7e6c
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2640441
Reviewed-by: Mirko Bonadei <mbonadei@chromium.org>
Commit-Queue: Danil Chapovalov <danilchap@chromium.org>
Cr-Commit-Position: refs/heads/master@{#845631}
GitOrigin-RevId: 7b42cd6053da3d3babc87b949dfb60518c022dd2
diff --git a/README.chromium b/README.chromium
index 4bff005..225b00d 100644
--- a/README.chromium
+++ b/README.chromium
@@ -4,7 +4,7 @@
License: Apache 2.0
License File: LICENSE
Version: 0
-Revision: 322ae2420d27fc96d0a8ab1167d7de33671048df
+Revision: b2dcbba18341d75f3fef486b717585cefda0195d
Security Critical: yes
Description:
diff --git a/absl/base/internal/spinlock.cc b/absl/base/internal/spinlock.cc
index a7d44f3..35c0696 100644
--- a/absl/base/internal/spinlock.cc
+++ b/absl/base/internal/spinlock.cc
@@ -125,8 +125,9 @@
// it as having a sleeper.
if ((lock_value & kWaitTimeMask) == 0) {
// Here, just "mark" that the thread is going to sleep. Don't store the
- // lock wait time in the lock as that will cause the current lock
- // owner to think it experienced contention.
+ // lock wait time in the lock -- the lock word stores the amount of time
+ // that the current holder waited before acquiring the lock, not the wait
+ // time of any thread currently waiting to acquire it.
if (lockword_.compare_exchange_strong(
lock_value, lock_value | kSpinLockSleeper,
std::memory_order_relaxed, std::memory_order_relaxed)) {
@@ -140,6 +141,14 @@
// this thread obtains the lock.
lock_value = TryLockInternal(lock_value, wait_cycles);
continue; // Skip the delay at the end of the loop.
+ } else if ((lock_value & kWaitTimeMask) == 0) {
+ // The lock is still held, without a waiter being marked, but something
+ // else about the lock word changed, causing our CAS to fail. For
+ // example, a new lock holder may have acquired the lock with
+ // kSpinLockDisabledScheduling set, whereas the previous holder had not
+ // set that flag. In this case, attempt again to mark ourselves as a
+ // waiter.
+ continue;
}
}
diff --git a/absl/base/internal/spinlock.h b/absl/base/internal/spinlock.h
index dce1c85..c73b5e0 100644
--- a/absl/base/internal/spinlock.h
+++ b/absl/base/internal/spinlock.h
@@ -137,8 +137,20 @@
//
// bit[0] encodes whether a lock is being held.
// bit[1] encodes whether a lock uses cooperative scheduling.
- // bit[2] encodes whether a lock disables scheduling.
+ // bit[2] encodes whether the current lock holder disabled scheduling when
+ // acquiring the lock. Only set when kSpinLockHeld is also set.
// bit[3:31] encodes time a lock spent on waiting as a 29-bit unsigned int.
+ // This is set by the lock holder to indicate how long it waited on
+ // the lock before eventually acquiring it. The number of cycles is
+ // encoded as a 29-bit unsigned int, or in the case that the current
+ // holder did not wait but another waiter is queued, the LSB
+ // (kSpinLockSleeper) is set. The implementation does not explicitly
+ // track the number of queued waiters beyond this. It must always be
+ // assumed that waiters may exist if the current holder was required to
+ // queue.
+ //
+ // Invariant: if the lock is not held, the value is either 0 or
+ // kSpinLockCooperative.
static constexpr uint32_t kSpinLockHeld = 1;
static constexpr uint32_t kSpinLockCooperative = 2;
static constexpr uint32_t kSpinLockDisabledScheduling = 4;
diff --git a/absl/strings/cord.cc b/absl/strings/cord.cc
index 7cadf75..c118bdd 100644
--- a/absl/strings/cord.cc
+++ b/absl/strings/cord.cc
@@ -208,9 +208,9 @@
size_t n = 0;
do {
const size_t len = std::min(length, kMaxFlatLength);
- CordRep* rep = CordRepFlat::New(len + alloc_hint);
+ CordRepFlat* rep = CordRepFlat::New(len + alloc_hint);
rep->length = len;
- memcpy(rep->data, data, len);
+ memcpy(rep->Data(), data, len);
reps[n++] = VerifyTree(rep);
data += len;
length -= len;
@@ -272,10 +272,10 @@
return data_.as_tree.rep;
}
- CordRep* result = CordRepFlat::New(len + extra_hint);
+ CordRepFlat* result = CordRepFlat::New(len + extra_hint);
result->length = len;
static_assert(kMinFlatLength >= sizeof(data_.as_chars), "");
- memcpy(result->data, data_.as_chars, sizeof(data_.as_chars));
+ memcpy(result->Data(), data_.as_chars, sizeof(data_.as_chars));
set_tree(result);
return result;
}
@@ -349,7 +349,7 @@
}
dst->length += size_increase;
- *region = dst->data + in_use;
+ *region = dst->flat()->Data() + in_use;
*size = size_increase;
return true;
}
@@ -381,7 +381,7 @@
CordRepFlat* new_node =
CordRepFlat::New(std::max(static_cast<size_t>(root->length), max_length));
new_node->length = std::min(new_node->Capacity(), max_length);
- *region = new_node->data;
+ *region = new_node->Data();
*size = new_node->length;
replace_tree(Concat(root, new_node));
}
@@ -407,7 +407,7 @@
// Allocate new node.
CordRepFlat* new_node = CordRepFlat::New(root->length);
new_node->length = new_node->Capacity();
- *region = new_node->data;
+ *region = new_node->Data();
*size = new_node->length;
replace_tree(Concat(root, new_node));
}
@@ -523,7 +523,7 @@
tree->flat()->Capacity() >= length &&
tree->refcount.IsOne()) {
// Copy in place if the existing FLAT node is reusable.
- memmove(tree->data, data, length);
+ memmove(tree->flat()->Data(), data, length);
tree->length = length;
VerifyTree(tree);
return *this;
@@ -578,8 +578,8 @@
root = CordRepFlat::New(std::max<size_t>(size1, size2));
appended = std::min(
src_size, root->flat()->Capacity() - inline_length);
- memcpy(root->data, data_.as_chars, inline_length);
- memcpy(root->data + inline_length, src_data, appended);
+ memcpy(root->flat()->Data(), data_.as_chars, inline_length);
+ memcpy(root->flat()->Data() + inline_length, src_data, appended);
root->length = inline_length + appended;
set_tree(root);
}
@@ -635,7 +635,7 @@
}
if (src_tree->tag >= FLAT) {
// src tree just has one flat node.
- contents_.AppendArray(src_tree->data, src_size);
+ contents_.AppendArray(src_tree->flat()->Data(), src_size);
return;
}
if (&src == this) {
@@ -1093,7 +1093,7 @@
CordRep* node = tree();
if (node->tag >= FLAT) {
- return absl::string_view(node->data, node->length);
+ return absl::string_view(node->flat()->Data(), node->length);
}
if (node->tag == EXTERNAL) {
@@ -1116,7 +1116,7 @@
}
if (node->tag >= FLAT) {
- return absl::string_view(node->data + offset, length);
+ return absl::string_view(node->flat()->Data() + offset, length);
}
assert((node->tag == EXTERNAL) && "Expect FLAT or EXTERNAL node here");
@@ -1329,7 +1329,7 @@
assert(node->tag == EXTERNAL || node->tag >= FLAT);
assert(length != 0);
const char* data =
- node->tag == EXTERNAL ? node->external()->base : node->data;
+ node->tag == EXTERNAL ? node->external()->base : node->flat()->Data();
current_chunk_ = absl::string_view(data + offset, length);
current_leaf_ = node;
return *this;
@@ -1362,8 +1362,8 @@
// Range to read is a proper subrange of the current chunk.
assert(current_leaf_ != nullptr);
CordRep* subnode = CordRep::Ref(current_leaf_);
- const char* data =
- subnode->tag == EXTERNAL ? subnode->external()->base : subnode->data;
+ const char* data = subnode->tag == EXTERNAL ? subnode->external()->base
+ : subnode->flat()->Data();
subnode = NewSubstring(subnode, current_chunk_.data() - data, n);
subcord.contents_.set_tree(VerifyTree(subnode));
RemoveChunkPrefix(n);
@@ -1375,8 +1375,8 @@
assert(current_leaf_ != nullptr);
CordRep* subnode = CordRep::Ref(current_leaf_);
if (current_chunk_.size() < subnode->length) {
- const char* data =
- subnode->tag == EXTERNAL ? subnode->external()->base : subnode->data;
+ const char* data = subnode->tag == EXTERNAL ? subnode->external()->base
+ : subnode->flat()->Data();
subnode = NewSubstring(subnode, current_chunk_.data() - data,
current_chunk_.size());
}
@@ -1444,7 +1444,7 @@
subnode = Concat(subnode, NewSubstring(CordRep::Ref(node), offset, n));
}
const char* data =
- node->tag == EXTERNAL ? node->external()->base : node->data;
+ node->tag == EXTERNAL ? node->external()->base : node->flat()->Data();
current_chunk_ = absl::string_view(data + offset + n, length - n);
current_leaf_ = node;
bytes_remaining_ -= n;
@@ -1511,7 +1511,7 @@
assert(node->tag == EXTERNAL || node->tag >= FLAT);
assert(length > n);
const char* data =
- node->tag == EXTERNAL ? node->external()->base : node->data;
+ node->tag == EXTERNAL ? node->external()->base : node->flat()->Data();
current_chunk_ = absl::string_view(data + offset + n, length - n);
current_leaf_ = node;
bytes_remaining_ -= n;
@@ -1529,7 +1529,7 @@
assert(offset < rep->length);
if (rep->tag >= FLAT) {
// Get the "i"th character directly from the flat array.
- return rep->data[offset];
+ return rep->flat()->Data()[offset];
} else if (rep->tag == EXTERNAL) {
// Get the "i"th character from the external array.
return rep->external()->base[offset];
@@ -1562,7 +1562,7 @@
if (total_size <= kMaxFlatLength) {
new_rep = CordRepFlat::New(total_size);
new_rep->length = total_size;
- new_buffer = new_rep->data;
+ new_buffer = new_rep->flat()->Data();
CopyToArraySlowPath(new_buffer);
} else {
new_buffer = std::allocator<char>().allocate(total_size);
@@ -1583,7 +1583,7 @@
/* static */ bool Cord::GetFlatAux(CordRep* rep, absl::string_view* fragment) {
assert(rep != nullptr);
if (rep->tag >= FLAT) {
- *fragment = absl::string_view(rep->data, rep->length);
+ *fragment = absl::string_view(rep->flat()->Data(), rep->length);
return true;
} else if (rep->tag == EXTERNAL) {
*fragment = absl::string_view(rep->external()->base, rep->length);
@@ -1591,8 +1591,8 @@
} else if (rep->tag == SUBSTRING) {
CordRep* child = rep->substring()->child;
if (child->tag >= FLAT) {
- *fragment =
- absl::string_view(child->data + rep->substring()->start, rep->length);
+ *fragment = absl::string_view(
+ child->flat()->Data() + rep->substring()->start, rep->length);
return true;
} else if (child->tag == EXTERNAL) {
*fragment = absl::string_view(
@@ -1680,7 +1680,7 @@
*os << "FLAT cap=" << rep->flat()->Capacity()
<< " [";
if (include_data)
- *os << absl::CEscape(std::string(rep->data, rep->length));
+ *os << absl::CEscape(std::string(rep->flat()->Data(), rep->length));
*os << "]\n";
}
if (stack.empty()) break;
diff --git a/absl/strings/internal/cord_internal.h b/absl/strings/internal/cord_internal.h
index 57e7046..b586ea3 100644
--- a/absl/strings/internal/cord_internal.h
+++ b/absl/strings/internal/cord_internal.h
@@ -166,7 +166,7 @@
struct CordRep {
CordRep() = default;
constexpr CordRep(Refcount::Immortal immortal, size_t l)
- : length(l), refcount(immortal), tag(EXTERNAL), data{} {}
+ : length(l), refcount(immortal), tag(EXTERNAL), storage{} {}
// The following three fields have to be less than 32 bytes since
// that is the smallest supported flat node size.
@@ -175,7 +175,7 @@
// If tag < FLAT, it represents CordRepKind and indicates the type of node.
// Otherwise, the node type is CordRepFlat and the tag is the encoded size.
uint8_t tag;
- char data[1]; // Starting point for flat array: MUST BE LAST FIELD of CordRep
+ char storage[1]; // Starting point for flat array: MUST BE LAST FIELD
inline CordRepConcat* concat();
inline const CordRepConcat* concat() const;
@@ -219,8 +219,8 @@
CordRep* left;
CordRep* right;
- uint8_t depth() const { return static_cast<uint8_t>(data[0]); }
- void set_depth(uint8_t depth) { data[0] = static_cast<char>(depth); }
+ uint8_t depth() const { return static_cast<uint8_t>(storage[0]); }
+ void set_depth(uint8_t depth) { storage[0] = static_cast<char>(depth); }
};
struct CordRepSubstring : public CordRep {
diff --git a/absl/strings/internal/cord_rep_flat.h b/absl/strings/internal/cord_rep_flat.h
index 80391a5..8c7d160 100644
--- a/absl/strings/internal/cord_rep_flat.h
+++ b/absl/strings/internal/cord_rep_flat.h
@@ -37,7 +37,7 @@
// ideally a 'nice' size aligning with allocation and cacheline sizes like 32.
// kMaxFlatSize is bounded by the size resulting in a computed tag no greater
// than MAX_FLAT_TAG. MAX_FLAT_TAG provides for additional 'high' tag values.
-static constexpr size_t kFlatOverhead = offsetof(CordRep, data);
+static constexpr size_t kFlatOverhead = offsetof(CordRep, storage);
static constexpr size_t kMinFlatSize = 32;
static constexpr size_t kMaxFlatSize = 4096;
static constexpr size_t kMaxFlatLength = kMaxFlatSize - kFlatOverhead;
@@ -115,6 +115,9 @@
#endif
}
+ char* Data() { return storage; }
+ const char* Data() const { return storage; }
+
// Returns the maximum capacity (payload size) of this instance.
size_t Capacity() const { return TagToLength(tag); }
diff --git a/absl/time/clock.cc b/absl/time/clock.cc
index 6862e01..c61dcc0 100644
--- a/absl/time/clock.cc
+++ b/absl/time/clock.cc
@@ -15,6 +15,7 @@
#include "absl/time/clock.h"
#include "absl/base/attributes.h"
+#include "absl/base/optimization.h"
#ifdef _WIN32
#include <windows.h>
@@ -85,13 +86,6 @@
::absl::time_internal::UnscaledCycleClockWrapperForGetCurrentTime::Now()
#endif
-// The following counters are used only by the test code.
-static int64_t stats_initializations;
-static int64_t stats_reinitializations;
-static int64_t stats_calibrations;
-static int64_t stats_slow_paths;
-static int64_t stats_fast_slow_paths;
-
namespace absl {
ABSL_NAMESPACE_BEGIN
namespace time_internal {
@@ -105,72 +99,6 @@
// uint64_t is used in this module to provide an extra bit in multiplications
-// Return the time in ns as told by the kernel interface. Place in *cycleclock
-// the value of the cycleclock at about the time of the syscall.
-// This call represents the time base that this module synchronizes to.
-// Ensures that *cycleclock does not step back by up to (1 << 16) from
-// last_cycleclock, to discard small backward counter steps. (Larger steps are
-// assumed to be complete resyncs, which shouldn't happen. If they do, a full
-// reinitialization of the outer algorithm should occur.)
-static int64_t GetCurrentTimeNanosFromKernel(uint64_t last_cycleclock,
- uint64_t *cycleclock) {
- // We try to read clock values at about the same time as the kernel clock.
- // This value gets adjusted up or down as estimate of how long that should
- // take, so we can reject attempts that take unusually long.
- static std::atomic<uint64_t> approx_syscall_time_in_cycles{10 * 1000};
-
- uint64_t local_approx_syscall_time_in_cycles = // local copy
- approx_syscall_time_in_cycles.load(std::memory_order_relaxed);
-
- int64_t current_time_nanos_from_system;
- uint64_t before_cycles;
- uint64_t after_cycles;
- uint64_t elapsed_cycles;
- int loops = 0;
- do {
- before_cycles = GET_CURRENT_TIME_NANOS_CYCLECLOCK_NOW();
- current_time_nanos_from_system = GET_CURRENT_TIME_NANOS_FROM_SYSTEM();
- after_cycles = GET_CURRENT_TIME_NANOS_CYCLECLOCK_NOW();
- // elapsed_cycles is unsigned, so is large on overflow
- elapsed_cycles = after_cycles - before_cycles;
- if (elapsed_cycles >= local_approx_syscall_time_in_cycles &&
- ++loops == 20) { // clock changed frequencies? Back off.
- loops = 0;
- if (local_approx_syscall_time_in_cycles < 1000 * 1000) {
- local_approx_syscall_time_in_cycles =
- (local_approx_syscall_time_in_cycles + 1) << 1;
- }
- approx_syscall_time_in_cycles.store(
- local_approx_syscall_time_in_cycles,
- std::memory_order_relaxed);
- }
- } while (elapsed_cycles >= local_approx_syscall_time_in_cycles ||
- last_cycleclock - after_cycles < (static_cast<uint64_t>(1) << 16));
-
- // Number of times in a row we've seen a kernel time call take substantially
- // less than approx_syscall_time_in_cycles.
- static std::atomic<uint32_t> seen_smaller{ 0 };
-
- // Adjust approx_syscall_time_in_cycles to be within a factor of 2
- // of the typical time to execute one iteration of the loop above.
- if ((local_approx_syscall_time_in_cycles >> 1) < elapsed_cycles) {
- // measured time is no smaller than half current approximation
- seen_smaller.store(0, std::memory_order_relaxed);
- } else if (seen_smaller.fetch_add(1, std::memory_order_relaxed) >= 3) {
- // smaller delays several times in a row; reduce approximation by 12.5%
- const uint64_t new_approximation =
- local_approx_syscall_time_in_cycles -
- (local_approx_syscall_time_in_cycles >> 3);
- approx_syscall_time_in_cycles.store(new_approximation,
- std::memory_order_relaxed);
- seen_smaller.store(0, std::memory_order_relaxed);
- }
-
- *cycleclock = after_cycles;
- return current_time_nanos_from_system;
-}
-
-
// ---------------------------------------------------------------------
// An implementation of reader-write locks that use no atomic ops in the read
// case. This is a generalization of Lamport's method for reading a multiword
@@ -222,32 +150,110 @@
kMinNSBetweenSamples,
"cannot represent kMaxBetweenSamplesNSScaled");
-// A reader-writer lock protecting the static locations below.
-// See SeqAcquire() and SeqRelease() above.
-ABSL_CONST_INIT static absl::base_internal::SpinLock lock(
- absl::kConstInit, base_internal::SCHEDULE_KERNEL_ONLY);
-ABSL_CONST_INIT static std::atomic<uint64_t> seq(0);
-
// data from a sample of the kernel's time value
struct TimeSampleAtomic {
- std::atomic<uint64_t> raw_ns; // raw kernel time
- std::atomic<uint64_t> base_ns; // our estimate of time
- std::atomic<uint64_t> base_cycles; // cycle counter reading
- std::atomic<uint64_t> nsscaled_per_cycle; // cycle period
+ std::atomic<uint64_t> raw_ns{0}; // raw kernel time
+ std::atomic<uint64_t> base_ns{0}; // our estimate of time
+ std::atomic<uint64_t> base_cycles{0}; // cycle counter reading
+ std::atomic<uint64_t> nsscaled_per_cycle{0}; // cycle period
// cycles before we'll sample again (a scaled reciprocal of the period,
// to avoid a division on the fast path).
- std::atomic<uint64_t> min_cycles_per_sample;
+ std::atomic<uint64_t> min_cycles_per_sample{0};
};
// Same again, but with non-atomic types
struct TimeSample {
- uint64_t raw_ns; // raw kernel time
- uint64_t base_ns; // our estimate of time
- uint64_t base_cycles; // cycle counter reading
- uint64_t nsscaled_per_cycle; // cycle period
- uint64_t min_cycles_per_sample; // approx cycles before next sample
+ uint64_t raw_ns = 0; // raw kernel time
+ uint64_t base_ns = 0; // our estimate of time
+ uint64_t base_cycles = 0; // cycle counter reading
+ uint64_t nsscaled_per_cycle = 0; // cycle period
+ uint64_t min_cycles_per_sample = 0; // approx cycles before next sample
};
-static struct TimeSampleAtomic last_sample; // the last sample; under seq
+struct ABSL_CACHELINE_ALIGNED TimeState {
+ std::atomic<uint64_t> seq{0};
+ TimeSampleAtomic last_sample; // the last sample; under seq
+
+ // The following counters are used only by the test code.
+ int64_t stats_initializations{0};
+ int64_t stats_reinitializations{0};
+ int64_t stats_calibrations{0};
+ int64_t stats_slow_paths{0};
+ int64_t stats_fast_slow_paths{0};
+
+ uint64_t last_now_cycles ABSL_GUARDED_BY(lock){0};
+
+ // Used by GetCurrentTimeNanosFromKernel().
+ // We try to read clock values at about the same time as the kernel clock.
+ // This value gets adjusted up or down as estimate of how long that should
+ // take, so we can reject attempts that take unusually long.
+ std::atomic<uint64_t> approx_syscall_time_in_cycles{10 * 1000};
+ // Number of times in a row we've seen a kernel time call take substantially
+ // less than approx_syscall_time_in_cycles.
+ std::atomic<uint32_t> kernel_time_seen_smaller{0};
+
+ // A reader-writer lock protecting the static locations below.
+ // See SeqAcquire() and SeqRelease() above.
+ absl::base_internal::SpinLock lock{absl::kConstInit,
+ base_internal::SCHEDULE_KERNEL_ONLY};
+};
+ABSL_CONST_INIT static TimeState time_state{};
+
+// Return the time in ns as told by the kernel interface. Place in *cycleclock
+// the value of the cycleclock at about the time of the syscall.
+// This call represents the time base that this module synchronizes to.
+// Ensures that *cycleclock does not step back by up to (1 << 16) from
+// last_cycleclock, to discard small backward counter steps. (Larger steps are
+// assumed to be complete resyncs, which shouldn't happen. If they do, a full
+// reinitialization of the outer algorithm should occur.)
+static int64_t GetCurrentTimeNanosFromKernel(uint64_t last_cycleclock,
+ uint64_t *cycleclock)
+ ABSL_EXCLUSIVE_LOCKS_REQUIRED(time_state.lock) {
+ uint64_t local_approx_syscall_time_in_cycles = // local copy
+ time_state.approx_syscall_time_in_cycles.load(std::memory_order_relaxed);
+
+ int64_t current_time_nanos_from_system;
+ uint64_t before_cycles;
+ uint64_t after_cycles;
+ uint64_t elapsed_cycles;
+ int loops = 0;
+ do {
+ before_cycles = GET_CURRENT_TIME_NANOS_CYCLECLOCK_NOW();
+ current_time_nanos_from_system = GET_CURRENT_TIME_NANOS_FROM_SYSTEM();
+ after_cycles = GET_CURRENT_TIME_NANOS_CYCLECLOCK_NOW();
+ // elapsed_cycles is unsigned, so is large on overflow
+ elapsed_cycles = after_cycles - before_cycles;
+ if (elapsed_cycles >= local_approx_syscall_time_in_cycles &&
+ ++loops == 20) { // clock changed frequencies? Back off.
+ loops = 0;
+ if (local_approx_syscall_time_in_cycles < 1000 * 1000) {
+ local_approx_syscall_time_in_cycles =
+ (local_approx_syscall_time_in_cycles + 1) << 1;
+ }
+ time_state.approx_syscall_time_in_cycles.store(
+ local_approx_syscall_time_in_cycles, std::memory_order_relaxed);
+ }
+ } while (elapsed_cycles >= local_approx_syscall_time_in_cycles ||
+ last_cycleclock - after_cycles < (static_cast<uint64_t>(1) << 16));
+
+ // Adjust approx_syscall_time_in_cycles to be within a factor of 2
+ // of the typical time to execute one iteration of the loop above.
+ if ((local_approx_syscall_time_in_cycles >> 1) < elapsed_cycles) {
+ // measured time is no smaller than half current approximation
+ time_state.kernel_time_seen_smaller.store(0, std::memory_order_relaxed);
+ } else if (time_state.kernel_time_seen_smaller.fetch_add(
+ 1, std::memory_order_relaxed) >= 3) {
+ // smaller delays several times in a row; reduce approximation by 12.5%
+ const uint64_t new_approximation =
+ local_approx_syscall_time_in_cycles -
+ (local_approx_syscall_time_in_cycles >> 3);
+ time_state.approx_syscall_time_in_cycles.store(new_approximation,
+ std::memory_order_relaxed);
+ time_state.kernel_time_seen_smaller.store(0, std::memory_order_relaxed);
+ }
+
+ *cycleclock = after_cycles;
+ return current_time_nanos_from_system;
+}
static int64_t GetCurrentTimeNanosSlowPath() ABSL_ATTRIBUTE_COLD;
@@ -315,14 +321,15 @@
// Acquire pairs with the barrier in SeqRelease - if this load sees that
// store, the shared-data reads necessarily see that SeqRelease's updates
// to the same shared data.
- seq_read0 = seq.load(std::memory_order_acquire);
+ seq_read0 = time_state.seq.load(std::memory_order_acquire);
- base_ns = last_sample.base_ns.load(std::memory_order_relaxed);
- base_cycles = last_sample.base_cycles.load(std::memory_order_relaxed);
+ base_ns = time_state.last_sample.base_ns.load(std::memory_order_relaxed);
+ base_cycles =
+ time_state.last_sample.base_cycles.load(std::memory_order_relaxed);
nsscaled_per_cycle =
- last_sample.nsscaled_per_cycle.load(std::memory_order_relaxed);
- min_cycles_per_sample =
- last_sample.min_cycles_per_sample.load(std::memory_order_relaxed);
+ time_state.last_sample.nsscaled_per_cycle.load(std::memory_order_relaxed);
+ min_cycles_per_sample = time_state.last_sample.min_cycles_per_sample.load(
+ std::memory_order_relaxed);
// This acquire fence pairs with the release fence in SeqAcquire. Since it
// is sequenced between reads of shared data and seq_read1, the reads of
@@ -333,7 +340,7 @@
// shared-data writes are effectively release ordered. Therefore if our
// shared-data reads see any of a particular update's shared-data writes,
// seq_read1 is guaranteed to see that update's SeqAcquire.
- seq_read1 = seq.load(std::memory_order_relaxed);
+ seq_read1 = time_state.seq.load(std::memory_order_relaxed);
// Fast path. Return if min_cycles_per_sample has not yet elapsed since the
// last sample, and we read a consistent sample. The fast path activates
@@ -346,9 +353,9 @@
// last_sample was updated). This is harmless, because delta_cycles will wrap
// and report a time much much bigger than min_cycles_per_sample. In that case
// we will take the slow path.
- uint64_t delta_cycles = now_cycles - base_cycles;
+ uint64_t delta_cycles;
if (seq_read0 == seq_read1 && (seq_read0 & 1) == 0 &&
- delta_cycles < min_cycles_per_sample) {
+ (delta_cycles = now_cycles - base_cycles) < min_cycles_per_sample) {
return base_ns + ((delta_cycles * nsscaled_per_cycle) >> kScale);
}
return GetCurrentTimeNanosSlowPath();
@@ -388,24 +395,25 @@
// TODO(absl-team): Remove this attribute when our compiler is smart enough
// to do the right thing.
ABSL_ATTRIBUTE_NOINLINE
-static int64_t GetCurrentTimeNanosSlowPath() ABSL_LOCKS_EXCLUDED(lock) {
+static int64_t GetCurrentTimeNanosSlowPath()
+ ABSL_LOCKS_EXCLUDED(time_state.lock) {
// Serialize access to slow-path. Fast-path readers are not blocked yet, and
// code below must not modify last_sample until the seqlock is acquired.
- lock.Lock();
+ time_state.lock.Lock();
// Sample the kernel time base. This is the definition of
// "now" if we take the slow path.
- static uint64_t last_now_cycles; // protected by lock
uint64_t now_cycles;
- uint64_t now_ns = GetCurrentTimeNanosFromKernel(last_now_cycles, &now_cycles);
- last_now_cycles = now_cycles;
+ uint64_t now_ns =
+ GetCurrentTimeNanosFromKernel(time_state.last_now_cycles, &now_cycles);
+ time_state.last_now_cycles = now_cycles;
uint64_t estimated_base_ns;
// ----------
// Read the "last_sample" values again; this time holding the write lock.
struct TimeSample sample;
- ReadTimeSampleAtomic(&last_sample, &sample);
+ ReadTimeSampleAtomic(&time_state.last_sample, &sample);
// ----------
// Try running the fast path again; another thread may have updated the
@@ -416,13 +424,13 @@
// so that blocked readers can make progress without blocking new readers.
estimated_base_ns = sample.base_ns +
((delta_cycles * sample.nsscaled_per_cycle) >> kScale);
- stats_fast_slow_paths++;
+ time_state.stats_fast_slow_paths++;
} else {
estimated_base_ns =
UpdateLastSample(now_cycles, now_ns, delta_cycles, &sample);
}
- lock.Unlock();
+ time_state.lock.Unlock();
return estimated_base_ns;
}
@@ -433,9 +441,10 @@
static uint64_t UpdateLastSample(uint64_t now_cycles, uint64_t now_ns,
uint64_t delta_cycles,
const struct TimeSample *sample)
- ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock) {
+ ABSL_EXCLUSIVE_LOCKS_REQUIRED(time_state.lock) {
uint64_t estimated_base_ns = now_ns;
- uint64_t lock_value = SeqAcquire(&seq); // acquire seqlock to block readers
+ uint64_t lock_value =
+ SeqAcquire(&time_state.seq); // acquire seqlock to block readers
// The 5s in the next if-statement limits the time for which we will trust
// the cycle counter and our last sample to give a reasonable result.
@@ -445,12 +454,16 @@
sample->raw_ns + static_cast<uint64_t>(5) * 1000 * 1000 * 1000 < now_ns ||
now_ns < sample->raw_ns || now_cycles < sample->base_cycles) {
// record this sample, and forget any previously known slope.
- last_sample.raw_ns.store(now_ns, std::memory_order_relaxed);
- last_sample.base_ns.store(estimated_base_ns, std::memory_order_relaxed);
- last_sample.base_cycles.store(now_cycles, std::memory_order_relaxed);
- last_sample.nsscaled_per_cycle.store(0, std::memory_order_relaxed);
- last_sample.min_cycles_per_sample.store(0, std::memory_order_relaxed);
- stats_initializations++;
+ time_state.last_sample.raw_ns.store(now_ns, std::memory_order_relaxed);
+ time_state.last_sample.base_ns.store(estimated_base_ns,
+ std::memory_order_relaxed);
+ time_state.last_sample.base_cycles.store(now_cycles,
+ std::memory_order_relaxed);
+ time_state.last_sample.nsscaled_per_cycle.store(0,
+ std::memory_order_relaxed);
+ time_state.last_sample.min_cycles_per_sample.store(
+ 0, std::memory_order_relaxed);
+ time_state.stats_initializations++;
} else if (sample->raw_ns + 500 * 1000 * 1000 < now_ns &&
sample->base_cycles + 50 < now_cycles) {
// Enough time has passed to compute the cycle time.
@@ -493,28 +506,32 @@
if (new_nsscaled_per_cycle != 0 &&
diff_ns < 100 * 1000 * 1000 && -diff_ns < 100 * 1000 * 1000) {
// record the cycle time measurement
- last_sample.nsscaled_per_cycle.store(
+ time_state.last_sample.nsscaled_per_cycle.store(
new_nsscaled_per_cycle, std::memory_order_relaxed);
uint64_t new_min_cycles_per_sample =
SafeDivideAndScale(kMinNSBetweenSamples, new_nsscaled_per_cycle);
- last_sample.min_cycles_per_sample.store(
+ time_state.last_sample.min_cycles_per_sample.store(
new_min_cycles_per_sample, std::memory_order_relaxed);
- stats_calibrations++;
+ time_state.stats_calibrations++;
} else { // something went wrong; forget the slope
- last_sample.nsscaled_per_cycle.store(0, std::memory_order_relaxed);
- last_sample.min_cycles_per_sample.store(0, std::memory_order_relaxed);
+ time_state.last_sample.nsscaled_per_cycle.store(
+ 0, std::memory_order_relaxed);
+ time_state.last_sample.min_cycles_per_sample.store(
+ 0, std::memory_order_relaxed);
estimated_base_ns = now_ns;
- stats_reinitializations++;
+ time_state.stats_reinitializations++;
}
- last_sample.raw_ns.store(now_ns, std::memory_order_relaxed);
- last_sample.base_ns.store(estimated_base_ns, std::memory_order_relaxed);
- last_sample.base_cycles.store(now_cycles, std::memory_order_relaxed);
+ time_state.last_sample.raw_ns.store(now_ns, std::memory_order_relaxed);
+ time_state.last_sample.base_ns.store(estimated_base_ns,
+ std::memory_order_relaxed);
+ time_state.last_sample.base_cycles.store(now_cycles,
+ std::memory_order_relaxed);
} else {
// have a sample, but no slope; waiting for enough time for a calibration
- stats_slow_paths++;
+ time_state.stats_slow_paths++;
}
- SeqRelease(&seq, lock_value); // release the readers
+ SeqRelease(&time_state.seq, lock_value); // release the readers
return estimated_base_ns;
}
diff --git a/symbols_arm64_dbg.def b/symbols_arm64_dbg.def
index df0b9fe..646e104 100644
--- a/symbols_arm64_dbg.def
+++ b/symbols_arm64_dbg.def
@@ -981,6 +981,7 @@
??0Time@absl@@QEAA@XZ
??0TimeConversion@absl@@QEAA@XZ
??0TimeInfo@TimeZone@absl@@QEAA@XZ
+ ??0TimeSample@absl@@QEAA@XZ
??0TimeZone@absl@@QEAA@Vtime_zone@cctz@time_internal@1@@Z
??0TimeZoneIf@cctz@time_internal@absl@@IEAA@XZ
??0TimeZoneInfo@cctz@time_internal@absl@@QEAA@XZ
@@ -1512,6 +1513,8 @@
?Crash@Helper@internal_statusor@absl@@SAXAEBVStatus@3@@Z
?CreateThreadIdentity@synchronization_internal@absl@@YAPEAUThreadIdentity@base_internal@2@XZ
?CurrentThreadIdentityIfPresent@base_internal@absl@@YAPEAUThreadIdentity@12@XZ
+ ?Data@CordRepFlat@cord_internal@absl@@QEAAPEADXZ
+ ?Data@CordRepFlat@cord_internal@absl@@QEBAPEBDXZ
?DataLength@Header@TimeZoneInfo@cctz@time_internal@absl@@QEBA_K_K@Z
?DataLossError@absl@@YA?AVStatus@1@Vstring_view@1@@Z
?DeadlineExceededError@absl@@YA?AVStatus@1@Vstring_view@1@@Z
diff --git a/symbols_x64_dbg.def b/symbols_x64_dbg.def
index 4f45507..9b6617e 100644
--- a/symbols_x64_dbg.def
+++ b/symbols_x64_dbg.def
@@ -983,6 +983,7 @@
??0Time@absl@@QEAA@XZ
??0TimeConversion@absl@@QEAA@XZ
??0TimeInfo@TimeZone@absl@@QEAA@XZ
+ ??0TimeSample@absl@@QEAA@XZ
??0TimeZone@absl@@QEAA@Vtime_zone@cctz@time_internal@1@@Z
??0TimeZoneIf@cctz@time_internal@absl@@IEAA@XZ
??0TimeZoneInfo@cctz@time_internal@absl@@QEAA@XZ
@@ -1513,6 +1514,8 @@
?Crash@Helper@internal_statusor@absl@@SAXAEBVStatus@3@@Z
?CreateThreadIdentity@synchronization_internal@absl@@YAPEAUThreadIdentity@base_internal@2@XZ
?CurrentThreadIdentityIfPresent@base_internal@absl@@YAPEAUThreadIdentity@12@XZ
+ ?Data@CordRepFlat@cord_internal@absl@@QEAAPEADXZ
+ ?Data@CordRepFlat@cord_internal@absl@@QEBAPEBDXZ
?DataLength@Header@TimeZoneInfo@cctz@time_internal@absl@@QEBA_K_K@Z
?DataLossError@absl@@YA?AVStatus@1@Vstring_view@1@@Z
?DeadlineExceededError@absl@@YA?AVStatus@1@Vstring_view@1@@Z
diff --git a/symbols_x86_dbg.def b/symbols_x86_dbg.def
index 74d7457..741966d 100644
--- a/symbols_x86_dbg.def
+++ b/symbols_x86_dbg.def
@@ -981,6 +981,7 @@
??0Time@absl@@QAE@XZ
??0TimeConversion@absl@@QAE@XZ
??0TimeInfo@TimeZone@absl@@QAE@XZ
+ ??0TimeSample@absl@@QAE@XZ
??0TimeZone@absl@@QAE@Vtime_zone@cctz@time_internal@1@@Z
??0TimeZoneIf@cctz@time_internal@absl@@IAE@XZ
??0TimeZoneInfo@cctz@time_internal@absl@@QAE@XZ
@@ -1510,6 +1511,8 @@
?Crash@Helper@internal_statusor@absl@@SAXABVStatus@3@@Z
?CreateThreadIdentity@synchronization_internal@absl@@YAPAUThreadIdentity@base_internal@2@XZ
?CurrentThreadIdentityIfPresent@base_internal@absl@@YAPAUThreadIdentity@12@XZ
+ ?Data@CordRepFlat@cord_internal@absl@@QAEPADXZ
+ ?Data@CordRepFlat@cord_internal@absl@@QBEPBDXZ
?DataLength@Header@TimeZoneInfo@cctz@time_internal@absl@@QBEII@Z
?DataLossError@absl@@YA?AVStatus@1@Vstring_view@1@@Z
?DeadlineExceededError@absl@@YA?AVStatus@1@Vstring_view@1@@Z
