chore: more efficient access to FiberInterface epoch field (#388)

Done to improve efficiency of fiber context switch operations.

Signed-off-by: Roman Gershman <romange@gmail.com>

Author: romange

util/fibers/detail/fiber_interface.cc

@@ -53,6 +53,8 @@ PMR_NS::memory_resource* default_stack_resource = nullptr;
 size_t default_stack_size = 64 * 1024;
 uint64_t g_tsc_cycles_per_ms = 0;
 
+__thread FiberInterface::TL FiberInterface::tl;
+
 // Per thread initialization structure.
 struct TL_FiberInitializer {
   TL_FiberInitializer* next = nullptr;
@@ -63,7 +65,6 @@ struct TL_FiberInitializer {
   // Per-thread scheduler instance.
   // Allows overriding the main dispatch loop
   Scheduler* sched;
-  uint64_t epoch = 0;
   uint64_t switch_delay_cycles = 0;  // switch delay in cycles.
 
   // Tracks fiber runtimes that took longer than 1ms.
@@ -379,7 +380,7 @@ FiberInterface* FiberInterface::SwitchSetup() {
   // When a kernel suspends we may get a negative delta because TSC is reset.
   // We ignore such cases (and they are very rare).
   if (tsc > cpu_tsc_) {
-    ++fb_initializer.epoch;
+    ++tl.epoch;
     DCHECK_GE(tsc, to_suspend->cpu_tsc_);
     fb_initializer.switch_delay_cycles += (tsc - cpu_tsc_);
 
@@ -437,10 +438,6 @@ void SetCustomDispatcher(DispatchPolicy* policy) {
   fb_init.sched->AttachCustomPolicy(policy);
 }
 
-uint64_t FiberSwitchEpoch() noexcept {
-  return detail::FbInitializer().epoch;
-}
-
 uint64_t FiberSwitchDelayUsec() noexcept {
   // in nanoseconds, so lets convert from cycles
   return detail::FbInitializer().switch_delay_cycles * 1000 / detail::g_tsc_cycles_per_ms;

util/fibers/detail/fiber_interface.h

@@ -223,6 +223,9 @@ class FiberInterface {
 
   size_t GetStackMargin(const void* stack_address) const;
 
+  static uint64_t TL_Epoch() {
+    return tl.epoch;
+  }
  protected:
   static constexpr uint16_t kTerminatedBit = 0x1;
   static constexpr uint16_t kBusyBit = 0x2;
@@ -262,6 +265,10 @@ class FiberInterface {
   uint32_t stack_size_ = 0;
   uint8_t* stack_bottom_ = nullptr;
 
+  static __thread struct TL {
+    uint64_t epoch = 0;
+  } tl;
+
  private:
 #ifndef NDEBUG
   std::function<std::string()> stacktrace_print_cb_;

util/fibers/fibers.h

@@ -103,7 +103,9 @@ class Fiber {
 };
 
 // Returns the context switch epoch number for this thread.
-uint64_t FiberSwitchEpoch() noexcept;
+inline uint64_t FiberSwitchEpoch() noexcept {
+  return detail::FiberInterface::TL_Epoch();
+}
 
 // Returns the aggregated delay between activation of fibers and
 // the time they were switched to in microseconds.