From 34f5177efa1eadec560069d3c4f2df522c668516 Mon Sep 17 00:00:00 2001 From: Denis Vaksman Date: Sat, 9 Dec 2023 23:11:35 +0300 Subject: [PATCH] Make master outlive confdata out of shared memory errors (#946) - events throttling approach with soft/hard OOM mode - external allocation heuristic checks to prevent OOM - timeout for reading binlog on updates to prevent master freezing - new emergency option for blacklisting confdata events - extend confdata monitoring in StatsHouse and Graphana Co-authored-by: Aleksandr Kirsanov --- runtime/array_decl.inl | 4 + runtime/confdata-global-manager.cpp | 5 +- runtime/confdata-global-manager.h | 5 +- runtime/confdata-keys.cpp | 9 + runtime/confdata-keys.h | 20 +- server/confdata-binlog-events.h | 4 + server/confdata-binlog-replay.cpp | 396 ++++++++++++++---- server/confdata-binlog-replay.h | 7 + server/confdata-stats.cpp | 32 +- server/confdata-stats.h | 21 +- server/php-engine-vars.cpp | 1 + server/php-engine-vars.h | 1 + server/php-engine.cpp | 30 +- server/signal-handlers.cpp | 2 +- server/statshouse/statshouse-manager.cpp | 43 +- server/statshouse/statshouse-manager.h | 8 +- tests/cpp/runtime/confdata-functions-test.cpp | 3 +- tests/python/lib/engine.py | 14 +- tests/python/lib/file_utils.py | 4 +- tests/python/tests/json_logs/test_signals.py | 4 +- 20 files changed, 506 insertions(+), 107 deletions(-) diff --git a/runtime/array_decl.inl b/runtime/array_decl.inl index b74921bd16..7071751743 100644 --- a/runtime/array_decl.inl +++ b/runtime/array_decl.inl @@ -252,6 +252,10 @@ public: // we can create true vector bool has_no_string_keys() const noexcept; + static size_t estimate_size(int64_t n, bool is_vector) noexcept { + return array_inner::estimate_size(n, is_vector); + } + T &operator[](int64_t int_key); T &operator[](int32_t key) { return (*this)[int64_t{key}]; } T &operator[](const string &s); diff --git a/runtime/confdata-global-manager.cpp b/runtime/confdata-global-manager.cpp index 7bb0ba6f44..db029550e8 100644 --- a/runtime/confdata-global-manager.cpp +++ b/runtime/confdata-global-manager.cpp @@ -95,11 +95,12 @@ ConfdataGlobalManager &ConfdataGlobalManager::get() noexcept { void ConfdataGlobalManager::init(size_t confdata_memory_limit, std::unordered_set &&predefined_wilrdcards, - std::unique_ptr &&blacklist_pattern) noexcept { + std::unique_ptr &&blacklist_pattern, + std::forward_list &&force_ignore_prefixes) noexcept { resource_.init(mmap_shared(confdata_memory_limit), confdata_memory_limit); confdata_samples_.init(resource_); predefined_wildcards_.set_wildcards(std::move(predefined_wilrdcards)); - key_blacklist_.set_blacklist(std::move(blacklist_pattern)); + key_blacklist_.set_blacklist(std::move(blacklist_pattern), std::move(force_ignore_prefixes)); } ConfdataGlobalManager::~ConfdataGlobalManager() noexcept { diff --git a/runtime/confdata-global-manager.h b/runtime/confdata-global-manager.h index 455366f126..57d223a275 100644 --- a/runtime/confdata-global-manager.h +++ b/runtime/confdata-global-manager.h @@ -51,8 +51,9 @@ class ConfdataGlobalManager : vk::not_copyable { static ConfdataGlobalManager &get() noexcept; void init(size_t confdata_memory_limit, - std::unordered_set &&predefined_wilrdcards, - std::unique_ptr &&blacklist_pattern) noexcept; + std::unordered_set &&predefined_wilrdcards, + std::unique_ptr &&blacklist_pattern, + std::forward_list &&force_ignore_prefixes) noexcept; void force_release_all_resources_acquired_by_this_proc_if_init() noexcept { if (is_initialized()) { diff --git a/runtime/confdata-keys.cpp b/runtime/confdata-keys.cpp index 5b17abdf7e..3e96c4054d 100644 --- a/runtime/confdata-keys.cpp +++ b/runtime/confdata-keys.cpp @@ -131,3 +131,12 @@ void ConfdataKeyMaker::forcibly_change_first_key_wildcard_dots_from_two_to_one() second_key_ = string::make_const_string_on_memory(first_key_end, second_key_len, second_key_buffer_.data(), second_key_buffer_.size()); first_key_type_ = ConfdataFirstKeyType::one_dot_wildcard; } + +bool ConfdataKeyBlacklist::is_key_forcibly_ignored_by_prefix(vk::string_view key) const noexcept { + for (const vk::string_view &prefix : force_ignore_prefixes_) { + if (key.starts_with(prefix)) { + return true; + } + } + return false; +} diff --git a/runtime/confdata-keys.h b/runtime/confdata-keys.h index 9821db893c..10a5720b4c 100644 --- a/runtime/confdata-keys.h +++ b/runtime/confdata-keys.h @@ -5,6 +5,7 @@ #pragma once #include #include +#include #include #include #include @@ -112,18 +113,31 @@ class ConfdataKeyMaker : vk::not_copyable { }; class ConfdataKeyBlacklist : vk::not_copyable { + bool is_key_forcibly_ignored_by_prefix(vk::string_view key) const noexcept; + public: - void set_blacklist(std::unique_ptr &&blacklist_pattern) noexcept { + void set_blacklist(std::unique_ptr &&blacklist_pattern, std::forward_list &&force_ignore_prefixes) noexcept { blacklist_pattern_ = std::move(blacklist_pattern); + force_ignore_prefixes_ = std::move(force_ignore_prefixes); } bool is_blacklisted(vk::string_view key) const noexcept { - return blacklist_pattern_ && + // from PHP class KphpConfiguration, e.g. for langs + if (blacklist_pattern_ && re2::RE2::FullMatch( re2::StringPiece(key.data(), static_cast(key.size())), - *blacklist_pattern_); + *blacklist_pattern_)) { + return true; + } + // emergency startup option to disable keys by prefix, e.g. 'highload.vid' + if (unlikely(!force_ignore_prefixes_.empty()) && + is_key_forcibly_ignored_by_prefix(key)) { + return true; + } + return false; } private: std::unique_ptr blacklist_pattern_; + std::forward_list force_ignore_prefixes_; }; diff --git a/server/confdata-binlog-events.h b/server/confdata-binlog-events.h index a4689f03dc..bfbe8c2dc2 100644 --- a/server/confdata-binlog-events.h +++ b/server/confdata-binlog-events.h @@ -90,6 +90,10 @@ struct lev_confdata_store_wrapper : impl_::confdata_event #include #include +#include #include #include +#include #include "common/binlog/binlog-replayer.h" #include "common/dl-utils-lite.h" @@ -25,13 +27,31 @@ #include "server/confdata-binlog-events.h" #include "server/confdata-stats.h" #include "server/server-log.h" +#include "server/statshouse/statshouse-manager.h" namespace { +struct { + const char *binlog_mask{nullptr}; + size_t memory_limit{2u * 1024u * 1024u * 1024u}; + double soft_oom_threshold_ratio{CONFDATA_DEFAULT_SOFT_OOM_RATIO}; + double hard_oom_threshold_ratio{CONFDATA_DEFAULT_HARD_OOM_RATIO}; + double confdata_update_timeout_sec = 0.3; + std::unique_ptr key_blacklist_pattern; + std::forward_list force_ignore_prefixes; + std::unordered_set predefined_wildcards; + + bool is_enabled() const noexcept { + return binlog_mask; + } +} confdata_settings; + class ConfdataBinlogReplayer : vk::binlog::replayer { public: enum class OperationStatus { no_update, + throttled_out, + timed_out, blacklisted, ttl_update_only, full_update @@ -44,6 +64,40 @@ class ConfdataBinlogReplayer : vk::binlog::replayer { using vk::binlog::replayer::replay; + // To prevent OOM errors on confdata overflows we start to throttle events, when too little shared memory left: + // Soft OOM (85%) -- replay only existing keys related events until server restart + // Hard OOM (95%) -- ignore all events, don't read binlog at all until server restart + enum class MemoryStatus { + NORMAL, + SOFT_OOM, + HARD_OOM + }; + + MemoryStatus current_memory_status() noexcept { + update_memory_status(); + if (hard_oom_reached_) return MemoryStatus::HARD_OOM; + if (soft_oom_reached_) return MemoryStatus::SOFT_OOM; + return MemoryStatus::NORMAL; + } + + bool check_has_enough_memory(size_t need_bytes, const char *msg) noexcept { + if (memory_resource_->is_enough_memory_for(need_bytes)) { + return true; + } + log_server_critical("Not enough confdata shared memory. Processing key with first part = '%s'. %zu bytes needed by estimation for %s", + processing_key_.get_first_key().c_str(), need_bytes, msg); + update_memory_status(true); + return false; + } + + void raise_confdata_oom_error(const char *msg) const noexcept { + log_server_critical("%s: too little confdata shared memory left (%zu used / %zu limit), %zu events throttled", + msg, + memory_resource_->get_memory_stats().real_memory_used, + memory_resource_->get_memory_stats().memory_limit, + event_counters_.throttled_out_total_events); + } + size_t try_reserve_for_snapshot(vk::string_view key, size_t search_from, vk::string_view &prev_key, array_size &counter) noexcept { auto dot_pos = key.find('.', search_from); @@ -121,21 +175,36 @@ class ConfdataBinlogReplayer : vk::binlog::replayer { } blacklist_enabled_ = true; size_hints_.clear(); + + if (current_memory_status() != MemoryStatus::NORMAL) { + raise_confdata_oom_error("Can't read confdata snapshot on start"); + return -1; + } + return 0; } OperationStatus delete_element(const char *key, short key_len) noexcept { - return generic_operation(key, key_len, -1, [this] { return delete_processing_element(); }); + auto memory_status = current_memory_status(); + return generic_operation(key, key_len, -1, memory_status, [this] (MemoryStatus memory_status) { + return delete_processing_element(memory_status); + }); } OperationStatus touch_element(const lev_confdata_touch &E) noexcept { - return generic_operation(E.key, static_cast(E.key_len), E.delay, [this] { return touch_processing_element(); }); + auto memory_status = current_memory_status(); + return generic_operation(E.key, static_cast(E.key_len), E.delay, memory_status, [this] (MemoryStatus memory_status) { + return touch_processing_element(memory_status); + }); } template OperationStatus store_element(const lev_confdata_store_wrapper &E) noexcept { + auto memory_status = current_memory_status(); // don't even try to capture E by value; it'll try to copy it and it would be very sad :( - return generic_operation(E.data, E.key_len, E.get_delay(), [this, &E] { return store_processing_element(E); }); + return generic_operation(E.data, E.key_len, E.get_delay(), memory_status, [this, &E] (MemoryStatus memory_status) { + return store_processing_element(E, memory_status); + }); } void unsupported_operation(const char *operation_name, const char *key, int key_len) noexcept { @@ -146,6 +215,10 @@ class ConfdataBinlogReplayer : vk::binlog::replayer { void init(memory_resource::unsynchronized_pool_resource &memory_pool) noexcept { assert(!updating_confdata_storage_); updating_confdata_storage_ = new(&confdata_mem_)confdata_sample_storage{confdata_sample_storage::allocator_type{memory_pool}}; + + memory_resource_ = &memory_pool; + soft_oom_memory_limit_ = static_cast(std::floor(confdata_settings.soft_oom_threshold_ratio * confdata_settings.memory_limit)); + hard_oom_memory_limit_ = static_cast(std::floor(confdata_settings.hard_oom_threshold_ratio * confdata_settings.memory_limit)); } struct ConfdataUpdateResult { @@ -179,10 +252,13 @@ class ConfdataBinlogReplayer : vk::binlog::replayer { return result; } - void try_use_previous_confdata_storage_as_init(const confdata_sample_storage &previous_confdata_storage) noexcept { + bool try_use_previous_confdata_storage_as_init(const confdata_sample_storage &previous_confdata_storage) noexcept { if (!confdata_has_any_updates_) { assert(garbage_from_previous_confdata_sample_->empty()); if (updating_confdata_storage_->empty()) { + if (!check_has_enough_memory(confdata_sample_storage::allocator_type::max_value_type_size() * previous_confdata_storage.size(), "RB-tree copying")) { + return false; + } *updating_confdata_storage_ = previous_confdata_storage; } else { // strictly speaking, they should be identical, but it's too hard to verify @@ -191,6 +267,7 @@ class ConfdataBinlogReplayer : vk::binlog::replayer { "but they have different sizes (%zu != %zu)\n", updating_confdata_storage_->size(), previous_confdata_storage.size())); } } + return true; } bool has_new_confdata() const noexcept { @@ -207,7 +284,10 @@ class ConfdataBinlogReplayer : vk::binlog::replayer { return; } assert(head->second.size() <= std::numeric_limits::max()); - delete_element(head->second.c_str(), static_cast(head->second.size())); + auto res = delete_element(head->second.c_str(), static_cast(head->second.size())); + if (res == OperationStatus::throttled_out || res == OperationStatus::timed_out) { + return; + } assert(expired_elements == expiration_trace_.size() + 1); expired_elements = expiration_trace_.size(); } @@ -220,59 +300,90 @@ class ConfdataBinlogReplayer : vk::binlog::replayer { const ConfdataStats::EventCounters &get_event_counters() const noexcept { return event_counters_; } + + void on_start_update_cycle(double update_timeout_sec) noexcept { + binlog_update_start_time_point_ = std::chrono::steady_clock::now(); + if (update_timeout_sec > 0) { + update_timeout_sec_ = std::chrono::duration{update_timeout_sec}; + } + } + bool on_finish_update_cycle() noexcept { + bool ok = !is_update_timeout_expired(); + update_timeout_sec_.reset(); + return ok; + } + + bool is_update_timeout_expired() const noexcept { + return update_timeout_sec_.has_value() && + std::chrono::steady_clock::now() - binlog_update_start_time_point_ > update_timeout_sec_; + } private: ConfdataBinlogReplayer() noexcept: garbage_from_previous_confdata_sample_(new(&garbage_mem_) GarbageList{}), key_blacklist_(ConfdataGlobalManager::get().get_key_blacklist()), predefined_wildcards_(ConfdataGlobalManager::get().get_predefined_wildcards()) { + add_handler([this](const lev_confdata_delete &E) { - update_event_stat(this->delete_element(E.key, E.key_len), event_counters_.delete_events); + return finish_operation(delete_element(E.key, E.key_len), event_counters_.delete_events); }); add_handler([this](const lev_confdata_touch &E) { - update_event_stat(this->touch_element(E), event_counters_.touch_events); + return finish_operation(touch_element(E), event_counters_.touch_events); }); add_handler([this](const lev_confdata_store_wrapper &E) { - update_event_stat(this->store_element(E), event_counters_.add_events); + return finish_operation(store_element(E), event_counters_.add_events); }); add_handler([this](const lev_confdata_store_wrapper &E) { - update_event_stat(this->store_element(E), event_counters_.set_events); + return finish_operation(store_element(E), event_counters_.set_events); }); add_handler([this](const lev_confdata_store_wrapper &E) { - update_event_stat(this->store_element(E), event_counters_.replace_events); + return finish_operation(store_element(E), event_counters_.replace_events); }); add_handler([this](const lev_confdata_store_wrapper &E) { - update_event_stat(this->store_element(E), event_counters_.add_forever_events); + return finish_operation(store_element(E), event_counters_.add_forever_events); }); add_handler([this](const lev_confdata_store_wrapper &E) { - update_event_stat(this->store_element(E), event_counters_.set_forever_events); + return finish_operation(store_element(E), event_counters_.set_forever_events); }); add_handler([this](const lev_confdata_store_wrapper &E) { - update_event_stat(this->store_element(E), event_counters_.replace_forever_events); + return finish_operation(store_element(E), event_counters_.replace_forever_events); }); add_handler([this](const lev_confdata_get &E) { ++event_counters_.get_events; - this->unsupported_operation("get", E.key, E.key_len); + unsupported_operation("get", E.key, E.key_len); }); add_handler([this](const lev_confdata_incr &E) { ++event_counters_.incr_events; - this->unsupported_operation("incr", E.key, E.key_len); + unsupported_operation("incr", E.key, E.key_len); }); add_handler_range([this](const lev_confdata_incr_tiny_range &E) { ++event_counters_.incr_tiny_events; - this->unsupported_operation("incr tiny", E.key, E.key_len); + unsupported_operation("incr tiny", E.key, E.key_len); }); add_handler([this](const lev_confdata_append &E) { ++event_counters_.append_events; - this->unsupported_operation("append", E.data, E.key_len); + unsupported_operation("append", E.data, E.key_len); }); } + void update_memory_status(bool force_oom = false) noexcept { + auto cur_usage = memory_resource_->get_memory_stats().real_memory_used; + hard_oom_reached_ = hard_oom_reached_ || cur_usage > hard_oom_memory_limit_ || force_oom; + soft_oom_reached_ = soft_oom_reached_ || cur_usage > soft_oom_memory_limit_ || force_oom; + } + template - OperationStatus generic_operation(const char *key, short key_len, int delay, const F &operation) noexcept { + OperationStatus generic_operation(const char *key, short key_len, int delay, MemoryStatus memory_status, const F &operation) noexcept { + // memory_status is unchanged during the whole operation + if (is_update_timeout_expired()) { + return OperationStatus::timed_out; + } + if (memory_status == MemoryStatus::HARD_OOM) { + return OperationStatus::throttled_out; + } // TODO assert? if (key_len < 0) { return OperationStatus::blacklisted; @@ -286,32 +397,40 @@ class ConfdataBinlogReplayer : vk::binlog::replayer { assert(processing_value_.is_null()); static_assert(std::is_same{}, "short is expected to be int16_t"); + // The single key can be copied upto 3 times in std::map. It's special denormalization for predefined wildcards and two-dot keys. + // So `operation()` can be applied upto 3 times for the single log event. + // There's an invariant that all these copies are consistent. See asserts below. + OperationStatus last_operation_status{OperationStatus::no_update}; const auto predefined_wildcard_lengths = predefined_wildcards_.make_predefined_wildcard_len_range_by_key(key_view); for (size_t wildcard_len : predefined_wildcard_lengths) { assert(wildcard_len <= std::numeric_limits::max()); processing_key_.update_with_predefined_wildcard(key, key_len, static_cast(wildcard_len)); - const auto operation_status = operation(); - assert(last_operation_status != OperationStatus::full_update || - operation_status == OperationStatus::full_update); + const auto operation_status = operation(memory_status); + if (operation_status == OperationStatus::throttled_out) { + return OperationStatus::throttled_out; + } + assert(last_operation_status != OperationStatus::full_update || operation_status == OperationStatus::full_update); last_operation_status = operation_status; if (operation_status != OperationStatus::full_update) { break; } } - if (predefined_wildcard_lengths.empty() || - last_operation_status == OperationStatus::full_update) { + if (predefined_wildcard_lengths.empty() || last_operation_status == OperationStatus::full_update) { const auto first_key_type = processing_key_.update(key, key_len); - if (predefined_wildcard_lengths.empty() || - first_key_type != ConfdataFirstKeyType::simple_key) { - const auto operation_status = operation(); - assert(last_operation_status != OperationStatus::full_update || - operation_status == OperationStatus::full_update); - if (operation_status == OperationStatus::full_update && - first_key_type == ConfdataFirstKeyType::two_dots_wildcard) { + if (predefined_wildcard_lengths.empty() || first_key_type != ConfdataFirstKeyType::simple_key) { + const auto operation_status = operation(memory_status); + if (operation_status == OperationStatus::throttled_out) { + return OperationStatus::throttled_out; + } + assert(last_operation_status != OperationStatus::full_update || operation_status == OperationStatus::full_update); + if (operation_status == OperationStatus::full_update && first_key_type == ConfdataFirstKeyType::two_dots_wildcard) { processing_key_.forcibly_change_first_key_wildcard_dots_from_two_to_one(); - const auto should_be_full = operation(); + const auto should_be_full = operation(memory_status); + if (operation_status == OperationStatus::throttled_out) { + return OperationStatus::throttled_out; + } assert(should_be_full == OperationStatus::full_update); } last_operation_status = operation_status; @@ -332,24 +451,40 @@ class ConfdataBinlogReplayer : vk::binlog::replayer { return last_operation_status; } - static void update_event_stat(OperationStatus status, ConfdataStats::EventCounters::Event &event) noexcept { - ++event.total; + replay_binlog_result finish_operation(OperationStatus status, ConfdataStats::EventCounters::Event &event) noexcept { switch (status) { case OperationStatus::no_update: ++event.ignored; - return; + break; + case OperationStatus::throttled_out: + ++event.throttled_out; + ++event_counters_.throttled_out_total_events; + break; case OperationStatus::blacklisted: ++event.blacklisted; - return; + break; case OperationStatus::ttl_update_only: ++event.ttl_updated; - return; + break; + case OperationStatus::timed_out: case OperationStatus::full_update: - return; + break; + } + if (current_memory_status() == MemoryStatus::HARD_OOM) { + return REPLAY_BINLOG_STOP_READING; + } + if (status == OperationStatus::timed_out) { + ++ConfdataStats::get().timed_out_updates; + return REPLAY_BINLOG_STOP_READING; } + ++event.total; + return REPLAY_BINLOG_OK; } - OperationStatus delete_processing_element() noexcept { + OperationStatus delete_processing_element(MemoryStatus memory_status) noexcept { + if (memory_status == MemoryStatus::HARD_OOM) { + return OperationStatus::throttled_out; + } auto first_key_it = updating_confdata_storage_->find(processing_key_.get_first_key()); if (first_key_it == updating_confdata_storage_->end()) { return OperationStatus::no_update; @@ -370,6 +505,11 @@ class ConfdataBinlogReplayer : vk::binlog::replayer { return OperationStatus::no_update; } + if (array_for_second_key.get_reference_counter() > 1) { // means it's shared and need to be detached before modifying + if (!check_has_enough_memory(array_for_second_key.calculate_memory_for_copying(), "array_for_second_key copying on delete")) { + return OperationStatus::throttled_out; + } + } // move deleted element data to garbage; it will be a copy with RC detachment put_confdata_var_into_garbage(array_for_second_key, ConfdataGarbageDestroyWay::shallow_first); @@ -393,7 +533,10 @@ class ConfdataBinlogReplayer : vk::binlog::replayer { return OperationStatus::full_update; } - OperationStatus touch_processing_element() noexcept { + OperationStatus touch_processing_element(MemoryStatus memory_status) noexcept { + if (memory_status == MemoryStatus::HARD_OOM) { + return OperationStatus::throttled_out; + } auto first_key_it = updating_confdata_storage_->find(processing_key_.get_first_key()); if (first_key_it == updating_confdata_storage_->end()) { return OperationStatus::no_update; @@ -412,11 +555,27 @@ class ConfdataBinlogReplayer : vk::binlog::replayer { } template - OperationStatus store_processing_element(const lev_confdata_store_wrapper &E) noexcept { + OperationStatus store_processing_element(const lev_confdata_store_wrapper &E, MemoryStatus memory_status) noexcept { + if (memory_status == MemoryStatus::HARD_OOM) { + return OperationStatus::throttled_out; + } + + size_t bytes_for_node_emplacement = confdata_sample_storage::allocator_type::max_value_type_size(); + size_t bytes_for_keys_copying = processing_key_.get_first_key().estimate_memory_usage() + processing_key_.get_second_key().estimate_memory_usage(); + size_t bytes_for_value_creating = 2 * 5 * E.get_data_size(); // 5 is just an approximate upper bound for zlib decoding factor + // by according to https://www.zlib.net/zlib_tech.html + // And twice larger just in case + size_t need_bytes_upper_bound_without_arrays_for_second_keys = bytes_for_node_emplacement + bytes_for_keys_copying + bytes_for_value_creating; + if (!check_has_enough_memory(need_bytes_upper_bound_without_arrays_for_second_keys, "for storing single entry")) { + return OperationStatus::throttled_out; + } + auto first_key_it = updating_confdata_storage_->find(processing_key_.get_first_key()); - bool element_exists = true; - if (first_key_it == updating_confdata_storage_->end()) { - element_exists = false; + bool element_exists = first_key_it != updating_confdata_storage_->end(); + if (!element_exists) { + if (memory_status == MemoryStatus::SOFT_OOM) { + return OperationStatus::throttled_out; + } // during the snapshot loading all keys are already sorted, so it makes sense to push it back first_key_it = updating_confdata_storage_->emplace_hint(first_key_it, processing_key_.make_first_key_copy(), mixed{}); } @@ -436,17 +595,50 @@ class ConfdataBinlogReplayer : vk::binlog::replayer { return OperationStatus::ttl_update_only; } + // here and below inner arrays appear (a.k.a. `array_for_second_key`) + // they need for keys with dots and predefined wilcards + assert(vk::any_of_equal(processing_key_.get_first_key_type(), + ConfdataFirstKeyType::one_dot_wildcard, + ConfdataFirstKeyType::two_dots_wildcard, + ConfdataFirstKeyType::predefined_wildcard)); + // null is inserted by the default if (first_key_it->second.is_null()) { - first_key_it->second = prepare_array_for(vk::string_view{first_key_it->first.c_str(), first_key_it->first.size()}); + if (memory_status == MemoryStatus::SOFT_OOM) { // todo: unreachable? + first_key_it->second = array{}; // to fit asserts that it's array + return OperationStatus::throttled_out; + } + // create array for keys parts after dot (a.k.a. `array_for_second_key`) + auto size_hint_it = size_hints_.find(vk::string_view{first_key_it->first.c_str(), first_key_it->first.size()}); + if (size_hint_it == size_hints_.end()) { + first_key_it->second = array{}; + } else { + if (!check_has_enough_memory(array::estimate_size(size_hint_it->second.size, size_hint_it->second.is_vector) + + need_bytes_upper_bound_without_arrays_for_second_keys, + "array_for_second_key creating on store")) { + first_key_it->second = array{}; + return OperationStatus::throttled_out; + } + first_key_it->second = array{size_hint_it->second}; + } } assert(first_key_it->second.is_array()); auto &array_for_second_key = first_key_it->second.as_array(); const auto *prev_value = element_exists ? array_for_second_key.find_value(processing_key_.get_second_key()) : nullptr; + + if (!prev_value && memory_status == MemoryStatus::SOFT_OOM) { + // in soft OOM we ignore new keys + return OperationStatus::throttled_out; + } if (!can_element_be_saved(E, prev_value != nullptr)) { return OperationStatus::no_update; } + size_t bytes_for_copying_array_for_second_key = 3 * array_for_second_key.calculate_memory_for_copying(); // for possible copying and reallocation on insert + if (!check_has_enough_memory(bytes_for_copying_array_for_second_key + need_bytes_upper_bound_without_arrays_for_second_keys, + "array_for_second_key copying on store")) { + return OperationStatus::throttled_out; + } if (!prev_value) { // move old element data to garbage; it will be a copy with RC detachment put_confdata_var_into_garbage(array_for_second_key, ConfdataGarbageDestroyWay::shallow_first); @@ -615,13 +807,6 @@ class ConfdataBinlogReplayer : vk::binlog::replayer { return processing_value_; } - array prepare_array_for(vk::string_view key) const noexcept { - auto size_hint_it = size_hints_.find(key); - return size_hint_it != size_hints_.end() - ? array{size_hint_it->second} - : array{}; - } - bool is_key_blacklisted(vk::string_view key) const noexcept { return blacklist_enabled_ && key_blacklist_.is_blacklisted(key); } @@ -632,6 +817,13 @@ class ConfdataBinlogReplayer : vk::binlog::replayer { using GarbageList = std::forward_list; + const memory_resource::unsynchronized_pool_resource *memory_resource_; + size_t soft_oom_memory_limit_, hard_oom_memory_limit_; + bool soft_oom_reached_{false}, hard_oom_reached_{false}; + + std::chrono::steady_clock::time_point binlog_update_start_time_point_{std::chrono::nanoseconds::zero()}; + std::optional> update_timeout_sec_; + std::aligned_storage_t confdata_mem_; confdata_sample_storage *updating_confdata_storage_{nullptr}; std::aligned_storage_t garbage_mem_; @@ -653,19 +845,12 @@ class ConfdataBinlogReplayer : vk::binlog::replayer { const ConfdataPredefinedWildcards &predefined_wildcards_; }; -struct { - const char *binlog_mask{nullptr}; - size_t memory_limit{2u * 1024u * 1024u * 1024u}; - std::unique_ptr key_blacklist_pattern; - std::unordered_set predefined_wildcards; - - bool is_enabled() const noexcept { - return binlog_mask; - } -} confdata_settings; - } // namespace +void set_confdata_soft_oom_ratio(double soft_oom_ratio) noexcept { + confdata_settings.soft_oom_threshold_ratio = soft_oom_ratio; +} + void set_confdata_binlog_mask(const char *mask) noexcept { confdata_settings.binlog_mask = mask; } @@ -678,6 +863,20 @@ void set_confdata_blacklist_pattern(std::unique_ptr &&key_blacklist_pa confdata_settings.key_blacklist_pattern = std::move(key_blacklist_pattern); } +void set_confdata_update_timeout(double timeout_sec) noexcept { + confdata_settings.confdata_update_timeout_sec = timeout_sec; +} + +void add_confdata_force_ignore_prefix(const char *key_ignore_prefix) noexcept { + assert(key_ignore_prefix && *key_ignore_prefix); + vk::string_view ignore_prefix{key_ignore_prefix}; + // 'highload.vid*' => 'highload.vid' + while (ignore_prefix.ends_with("*")) { + ignore_prefix.remove_suffix(1); + } + confdata_settings.force_ignore_prefixes.emplace_front(ignore_prefix); +} + void add_confdata_predefined_wildcard(const char *wildcard) noexcept { assert(wildcard && *wildcard); vk::string_view wildcard_value{wildcard}; @@ -716,7 +915,8 @@ void init_confdata_binlog_reader() noexcept { auto &confdata_manager = ConfdataGlobalManager::get(); confdata_manager.init(confdata_settings.memory_limit, std::move(confdata_settings.predefined_wildcards), - std::move(confdata_settings.key_blacklist_pattern)); + std::move(confdata_settings.key_blacklist_pattern), + std::move(confdata_settings.force_ignore_prefixes)); dl::set_current_script_allocator(confdata_manager.get_resource(), true); // engine_default_load_index and engine_default_read_binlog call exit(1) on errors, @@ -730,9 +930,11 @@ void init_confdata_binlog_reader() noexcept { auto &confdata_binlog_replayer = ConfdataBinlogReplayer::get(); confdata_binlog_replayer.init(confdata_manager.get_resource()); engine_default_load_index(confdata_settings.binlog_mask); - engine_default_read_binlog(); - confdata_binlog_replayer.delete_expired_elements(); - + update_confdata_state_from_binlog(true, 10 * confdata_settings.confdata_update_timeout_sec); + if (confdata_binlog_replayer.current_memory_status() != ConfdataBinlogReplayer::MemoryStatus::NORMAL) { + confdata_binlog_replayer.raise_confdata_oom_error("Can't read confdata binlog on start"); + exit(1); + } auto loaded_confdata = confdata_binlog_replayer.finish_confdata_update(); assert(loaded_confdata.previous_confdata_garbage.empty()); @@ -753,25 +955,48 @@ void confdata_binlog_update_cron() noexcept { return; } + auto &confdata_binlog_replayer = ConfdataBinlogReplayer::get(); + + switch (confdata_binlog_replayer.current_memory_status()) { + case ConfdataBinlogReplayer::MemoryStatus::HARD_OOM: + confdata_binlog_replayer.raise_confdata_oom_error("Confdata OOM hard - state is freezed until server restart"); + return; + case ConfdataBinlogReplayer::MemoryStatus::SOFT_OOM: + confdata_binlog_replayer.raise_confdata_oom_error("Confdata OOM soft - ignore new key events until server restart"); + break; + case ConfdataBinlogReplayer::MemoryStatus::NORMAL: + break; + } + auto &confdata_stats = ConfdataStats::get(); confdata_stats.total_updating_time -= std::chrono::steady_clock::now().time_since_epoch(); auto &confdata_manager = ConfdataGlobalManager::get(); auto &mem_resource = confdata_manager.get_resource(); dl::set_current_script_allocator(mem_resource, true); + auto rollback_guard = vk::finally([&] { + dl::restore_default_script_allocator(true); + confdata_stats.total_updating_time += std::chrono::steady_clock::now().time_since_epoch(); + }); + auto &previous_confdata_sample = confdata_manager.get_current(); - auto &confdata_binlog_replayer = ConfdataBinlogReplayer::get(); - confdata_binlog_replayer.try_use_previous_confdata_storage_as_init(previous_confdata_sample.get_confdata()); - binlog_try_read_events(); - confdata_binlog_replayer.delete_expired_elements(); + bool ok = confdata_binlog_replayer.try_use_previous_confdata_storage_as_init(previous_confdata_sample.get_confdata()); + if (!ok) { + return; + } + update_confdata_state_from_binlog(false, confdata_settings.confdata_update_timeout_sec); + + if (confdata_binlog_replayer.current_memory_status() == ConfdataBinlogReplayer::MemoryStatus::HARD_OOM) { + return; + } - if (confdata_binlog_replayer.has_new_confdata()){ + if (confdata_binlog_replayer.has_new_confdata()) { if (confdata_manager.can_next_be_updated()) { auto updated_confdata = confdata_binlog_replayer.finish_confdata_update(); - confdata_stats.on_update(updated_confdata.new_confdata, - updated_confdata.previous_confdata_garbage_size, - confdata_manager.get_predefined_wildcards()); + confdata_stats.on_update(updated_confdata.new_confdata, updated_confdata.previous_confdata_garbage_size, confdata_manager.get_predefined_wildcards()); + // save confdata stats here (not from master cron), because pointers to strings (key names) may become incorrect + StatsHouseManager::get().add_confdata_master_stats(confdata_stats); previous_confdata_sample.save_garbage(std::move(updated_confdata.previous_confdata_garbage)); const bool switched = confdata_manager.try_switch_to_next_sample(std::move(updated_confdata.new_confdata)); assert(switched); @@ -781,9 +1006,26 @@ void confdata_binlog_update_cron() noexcept { } confdata_manager.clear_unused_samples(); +} - dl::restore_default_script_allocator(true); - confdata_stats.total_updating_time += std::chrono::steady_clock::now().time_since_epoch(); +bool update_confdata_state_from_binlog(bool is_initial_reading, double timeout_sec) noexcept { + auto &confdata_binlog_replayer = ConfdataBinlogReplayer::get(); + confdata_binlog_replayer.on_start_update_cycle(timeout_sec); + + if (is_initial_reading) { + engine_default_read_binlog(); + } else { + binlog_try_read_events(); + } + + confdata_binlog_replayer.delete_expired_elements(); + + bool ok = confdata_binlog_replayer.on_finish_update_cycle(); + + if (!ok) { + log_server_warning("Confdata binlog %supdate timeout %f sec expired", is_initial_reading ? "initial " : "", timeout_sec); + } + return ok; } void write_confdata_stats_to(stats_t *stats) noexcept { @@ -792,6 +1034,6 @@ void write_confdata_stats_to(stats_t *stats) noexcept { auto &binlog_replayer = ConfdataBinlogReplayer::get(); confdata_stats.elements_with_delay = binlog_replayer.get_elements_with_delay_count(); confdata_stats.event_counters = binlog_replayer.get_event_counters(); - confdata_stats.write_stats_to(stats, ConfdataGlobalManager::get().get_resource().get_memory_stats()); + confdata_stats.write_stats_to(stats); } } diff --git a/server/confdata-binlog-replay.h b/server/confdata-binlog-replay.h index 074ebae5fc..20d30db915 100644 --- a/server/confdata-binlog-replay.h +++ b/server/confdata-binlog-replay.h @@ -9,15 +9,22 @@ #include "runtime/allocator.h" +static constexpr double CONFDATA_DEFAULT_SOFT_OOM_RATIO = 0.85; +static constexpr double CONFDATA_DEFAULT_HARD_OOM_RATIO = 0.95; + +void set_confdata_soft_oom_ratio(double soft_oom_ratio) noexcept; void set_confdata_binlog_mask(const char *mask) noexcept; void set_confdata_memory_limit(size_t memory_limit) noexcept; void set_confdata_blacklist_pattern(std::unique_ptr &&key_blacklist_pattern) noexcept; +void set_confdata_update_timeout(double timeout_sec) noexcept; +void add_confdata_force_ignore_prefix(const char *key_ignore_prefix) noexcept; void add_confdata_predefined_wildcard(const char *wildcard) noexcept; void clear_confdata_predefined_wildcards() noexcept; void init_confdata_binlog_reader() noexcept; void confdata_binlog_update_cron() noexcept; +bool update_confdata_state_from_binlog(bool is_initial_reading, double timeout_sec) noexcept; void write_confdata_stats_to(stats_t *stats) noexcept; diff --git a/server/confdata-stats.cpp b/server/confdata-stats.cpp index 566a1b9624..c1c4c7d360 100644 --- a/server/confdata-stats.cpp +++ b/server/confdata-stats.cpp @@ -16,6 +16,7 @@ void write_event_stats(stats_t *stats, const char *name, const ConfdataStats::Ev stats->add_gauge_stat_with_type_tag(name, "total", event.total); stats->add_gauge_stat_with_type_tag(name, "blacklisted", event.blacklisted); stats->add_gauge_stat_with_type_tag(name, "ignored", event.ignored); + stats->add_gauge_stat_with_type_tag(name, "throttled_out", event.throttled_out); stats->add_gauge_stat_with_type_tag(name, "ttl_updated", event.ttl_updated); } @@ -35,6 +36,8 @@ void ConfdataStats::on_update(const confdata_sample_storage &new_confdata, two_dots_wildcard_elements = 0; predefined_wildcards = 0; predefined_wildcard_elements = 0; + heaviest_sections_by_count.clear(); + for (const auto §ion: new_confdata) { const vk::string_view first_key{section.first.c_str(), section.first.size()}; switch (confdata_predefined_wildcards.detect_first_key_type(first_key)) { @@ -49,6 +52,7 @@ void ConfdataStats::on_update(const confdata_sample_storage &new_confdata, if (!confdata_predefined_wildcards.has_wildcard_for_key(first_key)) { total_elements += section.second.as_array().count(); } + heaviest_sections_by_count.register_section(§ion.first, section.second.as_array().count()); break; } case ConfdataFirstKeyType::two_dots_wildcard: @@ -73,7 +77,13 @@ void ConfdataStats::on_update(const confdata_sample_storage &new_confdata, last_update_time_point = std::chrono::steady_clock::now(); } -void ConfdataStats::write_stats_to(stats_t *stats, const memory_resource::MemoryStats &memory_stats) const noexcept { +const memory_resource::MemoryStats &ConfdataStats::get_memory_stats() const noexcept { + // both ConfdataStats and ConfdataGlobalManager are singletons + return ConfdataGlobalManager::get().get_resource().get_memory_stats(); +} + +void ConfdataStats::write_stats_to(stats_t *stats) const noexcept { + const auto &memory_stats = get_memory_stats(); memory_stats.write_stats_to(stats, "confdata"); stats->add_gauge_stat("confdata.initial_loading_duration", to_seconds(initial_loading_time)); @@ -82,6 +92,7 @@ void ConfdataStats::write_stats_to(stats_t *stats, const memory_resource::Memory stats->add_gauge_stat("confdata.updates.ignored", ignored_updates); stats->add_gauge_stat("confdata.updates.total", total_updates); + stats->add_gauge_stat("confdata.updates.timed_out", timed_out_updates); stats->add_gauge_stat("confdata.elements.total", total_elements); @@ -128,4 +139,23 @@ void ConfdataStats::write_stats_to(stats_t *stats, const memory_resource::Memory stats->add_gauge_stat_with_type_tag("confdata.binlog_events", "incr_tiny", event_counters.incr_tiny_events); stats->add_gauge_stat_with_type_tag("confdata.binlog_events", "append", event_counters.append_events); stats->add_gauge_stat_with_type_tag("confdata.binlog_events", "unsupported_total", event_counters.unsupported_total_events); + stats->add_gauge_stat_with_type_tag("confdata.binlog_events", "throttled_out_total", event_counters.throttled_out_total_events); +} + +void ConfdataStats::HeaviestSections::clear() { + for (int i = 0; i < LEN; ++i) { + sorted_desc[i] = std::make_pair(nullptr, 0); + } +} + +void ConfdataStats::HeaviestSections::register_section(const string *section_name, size_t size) { + for (int ins_pos = 0; ins_pos < LEN; ++ins_pos) { + if (size > sorted_desc[ins_pos].second) { + for (int shift_pos = LEN - 1; shift_pos > ins_pos; --shift_pos) { + sorted_desc[shift_pos] = sorted_desc[shift_pos - 1]; + } + sorted_desc[ins_pos] = std::make_pair(section_name, size); + break; + } + } } diff --git a/server/confdata-stats.h b/server/confdata-stats.h index d538776838..9b9063f00b 100644 --- a/server/confdata-stats.h +++ b/server/confdata-stats.h @@ -21,6 +21,7 @@ struct ConfdataStats : private vk::not_copyable { size_t total_updates{0}; size_t ignored_updates{0}; + size_t timed_out_updates{0}; size_t last_garbage_size{0}; std::array garbage_statistic_{{0}}; @@ -40,6 +41,7 @@ struct ConfdataStats : private vk::not_copyable { size_t total{0}; size_t blacklisted{0}; size_t ignored{0}; + size_t throttled_out{0}; size_t ttl_updated{0}; }; @@ -62,12 +64,29 @@ struct ConfdataStats : private vk::not_copyable { size_t append_events{0}; size_t unsupported_total_events{0}; + size_t throttled_out_total_events{0}; } event_counters; + struct HeaviestSections { + static constexpr int LEN = 10; + + // store pointers, not to copy strings when sorting (they point to array keys located in shared mem) + std::array, LEN> sorted_desc; + + void clear(); + void register_section(const string *section_name, size_t size); + }; + + HeaviestSections heaviest_sections_by_count; + // in the future, we may want distribution by estimate_memory_usage(): + // HeaviestSections heaviest_sections_by_mem; + + const memory_resource::MemoryStats &get_memory_stats() const noexcept; + void on_update(const confdata_sample_storage &new_confdata, size_t previous_garbage_size, const ConfdataPredefinedWildcards &predefined_wildcards) noexcept; - void write_stats_to(stats_t *stats, const memory_resource::MemoryStats &memory_stats) const noexcept; + void write_stats_to(stats_t *stats) const noexcept; private: ConfdataStats() = default; diff --git a/server/php-engine-vars.cpp b/server/php-engine-vars.cpp index e94d0b39fc..00e74249f9 100644 --- a/server/php-engine-vars.cpp +++ b/server/php-engine-vars.cpp @@ -22,6 +22,7 @@ double oom_handling_memory_ratio = 0.00; int worker_id = -1; int pid = -1; +int master_pid = -1; ProcessType process_type = ProcessType::master; diff --git a/server/php-engine-vars.h b/server/php-engine-vars.h index 37f33ae9ec..80a11a134b 100644 --- a/server/php-engine-vars.h +++ b/server/php-engine-vars.h @@ -32,6 +32,7 @@ extern double oom_handling_memory_ratio; extern int worker_id; extern int pid; +extern int master_pid; extern ProcessType process_type; diff --git a/server/php-engine.cpp b/server/php-engine.cpp index 3b50adf2a9..8e1ca596ae 100644 --- a/server/php-engine.cpp +++ b/server/php-engine.cpp @@ -1692,6 +1692,7 @@ void init_all() { auto end_time = std::chrono::steady_clock::now(); uint64_t total_init_ns = std::chrono::duration_cast(end_time - start_time).count(); StatsHouseManager::get().add_init_master_stats(total_init_ns, ConfdataStats::get().initial_loading_time.count()); + StatsHouseManager::get().add_confdata_master_stats(ConfdataStats::get()); } void init_logname(const char *src) { @@ -2206,6 +2207,26 @@ int main_args_handler(int i, const char *long_option) { case 2036: { return read_option_to(long_option, 0U, 2048U, thread_pool_size); } + case 2037: { + if (!*optarg) { + kprintf("--%s option is empty\n", long_option); + return -1; + } + add_confdata_force_ignore_prefix(optarg); + return 0; + } + case 2038: { + double timeout_sec; + int res = read_option_to(long_option, 0.0, std::numeric_limits::max(), timeout_sec); + set_confdata_update_timeout(timeout_sec); + return res; + } + case 2039: { + double soft_oom_ratio; + int res = read_option_to(long_option, 0.0, CONFDATA_DEFAULT_HARD_OOM_RATIO, soft_oom_ratio); + set_confdata_soft_oom_ratio(soft_oom_ratio); + return res; + } default: return -1; } @@ -2274,7 +2295,7 @@ void parse_main_args(int argc, char *argv[]) { parse_option("tasks-config", required_argument, 'S', "get lease worker settings from config file: mode and actor"); parse_option("confdata-binlog", required_argument, 2004, "confdata binlog mask"); parse_option("confdata-memory-limit", required_argument, 2005, "memory limit for confdata"); - parse_option("confdata-blacklist", required_argument, 2006, "confdata key blacklist regex pattern"); + parse_option("confdata-blacklist", required_argument, 2006, "confdata key blacklist regex pattern from PHP code, class KphpConfiguration"); parse_option("confdata-predefined-wildcard", required_argument, 2007, "perdefine confdata wildcard for better performance"); parse_option("php-version", no_argument, 2008, "show the compiled php code version and exit"); parse_option("php-warnings-minimal-verbosity", required_argument, 2009, "set minimum verbosity level for php warnings"); @@ -2315,6 +2336,12 @@ void parse_main_args(int argc, char *argv[]) { parse_option("hard-time-limit", required_argument, 2034, "time limit for script termination after the main timeout has expired (default: 1 sec). Use 0 to disable"); parse_option("thread-pool-ratio", required_argument, 2035, "the thread pool size ratio of the overall cpu numbers"); parse_option("thread-pool-size", required_argument, 2036, "the total threads num per worker"); + parse_option("confdata-force-ignore-keys-prefix", required_argument, 2037, "an emergency option, e.g. 'highload.vid*', to forcibly drop keys from snapshot/binlog; may be used multiple times"); + parse_option("confdata-update-timeout", required_argument, 2038, "cron confdata binlog replaying will be forcibly stopped after the specified timeout (default: 0.3 sec)" + "Initial binlog is readed with x10 times larger timeout"); + parse_option("confdata-soft-oom-ratio", required_argument, 2039, "Memory limit ratio to start ignoring new keys related events (default: 0.85)." + "Can't be > hard oom ratio (0.95)"); + parse_engine_options_long(argc, argv, main_args_handler); parse_main_args_till_option(argc, argv); // TODO: remove it after successful migration from kphb.readyV2 to kphb.readyV3 @@ -2333,6 +2360,7 @@ void init_default() { now = (int)time(nullptr); pid = getpid(); + master_pid = getpid(); // RPC part PID.port = (short)rpc_port; diff --git a/server/signal-handlers.cpp b/server/signal-handlers.cpp index 44ba812b51..701cd567ca 100644 --- a/server/signal-handlers.cpp +++ b/server/signal-handlers.cpp @@ -217,7 +217,7 @@ void sigabrt_handler(int, siginfo_t *info, void *) { print_http_data(); dl_print_backtrace(trace, trace_size); kill_workers(); - if (static_cast(info->si_value.sival_int) == ExtraSignalAction::GENERATE_COREDUMP) { + if (getpid() == master_pid || static_cast(info->si_value.sival_int) == ExtraSignalAction::GENERATE_COREDUMP) { raise(SIGQUIT); // hack for generate core dump } _exit(EXIT_FAILURE); diff --git a/server/statshouse/statshouse-manager.cpp b/server/statshouse/statshouse-manager.cpp index bbd4100493..8e4f36f2bd 100644 --- a/server/statshouse/statshouse-manager.cpp +++ b/server/statshouse/statshouse-manager.cpp @@ -10,6 +10,7 @@ #include "common/resolver.h" #include "runtime/instance-cache.h" #include "server/job-workers/shared-memory-manager.h" +#include "server/confdata-stats.h" #include "server/json-logger.h" #include "server/php-runner.h" #include "server/server-config.h" @@ -171,9 +172,10 @@ void StatsHouseManager::add_worker_memory_stats(const mem_info_t &mem_stats) { client.metric("kphp_by_host_workers_memory", true).tag(worker_type).tag("rss_peak").write_value(mem_stats.rss_peak); } -void StatsHouseManager::add_common_master_stats(const workers_stats_t &workers_stats, const memory_resource::MemoryStats &memory_stats, double cpu_s_usage, - double cpu_u_usage, long long int instance_cache_memory_swaps_ok, - long long int instance_cache_memory_swaps_fail) { +void StatsHouseManager::add_common_master_stats(const workers_stats_t &workers_stats, + const memory_resource::MemoryStats &instance_cache_memory_stats, + double cpu_s_usage, double cpu_u_usage, + long long int instance_cache_memory_swaps_ok, long long int instance_cache_memory_swaps_fail) { if (engine_tag) { client.metric("kphp_version").write_value(atoll(engine_tag)); } @@ -208,14 +210,14 @@ void StatsHouseManager::add_common_master_stats(const workers_stats_t &workers_s client.metric("kphp_server_total_json_logs_count").write_value(std::get<0>(total_workers_json_count) + master_json_logs_count); client.metric("kphp_server_total_json_traces_count").write_value(std::get<1>(total_workers_json_count)); - client.metric("kphp_instance_cache_memory").tag("limit").write_value(memory_stats.memory_limit); - client.metric("kphp_instance_cache_memory").tag("used").write_value(memory_stats.memory_used); - client.metric("kphp_instance_cache_memory").tag("real_used").write_value(memory_stats.real_memory_used); + client.metric("kphp_instance_cache_memory").tag("limit").write_value(instance_cache_memory_stats.memory_limit); + client.metric("kphp_instance_cache_memory").tag("used").write_value(instance_cache_memory_stats.memory_used); + client.metric("kphp_instance_cache_memory").tag("real_used").write_value(instance_cache_memory_stats.real_memory_used); - client.metric("kphp_instance_cache_memory_defragmentation_calls").write_value(memory_stats.defragmentation_calls); + client.metric("kphp_instance_cache_memory_defragmentation_calls").write_value(instance_cache_memory_stats.defragmentation_calls); - client.metric("kphp_instance_cache_memory_pieces").tag("huge").write_value(memory_stats.huge_memory_pieces); - client.metric("kphp_instance_cache_memory_pieces").tag("small").write_value(memory_stats.small_memory_pieces); + client.metric("kphp_instance_cache_memory_pieces").tag("huge").write_value(instance_cache_memory_stats.huge_memory_pieces); + client.metric("kphp_instance_cache_memory_pieces").tag("small").write_value(instance_cache_memory_stats.small_memory_pieces); client.metric("kphp_instance_cache_memory_buffer_swaps").tag("ok").write_value(instance_cache_memory_swaps_ok); client.metric("kphp_instance_cache_memory_buffer_swaps").tag("fail").write_value(instance_cache_memory_swaps_fail); @@ -302,3 +304,26 @@ size_t StatsHouseManager::add_job_workers_shared_memory_buffers_stats(const job_ return memory_used; } + +void StatsHouseManager::add_confdata_master_stats(const ConfdataStats &confdata_stats) { + const auto &memory_stats = confdata_stats.get_memory_stats(); + client.metric("kphp_confdata_memory").tag("limit").write_value(memory_stats.memory_limit); + client.metric("kphp_confdata_memory").tag("used").write_value(memory_stats.memory_used); + client.metric("kphp_confdata_memory").tag("real_used").write_value(memory_stats.real_memory_used); + + const auto &events = confdata_stats.event_counters; + client.metric("kphp_confdata_events").tag("set").write_value(events.set_events.total + events.set_forever_events.total); + client.metric("kphp_confdata_events").tag("set_blacklisted").write_value(events.set_events.blacklisted + events.set_forever_events.blacklisted); + client.metric("kphp_confdata_events").tag("delete").write_value(events.delete_events.total); + client.metric("kphp_confdata_events").tag("delete_blacklisted").write_value(events.delete_events.blacklisted); + client.metric("kphp_confdata_events").tag("throttled_out").write_value(events.throttled_out_total_events); + + client.metric("kphp_confdata_update_fails").tag("ignored").write_value(confdata_stats.ignored_updates); + client.metric("kphp_confdata_update_fails").tag("timed_out").write_value(confdata_stats.timed_out_updates); + + for (const auto &[section_name, size] : confdata_stats.heaviest_sections_by_count.sorted_desc) { + if (section_name != nullptr && size > 0) { // section_name looks like "highload." + client.metric("kphp_confdata_sections_by_count").tag(section_name->c_str()).write_value(size); + } + } +} diff --git a/server/statshouse/statshouse-manager.h b/server/statshouse/statshouse-manager.h index e663936f6b..f848f631e3 100644 --- a/server/statshouse/statshouse-manager.h +++ b/server/statshouse/statshouse-manager.h @@ -19,6 +19,8 @@ using normalization_function = std::function; enum class script_error_t : uint8_t; +struct ConfdataStats; + class StatsHouseManager : vk::not_copyable { public: static void init(const std::string &ip, int port) { @@ -69,7 +71,9 @@ class StatsHouseManager : vk::not_copyable { /** * Must be called from master process only */ - void add_common_master_stats(const workers_stats_t &workers_stats, const memory_resource::MemoryStats &memory_stats, double cpu_s_usage, double cpu_u_usage, + void add_common_master_stats(const workers_stats_t &workers_stats, + const memory_resource::MemoryStats &instance_cache_memory_stats, + double cpu_s_usage, double cpu_u_usage, long long int instance_cache_memory_swaps_ok, long long int instance_cache_memory_swaps_fail); /** @@ -82,6 +86,8 @@ class StatsHouseManager : vk::not_copyable { */ void add_extended_instance_cache_stats(std::string_view type, std::string_view status, const string &key, uint64_t size = 0); + void add_confdata_master_stats(const ConfdataStats &confdata_stats); + private: StatsHouseClient client; bool need_write_enable_tag_host = false; diff --git a/tests/cpp/runtime/confdata-functions-test.cpp b/tests/cpp/runtime/confdata-functions-test.cpp index 726b44d703..4b905c827d 100644 --- a/tests/cpp/runtime/confdata-functions-test.cpp +++ b/tests/cpp/runtime/confdata-functions-test.cpp @@ -13,8 +13,9 @@ void init_global_confdata_confdata() { initiated = true; pid = 0; + std::forward_list force_ignore_prefixes; auto &global_manager = ConfdataGlobalManager::get(); - global_manager.init(1024 * 1024 * 16, std::unordered_set{}, nullptr); + global_manager.init(1024 * 1024 * 16, std::unordered_set{}, nullptr, std::move(force_ignore_prefixes)); auto confdata_sample_storage = global_manager.get_current().get_confdata(); confdata_sample_storage[string{"_key_1"}] = string{"value_1"}; diff --git a/tests/python/lib/engine.py b/tests/python/lib/engine.py index d5586d047b..be9c996087 100644 --- a/tests/python/lib/engine.py +++ b/tests/python/lib/engine.py @@ -115,11 +115,15 @@ def start(self, start_msgs=None): self._stats_receiver.start() cmd = [self._engine_bin] - for option, value in self._options.items(): - if value is not None: - cmd.append(option) - if value is not True: - cmd.append(str(value)) + for option, value_raw in self._options.items(): + if value_raw is not None: + value_list = value_raw + if type(value_raw) is not list: + value_list = [value_raw] + for val in value_list: + cmd.append(option) + if val is not True: + cmd.append(str(val)) if self._binlog_path: cmd.append(self._binlog_path) diff --git a/tests/python/lib/file_utils.py b/tests/python/lib/file_utils.py index 263385aa7a..b8ef2c7d1c 100644 --- a/tests/python/lib/file_utils.py +++ b/tests/python/lib/file_utils.py @@ -34,11 +34,11 @@ def search_kphp2cpp(): def search_engine_bin(engine_name): - return _check_file("bin/" + engine_name, _ENGINE_INSTALL_PATH, _check_bin) + return _check_file("bin/" + engine_name, _ENGINE_INSTALL_PATH, _check_bin) if _ENGINE_INSTALL_PATH else engine_name def search_tl_client(): - return _check_file("bin/tlclient", _ENGINE_INSTALL_PATH, _check_bin) + return _check_file("bin/tlclient", _ENGINE_INSTALL_PATH, _check_bin) if _ENGINE_INSTALL_PATH else "tlclient" def search_combined_tlo(working_dir): diff --git a/tests/python/tests/json_logs/test_signals.py b/tests/python/tests/json_logs/test_signals.py index 2ab3081fc0..2e421461d3 100644 --- a/tests/python/tests/json_logs/test_signals.py +++ b/tests/python/tests/json_logs/test_signals.py @@ -90,7 +90,9 @@ def test_master_sigbus(self): def test_master_sigabrt(self): self.kphp_server.send_signal(signal.SIGABRT) - self.kphp_server.restart() + with self.assertRaises(RuntimeError): + self.kphp_server.stop() + self.kphp_server.start() self.kphp_server.assert_json_log( expect=[{ "version": 0, "type": -1, "env": "", "msg": "SIGABRT terminating program",