Fix ubuntu 24.04 noble build (#1224)

* Fix rules generator
* Get rid of thread pool

builtin-functions/kphp-full/_functions.txt

@@ -679,7 +679,6 @@ function wordwrap ($str ::: string, $width ::: int = 75, $break ::: string = '\n
 function ip2long ($ip ::: string) ::: int | false;
 function ip2ulong ($ip ::: string) ::: string | false;
 function long2ip ($ip ::: int) ::: string;
-function thread_pool_test_load($size ::: int, $n ::: int, $a ::: float, $b ::: float) ::: float;
 
 function get_magic_quotes_gpc() ::: bool;
 function php_sapi_name() ::: string;

common/dl-utils-lite.cpp

@@ -21,7 +21,6 @@
 #include <sys/resource.h>
 #include <time.h>
 #include <unistd.h>
-#include <dirent.h>
 
 #include "common/server/crash-dump.h"
 #include "common/stats/provider.h"
@@ -276,24 +275,6 @@ mem_info_t get_self_mem_stats() {
   return info;
 }
 
-int get_self_threads_count() {
-  DIR * tasks = opendir("/proc/self/task");
-  if (tasks == nullptr) {
-    return -1;
-  }
-  int count = 0;
-  struct dirent * task = readdir(tasks);
-  while (task != nullptr && count < 1000) {
-    count++;
-    task = readdir(tasks);
-  }
-  closedir(tasks);
-  /**
-   * subtract from the result ".", "..", "self"
-   * */
-  return count - 3;
-}
-
 int get_pid_info (pid_t pid, pid_info_t *info) {
   constexpr size_t TMEM_SIZE = 10000;
   static char mem[TMEM_SIZE];

common/dl-utils-lite.h

@@ -73,7 +73,6 @@ struct process_rusage_t {
 };
 
 mem_info_t get_self_mem_stats();
-int get_self_threads_count();
 int get_pid_info (pid_t pid, pid_info_t *info);
 unsigned long long get_pid_start_time (pid_t pid);
 int get_cpu_total (unsigned long long *cpu_total);

common/stats/provider.cpp

@@ -10,7 +10,6 @@
 #include <cstdlib>
 #include <fcntl.h>
 #include <sys/resource.h>
-#include <thread>
 #include <unistd.h>
 #include <vector>
 
@@ -274,7 +273,6 @@ static void resource_usage_statistics(stats_t *stats, const char *prefix,
   stats->add_general_stat(stat_temp_format("%sblock_input_operations", prefix), "%ld", usage->ru_inblock);
   stats->add_general_stat(stat_temp_format("%sblock_output_operations", prefix), "%ld", usage->ru_oublock);
 
-  stats->add_general_stat(stat_temp_format("%shardware_concurrency", prefix), "%u", std::thread::hardware_concurrency());
   stats->add_histogram_stat(stat_temp_format("%svoluntary_context_switches", prefix), usage->ru_nvcsw);
   stats->add_histogram_stat(stat_temp_format("%sivoluntary_context_switches", prefix), usage->ru_nivcsw);
 }

compiler/pipes/register-kphp-configuration.cpp

@@ -141,8 +141,7 @@ void RegisterKphpConfiguration::handle_constant_runtime_options(const ClassMembe
       register_net_dc_mask(opt_pair->value());
     } else if (vk::any_of_equal(*opt_key,
                                 warmup_workers_part_key_, warmup_instance_cache_elements_part_key_, warmup_timeout_sec_key_,
-                                oom_handling_memory_ratio_key_, mysql_db_name_key_, job_workers_shared_memory_distribution_weights_,
-                                thread_pool_ratio_key_, thread_pool_size_key_)) {
+                                oom_handling_memory_ratio_key_, mysql_db_name_key_, job_workers_shared_memory_distribution_weights_)) {
       generic_register_simple_option(opt_pair->value(), *opt_key);
     } else {
       kphp_error(0, fmt_format("Got unexpected option {}::{}['{}']",

compiler/pipes/register-kphp-configuration.h

@@ -47,8 +47,6 @@ class RegisterKphpConfiguration final : public SyncPipeF<FunctionPtr> {
 
   const vk::string_view job_workers_shared_memory_distribution_weights_{"--job-workers-shared-memory-distribution-weights"};
 
-  const vk::string_view thread_pool_ratio_key_{"--thread-pool-ratio"};
-  const vk::string_view thread_pool_size_key_{"--thread-pool-size"};
 public:
   void execute(FunctionPtr function, DataStream<FunctionPtr> &unused_os) final;
   void on_finish(DataStream<FunctionPtr> &os) override;

compiler/rewrite-rules/impl/rules_generator.py

@@ -245,7 +245,7 @@ def print_matcher_cond(self, e: Expr, name: str, printer: CodePrinter, vars_decl
         if e.vertex_string:
             printer.write_line(f"if ({name}->get_string() != std::string_view({e.vertex_string})) {{ break; }}", e.line)
         if named_range:
-            printer.write_line(f"const auto &{named_range[0]} = {name}->{named_range[1]}();", e.line)
+            printer.write_line(f"const auto {named_range[0]} = {name}->{named_range[1]}();", e.line)
             return
         for i, subexpr in enumerate(e.members):
             arg_info, range_offset = vertex_info.get_arg(i, len(e.members))
@@ -261,9 +261,9 @@ def print_matcher_cond(self, e: Expr, name: str, printer: CodePrinter, vars_decl
             else:
                 if subexpr.op == Expr.OP_ANY:
                     if new_name != "_":
-                        printer.write_line(f"const auto &{new_name} = {arg_expr};", subexpr.line)
+                        printer.write_line(f"const auto {new_name} = {arg_expr};", subexpr.line)
                 else:
-                    printer.write_line(f"const auto &{new_name} = vertex_cast<{subexpr.op}>({arg_expr});", subexpr.line)
+                    printer.write_line(f"const auto {new_name} = vertex_cast<{subexpr.op}>({arg_expr});", subexpr.line)
                     printer.write_line(f"if (!{new_name}) {{ break; }}", subexpr.line)
                 if subexpr.name and subexpr.name != "_":
                     vars_declared.add(subexpr.name)

runtime/interface.cpp

@@ -56,7 +56,6 @@
 #include "runtime/rpc.h"
 #include "runtime/streams.h"
 #include "runtime/tcp.h"
-#include "runtime/thread-pool.h"
 #include "runtime/typed_rpc.h"
 #include "runtime/udp.h"
 #include "runtime/url.h"
@@ -766,27 +765,7 @@ void f$die(const mixed &v) {
   f$exit(v);
 }
 
-double f$thread_pool_test_load(int64_t size, int64_t n, double a, double b) {
-  constexpr auto job = [](int64_t n, double a, double b) {
-    double res = 0;
-    for (int i = 0; i < n; ++i) {
-      res += (i * a + 1) / (i * b + 1);
-    }
-    return res;
-  };
-  auto & pool = vk::singleton<ThreadPool>::get().pool();
-  double result = 0;
-  {
-    dl::CriticalSectionGuard guard;
-    BS::multi_future<double> futures;
-    for (int thread = 0; thread < size; ++thread) {
-      futures.push_back(pool.submit(job, n, a, b));
-    }
-    auto results = futures.get();
-    std::for_each(results.begin(), results.end(), [&](int64_t local){result += local;});
-  }
-  return result;
-}
+
 
 Optional<array<string>> f$gethostbynamel(const string &name) {
   dl::enter_critical_section();//OK
@@ -2465,7 +2444,6 @@ void worker_global_init(WorkerType worker_type) noexcept {
   worker_global_init_slot_factories();
   vk::singleton<JsonLogger>::get().reset_json_logs_count();
   worker_global_init_handlers(worker_type);
-  vk::singleton<ThreadPool>::get().init();
   init_kphp_ml_runtime_in_worker();
   init_php_scripts_in_each_worker(PhpScriptMutableGlobals::current());
 }

runtime/interface.h

@@ -105,13 +105,10 @@ void f$exit(const mixed &v = 0);
 __attribute__((noreturn))
 void f$die(const mixed &v = 0);
 
-double f$thread_pool_test_load(int64_t size, int64_t n, double a, double b);
-
 Optional<array<string>> f$gethostbynamel(const string &name);
 
 Optional<string> f$inet_pton(const string &address);
 
-
 void print(const char *s, size_t s_len);
 
 void print(const char *s);

runtime/runtime.cmake

@@ -111,7 +111,6 @@ prepend(KPHP_RUNTIME_SOURCES ${BASE_DIR}/runtime/
         uber-h3.cpp
         udp.cpp
         tcp.cpp
-        thread-pool.cpp
         url.cpp
         vkext.cpp
         vkext_stats.cpp

server/php-engine.cpp

@@ -64,7 +64,7 @@
 #include "runtime/kphp_ml/kphp_ml_init.h"
 #include "runtime/profiler.h"
 #include "runtime/rpc.h"
-#include "runtime/thread-pool.h"
+#include "server/server-config.h"
 #include "server/confdata-binlog-replay.h"
 #include "server/confdata-stats.h"
 #include "server/database-drivers/adaptor.h"
@@ -88,7 +88,6 @@
 #include "server/php-runner.h"
 #include "server/php-sql-connections.h"
 #include "server/php-worker.h"
-#include "server/server-config.h"
 #include "server/server-log.h"
 #include "server/server-stats.h"
 #include "server/shared-data-worker-cache.h"
@@ -2196,15 +2195,6 @@ int main_args_handler(int i, const char *long_option) {
     case 2034: {
       return read_option_to(long_option, 0.0, 5.0, hard_timeout);
     }
-    case 2035: {
-      double thread_pool_ratio = 0.0;
-      int res = read_option_to(long_option, 0.0, 10.0, thread_pool_ratio);
-      thread_pool_size = static_cast<int>(std::ceil(std::thread::hardware_concurrency() * thread_pool_ratio));
-      return res;
-    }
-    case 2036: {
-      return read_option_to(long_option, 0U, 2048U, thread_pool_size);
-    }
     case 2037: {
       if (!*optarg) {
         kprintf("--%s option is empty\n", long_option);
@@ -2351,15 +2341,14 @@ void parse_main_args(int argc, char *argv[]) {
   parse_option("runtime-config", required_argument, 2032, "JSON file path that will be available at runtime as 'mixed' via 'kphp_runtime_config()");
   parse_option("oom-handling-memory-ratio", required_argument, 2033, "memory ratio of overall script memory to handle OOM errors (default: 0.00)");
   parse_option("hard-time-limit", required_argument, 2034, "time limit for script termination after the main timeout has expired (default: 1 sec). Use 0 to disable");
-  parse_option("thread-pool-ratio", required_argument, 2035, "the thread pool size ratio of the overall cpu numbers");
-  parse_option("thread-pool-size", required_argument, 2036, "the total threads num per worker");
   parse_option("confdata-force-ignore-keys-prefix", required_argument, 2037, "an emergency option, e.g. 'highload.vid*', to forcibly drop keys from snapshot/binlog; may be used multiple times");
   parse_option("confdata-update-timeout", required_argument, 2038, "cron confdata binlog replaying will be forcibly stopped after the specified timeout (default: 0.3 sec)"
                                                                    "Initial binlog is readed with x10 times larger timeout");
   parse_option("confdata-soft-oom-ratio", required_argument, 2039, "Memory limit ratio to start ignoring new keys related events (default: 0.85)."
                                                                    "Can't be > hard oom ratio (0.95)");
   parse_option("kml-dir", required_argument, 2040, "Directory that contains .kml files");
 
+
   parse_engine_options_long(argc, argv, main_args_handler);
   parse_main_args_till_option(argc, argv);
   // TODO: remove it after successful migration from kphb.readyV2 to kphb.readyV3

server/php-master.cpp

@@ -53,7 +53,6 @@
 #include "runtime/memory_resource_impl/memory_resource_stats.h"
 #include "runtime/confdata-global-manager.h"
 #include "runtime/instance-cache.h"
-#include "runtime/thread-pool.h"
 #include "server/confdata-binlog-replay.h"
 #include "server/http-server-context.h"
 #include "server/lease-rpc-client.h"
@@ -1141,9 +1140,6 @@ STATS_PROVIDER_TAGGED(kphp_stats, 100, stats_tag_kphp_server) {
   stats->add_gauge_stat("cpu.stime", cpu_stats.cpu_s_usage);
   stats->add_gauge_stat("cpu.utime", cpu_stats.cpu_u_usage);
 
-  uint64_t threads_count_stat = vk::singleton<ServerStats>::get().collect_threads_count_stat();
-  stats->add_gauge_stat("server.total_threads_count", threads_count_stat);
-
   auto total_workers_json_count = vk::singleton<ServerStats>::get().collect_json_count_stat();
   uint64_t master_json_logs_count = vk::singleton<JsonLogger>::get().get_json_logs_count();
   stats->add_gauge_stat("server.total_json_logs_count", std::get<0>(total_workers_json_count) + master_json_logs_count);

server/server-stats.cpp

@@ -137,7 +137,6 @@ struct MiscStat : WithStatType<uint64_t> {
     worker_status,
     json_logs_count,
     json_traces_count,
-    threads_count,
     worker_activity_counter,
     types_count
   };
@@ -507,7 +506,6 @@ struct WorkerProcessStats : private vk::not_copyable {
     misc_stats.inc_stat(MiscStat::Key::worker_activity_counter, worker_index);
     misc_stats.set_stat(MiscStat::Key::json_logs_count, worker_index, vk::singleton<JsonLogger>::get().get_json_logs_count());
     misc_stats.set_stat(MiscStat::Key::json_traces_count, worker_index, vk::singleton<JsonLogger>::get().get_json_traces_count());
-    misc_stats.set_stat(MiscStat::Key::threads_count, worker_index, get_self_threads_count());
   }
 
   void update_worker_special_connections(uint64_t active_connections, uint64_t max_connections, uint16_t worker_index) noexcept {
@@ -530,7 +528,6 @@ struct WorkerProcessStats : private vk::not_copyable {
     misc_stats.set_stat(MiscStat::Key::worker_status, worker_index, MiscStat::worker_idle);
     misc_stats.set_stat(MiscStat::Key::json_logs_count, worker_index, 0);
     misc_stats.set_stat(MiscStat::Key::json_traces_count, worker_index, 0);
-    misc_stats.set_stat(MiscStat::Key::threads_count, worker_index, get_self_threads_count());
     update_worker_special_connections(active_connections, max_connections, worker_index);
     update_worker_stats(worker_index);
   }
@@ -960,16 +957,6 @@ std::tuple<uint64_t, uint64_t> ServerStats::collect_json_count_stat() const noex
   return {sum_json_logs_count, sum_json_traces_count};
 }
 
-uint64_t ServerStats::collect_threads_count_stat() const noexcept {
-  const auto &workers_misc = shared_stats_->workers.misc_stats;
-  uint64_t sum_threads_count = 0;
-  const auto &workers_control = vk::singleton<WorkersControl>::get();
-  for (uint16_t w = 0; w != workers_control.get_total_workers_count(); ++w) {
-    sum_threads_count += workers_misc.get_stat(MiscStat::Key::threads_count, w);
-  }
-  return sum_threads_count;
-}
-
 uint64_t ServerStats::get_worker_activity_counter(uint16_t worker_process_id) const noexcept {
   return shared_stats_->workers.misc_stats.get_stat(MiscStat::Key::worker_activity_counter, worker_process_id);
 }

server/server-stats.h

@@ -53,7 +53,6 @@ class ServerStats : vk::not_copyable {
   };
   WorkersStat collect_workers_stat(WorkerType worker_type) const noexcept;
   std::tuple<uint64_t, uint64_t> collect_json_count_stat() const noexcept;
-  uint64_t collect_threads_count_stat() const noexcept;
 
 private:
   friend class vk::singleton<ServerStats>;