Merge branch 'master' into andreylzmw-gh-install

.github/workflows/Build.yml

@@ -128,7 +128,7 @@ jobs:
 
     - name: Run php dummy script
       run: docker exec kphp-build-container-${{matrix.os}} bash -c
-             'cd ${{env.kphp_build_dir}} && su kitten -c "kphp --version && ./kphp_out/cli -o --user kitten'
+            "сd ${{env.kphp_build_dir}} && echo 'hello world' > test.php && kphp --mode=cli test.php && ./kphp_out/cli"
 
     - name: Remove docker container
       run: docker rm -f kphp-build-container-${{matrix.os}}
@@ -197,6 +197,26 @@ jobs:
 
   #   - name: Build all
   #     run: make -C ${{runner.workspace}}/build -j$(nproc) all
+
+#     # because of https://github.com/orgs/Homebrew/discussions/4612
+#     - name: Check Environment
+#       run: |
+#         export HOMEBREW_NO_INSTALL_FROM_API=0
+#         brew untap --force homebrew/cask
+
+#     - name: Setup Environment
+#       run: |
+#         brew tap shivammathur/php
+#         brew update
+#         brew install re2c cmake coreutils openssl libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/php@7.4 python@3.12
+#         brew link --overwrite --force shivammathur/php/php@7.4
+#         /usr/local/Frameworks/Python.framework/Versions/3.12/bin/python3.12 -m pip install --upgrade pip --break-system-packages && /usr/local/Frameworks/Python.framework/Versions/3.12/bin/pip3 install jsonschema install --break-system-packages jsonschema
+
+#     - name: Run cmake
+#       run: cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DDOWNLOAD_MISSING_LIBRARIES=On -S $GITHUB_WORKSPACE -B ${{runner.workspace}}/build
+
+#     - name: Build all
+#       run: make -C ${{runner.workspace}}/build -j$(nproc) all
 
   #   - name: Run unit tests
   #     run: make -C ${{runner.workspace}}/build -j$(nproc) test

.github/workflows/macos.yml

@@ -22,25 +22,21 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-
-    - name: Get polyfills repo
-      uses: actions/checkout@v3
-      with:
-        repository: 'uber/h3'
-        path: 'h3'
-        ref: stable-3.x
+
+      # because of https://github.com/orgs/Homebrew/discussions/4612
+    - name: Check Environment
+      run: |
+        export HOMEBREW_NO_INSTALL_FROM_API=0
+        brew untap --force homebrew/cask
 
     - name: Setup Environment
       run: |
         brew tap shivammathur/php
         brew update
-        brew install re2c cmake coreutils openssl libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/php@7.4
+        brew install re2c cmake coreutils openssl libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/php@7.4 python@3.12
         brew link --overwrite --force shivammathur/php/php@7.4
-        /usr/local/Frameworks/Python.framework/Versions/3.11/bin/pip3 install jsonschema
-
-    - name: Build Uber h3
-      run: cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -S $GITHUB_WORKSPACE/h3 -B ${{runner.workspace}}/h3/build && make install -C ${{runner.workspace}}/h3/build -j$(nproc) all
-
+        /usr/local/Frameworks/Python.framework/Versions/3.12/bin/python3.12 -m pip install --upgrade pip --break-system-packages && /usr/local/Frameworks/Python.framework/Versions/3.12/bin/pip3 install jsonschema install --break-system-packages jsonschema
+      
     - name: Run cmake
       run: cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DDOWNLOAD_MISSING_LIBRARIES=On -S $GITHUB_WORKSPACE -B ${{runner.workspace}}/build
 

builtin-functions/_functions.txt

@@ -14,6 +14,7 @@ if (0) {
 }
 
 require_once __DIR__ . '/kphp_internal.txt';
+require_once __DIR__ . '/kphp_toggles.txt';
 require_once __DIR__ . '/kphp_tracing.txt';
 require_once __DIR__ . '/uberh3.txt';
 require_once __DIR__ . '/spl.txt';
@@ -497,6 +498,7 @@ function openssl_sign ($data ::: string, &$signature ::: string, $priv_key_id ::
 function openssl_verify ($data ::: string, $signature ::: string, $pub_key_id ::: string, $signature_alg ::: int = 1) ::: int;
 function openssl_random_pseudo_bytes ($length ::: int) ::: string | false;
 function openssl_x509_parse ($x509cert ::: string, $shortnames ::: bool = true) ::: mixed[] | false;
+function openssl_x509_verify ($x509cert ::: string, $public_key ::: string) ::: int;
 function openssl_x509_checkpurpose ($x509cert ::: string, $purpose ::: int) ::: mixed;
 
 define('PKCS7_TEXT', 0x1);
@@ -1428,16 +1430,6 @@ function zstd_uncompress(string $data) ::: string | false;
 function zstd_compress_dict(string $data, string $dict) ::: string | false;
 function zstd_uncompress_dict(string $data, string $dict) ::: string | false;
 
-function set_migration_php8_warning ($mask ::: int) ::: void;
-
-function set_detect_incorrect_encoding_names_warning(bool $show) ::: void;
-
-function set_json_log_on_timeout_mode(bool $enabled) ::: void;
-
-function set_json_log_demangle_stacktrace(bool $enable) ::: void;
-
-function set_use_updated_gmmktime(bool $enable) ::: void;
-
 // re-initialize given ArrayIterator with another array;
 // in KPHP it returns the same ArrayIterator that is ready to be used
 // in PHP (via polyfills) it returns a newly allocated object

builtin-functions/kphp_toggles.txt

@@ -0,0 +1,17 @@
+<?php
+
+// Functions declared in this file are actually toggles to enable or disable some functionality in KPHP runtime.
+
+function set_migration_php8_warning ($mask ::: int) ::: void;
+
+function set_detect_incorrect_encoding_names_warning(bool $show) ::: void;
+
+function set_json_log_on_timeout_mode(bool $enabled) ::: void;
+
+function set_json_log_demangle_stacktrace(bool $enable) ::: void;
+
+function set_use_updated_gmmktime(bool $enable) ::: void;
+
+function kphp_turn_on_host_tag_in_inner_statshouse_metrics_toggle();
+
+function kphp_extended_instance_cache_metrics_init(callable(string $key):string $normalization_function) ::: void;

cmake/external-libraries.cmake

@@ -54,6 +54,23 @@ else()
     add_link_options(-L${kphp-timelib_SOURCE_DIR}/objs)
 endif()
 
+find_library(KPHP_H3 h3)
+if(KPHP_H3)
+    add_library(h3 STATIC IMPORTED ${KPHP_H3})
+else()
+    handle_missing_library("h3")
+    FetchContent_Declare(
+            h3
+            GIT_REPOSITORY https://github.com/VKCOM/uber-h3.git
+            GIT_TAG        fetch_content
+    )
+    message(STATUS "---------------------")
+    FetchContent_MakeAvailable(h3)
+    include_directories(${h3_BINARY_DIR}/src/include)
+    add_definitions(-DKPHP_H3_LIB_DIR="${h3_BINARY_DIR}/lib")
+    add_link_options(-L${h3_BINARY_DIR}/lib)
+endif()
+
 if(APPLE)
     if (DEFINED ENV{EPOLL_SHIM_REPO})
         FetchContent_Declare(

common/common.cmake

@@ -35,9 +35,9 @@ prepend(COMMON_TL_METHODS_SOURCES ${COMMON_DIR}/tl/methods/
         rwm.cpp
         string.cpp)
 
-if (APPLE AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64")
+if (NOT CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
     prepend(COMMON_UCONTEXT_SOURCES ${COMMON_DIR}/ucontext/
-            ucontext.cpp)
+            ucontext-arm.cpp)
 endif()
 
 set(COMMON_ALL_SOURCES

common/dl-utils-lite.cpp

@@ -18,10 +18,12 @@
 #include <string.h>
 #include <sys/wait.h>
 #include <sys/time.h>
+#include <sys/resource.h>
 #include <time.h>
 #include <unistd.h>
 #include <dirent.h>
 
+#include "common/server/crash-dump.h"
 #include "common/stats/provider.h"
 #include "common/wrappers/pathname.h"
 
@@ -91,18 +93,26 @@ const char *dl_get_assert_message() noexcept {
 
 // This function redefines glibc __assert_fail, so we can track assert messages
 void __assert_fail(const char *assertion, const char *file, unsigned int line, const char *function) {
-  dl_assert__(assertion, file, function, line, assertion, false);
+  dl_assert__(assertion, file, function, line, assertion, false, false);
   __builtin_unreachable();
 }
 
-void dl_assert__(const char *expr __attribute__((unused)), const char *file_name, const char *func_name,
-                 int line, const char *desc, int use_perror) {
+void dl_assert__([[maybe_unused]] const char *expr, const char *file_name, const char *func_name,
+                 int line, const char *desc, int use_perror, [[maybe_unused]] int generate_coredump) {
   snprintf(assert_message.data(), assert_message.size(),
            "dl_assert failed [%s:%d: %s]: %s%s%s", kbasename(file_name), line, func_name, desc,
            use_perror ? "; errno message = " : "",
            use_perror ? strerror(errno) : "");
   fprintf(stderr, "%s\n", assert_message.data());
+#if _POSIX_C_SOURCE >= 199309L
+  sigval value{0};
+  if (generate_coredump) {
+    value.sival_int = static_cast<int>(ExtraSignalAction::GENERATE_COREDUMP);
+  }
+  sigqueue(getpid(), SIGABRT, value);
+#else
   abort();
+#endif
 }
 
 static sigset_t old_mask;
@@ -146,6 +156,15 @@ void dl_restore_signal_mask () {
 void dl_block_all_signals () {
   sigset_t mask;
   sigfillset (&mask);
+  // According linux man sigprocmask page
+  // if SIGBUS, SIGFPE, SIGILL, or SIGSEGV are generated while they are blocked, the result is undefined,
+  // unless the signal was generated by kill(2), sigqueue(3), or raise(3).
+  sigdelset(&mask, SIGSEGV);
+  sigdelset(&mask, SIGBUS);
+  sigdelset(&mask, SIGFPE);
+  sigdelset(&mask, SIGILL);
+  // Allow SIGQUIT to generate coredump in signal handlers
+  sigdelset(&mask, SIGQUIT);
   int err = sigprocmask (SIG_SETMASK, &mask, &old_mask);
   old_mask_inited = 1;
   dl_passert (err != -1, "failed to block all signals");
@@ -159,16 +178,21 @@ void dl_allow_all_signals () {
   dl_passert (err != -1, "failed to allow all signals");
 }
 
-static void runtime_handler (const int sig) {
-  fprintf (stderr, "%s caught, terminating program\n", sig == SIGSEGV ? "SIGSEGV" : "SIGABRT");
+static void runtime_handler (const int sig, siginfo_t *info __attribute__((unused)), void *ucontext) {
+  fprintf (stderr, "%s caught, terminating program\n", strsignal(sig));
+  crash_dump_write(static_cast<ucontext_t *>(ucontext));
   dl_print_backtrace();
   dl_print_backtrace_gdb();
+  raise(SIGQUIT);
   _exit (EXIT_FAILURE);
 }
 
 void dl_set_default_handlers () {
-  dl_signal (SIGSEGV, runtime_handler);
-  dl_signal (SIGABRT, runtime_handler);
+  dl_sigaction(SIGSEGV, nullptr, dl_get_empty_sigset(), SA_SIGINFO | SA_ONSTACK | SA_RESTART, runtime_handler);
+  dl_sigaction(SIGBUS, nullptr, dl_get_empty_sigset(), SA_SIGINFO | SA_ONSTACK | SA_RESTART, runtime_handler);
+  dl_sigaction(SIGFPE, nullptr, dl_get_empty_sigset(), SA_SIGINFO | SA_ONSTACK | SA_RESTART, runtime_handler);
+  dl_sigaction(SIGILL, nullptr, dl_get_empty_sigset(), SA_SIGINFO | SA_ONSTACK | SA_RESTART, runtime_handler);
+  dl_sigaction(SIGABRT, nullptr, dl_get_empty_sigset(), SA_SIGINFO | SA_ONSTACK | SA_RESTART, runtime_handler);
 }
 
 char *dl_pstr (char const *msg, ...) {
@@ -360,3 +384,15 @@ int get_cpu_total (unsigned long long *cpu_total) {
   close (fd);
   return 1;
 }
+
+process_rusage_t get_rusage_info() {
+  rusage usage;
+  memset(&usage, 0, sizeof(rusage));
+  if (getrusage(RUSAGE_SELF, &usage) != 0) {
+    fprintf(stderr, "error while colling getrusage %s\n", strerror(errno));
+  }
+  double user_time = usage.ru_utime.tv_sec + (usage.ru_utime.tv_usec / 1E6);
+  double system_time = usage.ru_stime.tv_sec + (usage.ru_stime.tv_usec / 1E6);
+
+  return {user_time, system_time, usage.ru_nvcsw, usage.ru_nivcsw};
+}

common/dl-utils-lite.h

@@ -11,6 +11,10 @@
 #include "common/cycleclock.h"
 #include "common/wrappers/likely.h"
 
+enum class ExtraSignalAction {
+  GENERATE_COREDUMP = 1
+};
+
 double dl_time ();
 
 sigset_t dl_get_empty_sigset ();
@@ -31,15 +35,16 @@ char* dl_pstr (char const *message, ...) __attribute__ ((format (printf, 1, 2)))
 const char *dl_get_assert_message() noexcept;
 
 void dl_assert__ (const char *expr, const char *file_name, const char *func_name,
-                  int line, const char *desc, int use_perror);
+                  int line, const char *desc, int use_perror, int generate_coredump);
 
-#define dl_assert_impl(f, str, use_perror) \
+#define dl_assert_impl(f, str, use_perror, generate_coredump) \
   if (unlikely(!(f))) {\
-    dl_assert__ (#f, __FILE__, __FUNCTION__, __LINE__, str, use_perror);\
+    dl_assert__ (#f, __FILE__, __FUNCTION__, __LINE__, str, use_perror, generate_coredump);\
   }
 
-#define dl_assert(f, str) dl_assert_impl (f, str, 0)
-#define dl_passert(f, str) dl_assert_impl (f, str, 1)
+#define dl_assert(f, str) dl_assert_impl (f, str, 0, 0)
+#define dl_passert(f, str) dl_assert_impl (f, str, 1, 0)
+#define dl_assert_with_coredump(f, str) dl_assert_impl(f, str, 0, 1)
 #define dl_unreachable(str) dl_assert (0, str)
 
 struct pid_info_t {
@@ -59,8 +64,17 @@ struct mem_info_t {
   uint32_t rss_shmem;
 };
 
+struct process_rusage_t {
+  double user_time;
+  double system_time;
+  long voluntary_context_switches;
+  long involuntary_context_switches;
+};
+
 mem_info_t get_self_mem_stats();
 int get_self_threads_count();
 int get_pid_info (pid_t pid, pid_info_t *info);
 unsigned long long get_pid_start_time (pid_t pid);
 int get_cpu_total (unsigned long long *cpu_total);
+process_rusage_t get_rusage_info();
+

common/fast-backtrace.cpp

@@ -60,23 +60,26 @@ int fast_backtrace_without_recursions(void **buffer, int size) noexcept {
     stack_end = static_cast<char *>(__libc_stack_end);
   }
 #endif
+  return fast_backtrace_without_recursions_by_bp(get_bp(), stack_end, buffer, size);
+}
 
-  auto *bp = static_cast<stack_frame *>(get_bp());
+int fast_backtrace_without_recursions_by_bp(void *bp, void *stack_end_, void **buffer, int size) noexcept {
+  stack_frame *current_bp = static_cast<stack_frame *>(bp);
   int i = 0;
-  while (i < size && reinterpret_cast<char *>(bp) <= stack_end && !(reinterpret_cast<long>(bp) & (sizeof(long) - 1))) {
-    buffer[i++] = bp->ip;
+  while (i < size && reinterpret_cast<char *>(current_bp) <= stack_end_ && !(reinterpret_cast<long>(current_bp) & (sizeof(long) - 1))) {
+    buffer[i++] = current_bp->ip;
     for (int possible_recursion_len = 1; i >= 2 * possible_recursion_len; ++possible_recursion_len) {
       if (std::equal(buffer + i - possible_recursion_len, buffer + i, buffer + i - possible_recursion_len * 2)) {
         i -= possible_recursion_len;
         break;
       }
     }
 
-    stack_frame *p = bp->bp;
-    if (p <= bp) {
+    stack_frame *p = current_bp->bp;
+    if (p <= current_bp) {
       break;
     }
-    bp = p;
+    current_bp = p;
   }
   return i;
 }

common/fast-backtrace.h

@@ -15,5 +15,6 @@ extern char *stack_end;
 
 int fast_backtrace (void **buffer, int size) __attribute__ ((noinline));
 int fast_backtrace_without_recursions(void **buffer, int size) noexcept;
+int fast_backtrace_without_recursions_by_bp(void *bp, void *stack_end_, void **buffer, int size) noexcept;
 
 #endif