Merge branch 'master' into andreylzmw-gh-install

.github/workflows/Build.yml

@@ -19,21 +19,21 @@ jobs:
     strategy:
       matrix:
         include:
-          - os: buster
-            compiler: g++
-            cpp: 17
-            asan: off
-            ubsan: off
-          - os: focal
-            compiler: clang++
-            cpp: 17
-            asan: off
-            ubsan: on
-          - os: focal
-            compiler: g++-10
-            cpp: 20
-            asan: on
-            ubsan: off
+          # - os: buster
+          #   compiler: g++
+          #   cpp: 17
+          #   asan: off
+          #   ubsan: off
+          # - os: focal
+          #   compiler: clang++
+          #   cpp: 17
+          #   asan: off
+          #   ubsan: on
+          # - os: focal
+          #   compiler: g++-10
+          #   cpp: 20
+          #   asan: on
+          #   ubsan: off
           - os: jammy
             compiler: g++
             cpp: 20
@@ -93,26 +93,26 @@ jobs:
       run: docker exec kphp-build-container-${{matrix.os}} bash -c
               "composer install -d ${{env.kphp_polyfills_dir}}"
 
-    - name: Run python tests
-      id: python_tests
-      continue-on-error: true
-      run: docker exec kphp-build-container-${{matrix.os}} bash -c
-              "chown -R kitten /home && su kitten -c 'GITHUB_ACTIONS=1 KPHP_TESTS_POLYFILLS_REPO=${{env.kphp_polyfills_dir}} KPHP_CXX=${{matrix.compiler}} python3.7 -m pytest --tb=native -n$(nproc) ${{env.kphp_root_dir}}/tests/python/'"
+    # - name: Run python tests
+    #   id: python_tests
+    #   continue-on-error: true
+    #   run: docker exec kphp-build-container-${{matrix.os}} bash -c
+    #           "chown -R kitten /home && su kitten -c 'GITHUB_ACTIONS=1 KPHP_TESTS_POLYFILLS_REPO=${{env.kphp_polyfills_dir}} KPHP_CXX=${{matrix.compiler}} python3.7 -m pytest --tb=native -n$(nproc) ${{env.kphp_root_dir}}/tests/python/'"
 
-    - name: Prepare python tests artifacts
-      if: steps.python_tests.outcome == 'failure'
-      run: docker cp kphp-build-container-${{matrix.os}}:${{env.kphp_root_dir}}/tests/python/_tmp/ ${{runner.temp}} &&
-           rm -rf ${{runner.temp}}/_tmp/*/working_dir
+    # - name: Prepare python tests artifacts
+    #   if: steps.python_tests.outcome == 'failure'
+    #   run: docker cp kphp-build-container-${{matrix.os}}:${{env.kphp_root_dir}}/tests/python/_tmp/ ${{runner.temp}} &&
+    #        rm -rf ${{runner.temp}}/_tmp/*/working_dir
 
-    - name: Upload python tests artifacts
-      uses: actions/upload-artifact@v3
-      if: steps.python_tests.outcome == 'failure'
-      with:
-        path: ${{runner.temp}}/_tmp/
+    # - name: Upload python tests artifacts
+    #   uses: actions/upload-artifact@v3
+    #   if: steps.python_tests.outcome == 'failure'
+    #   with:
+    #     path: ${{runner.temp}}/_tmp/
 
-    - name: Fail pipeline if python tests failed
-      if: steps.python_tests.outcome == 'failure'
-      run: exit 1
+    # - name: Fail pipeline if python tests failed
+    #   if: steps.python_tests.outcome == 'failure'
+    #   run: exit 1
 
     - name: Build deb package
       run: docker exec kphp-build-container-${{matrix.os}} bash -c
@@ -124,31 +124,31 @@ jobs:
 
     - name: Run php dummy script
       run: docker exec kphp-build-container-${{matrix.os}} bash -c
-            "сd ${{env.kphp_build_dir}} && echo 'hello world' > test.php && kphp --mode=cli test.php && ./kphp_out/cli"
+            "su kitten && cd ${{env.kphp_build_dir}} && echo 'hello world' > test.php && kphp --user=root --mode=cli test.php && ./kphp_out/cli"
 
     - name: Remove docker container
       run: docker rm -f kphp-build-container-${{matrix.os}}
 
-  install-linux:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        include:
-  #         - os: buster
-  #         - os: focal
-          - os: jammy
+  # install-linux:
+  #   runs-on: ubuntu-latest
+  #   strategy:
+  #     matrix:
+  #       include:
+  # #         - os: buster
+  # #         - os: focal
+  #         - os: jammy
 
-    steps:
-    - uses: actions/checkout@v3
+  #   steps:
+  #   - uses: actions/checkout@v3
 
-    - name: Build and start Docker container
-      run: |
-        docker build -f $GITHUB_WORKSPACE/.github/workflows/Dockerfile.${{matrix.os}}.install $GITHUB_WORKSPACE -t kphp-build-img-${{matrix.os}}-install
-        docker run -dt --name kphp-build-container-${{matrix.os}}-install kphp-build-img-${{matrix.os}}-install
+  #   - name: Build and start Docker container
+  #     run: |
+  #       docker build -f $GITHUB_WORKSPACE/.github/workflows/Dockerfile.${{matrix.os}}.install $GITHUB_WORKSPACE -t kphp-build-img-${{matrix.os}}-install
+  #       docker run -dt --name kphp-build-container-${{matrix.os}}-install kphp-build-img-${{matrix.os}}-install
 
-    - name: Run php dummy script
-      run: docker exec -u kitten kphp-build-container-${{matrix.os}}-install bash -c 
-        "cd ${{env.demo_dir}} && echo 'hello world' > demo.php && kphp --mode=cli --cxx=g++ demo.php && ./kphp_out/cli -o --user kitten"
+  #   - name: Run php dummy script
+  #     run: docker exec -u kitten kphp-build-container-${{matrix.os}}-install bash -c 
+  #       "cd ${{env.demo_dir}} && echo 'hello world' > demo.php && kphp --mode=cli --cxx=g++ demo.php && ./kphp_out/cli -o --user kitten"
 
   # build-macos:
   #   runs-on: ${{matrix.os}}-12

.github/workflows/Dockerfile.buster

@@ -11,7 +11,7 @@ RUN apt-get update && \
     wget -qO - https://packages.sury.org/php/apt.gpg | apt-key add - && \
     echo "deb https://packages.sury.org/php/ buster main" >> /etc/apt/sources.list.d/php.list && \
     TEMP_DEB="$(mktemp)" && \
-    wget -O "$TEMP_DEB" 'https://dev.mysql.com/get/mysql-apt-config_0.8.22-1_all.deb' && \
+    wget -O "$TEMP_DEB" 'https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb' && \
     dpkg -i "$TEMP_DEB" && \
     rm -f "$TEMP_DEB" && \
     echo "deb http://apt.postgresql.org/pub/repos/apt buster-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \

.github/workflows/macos.yml

@@ -33,7 +33,7 @@ jobs:
       run: |
         brew tap shivammathur/php
         brew update
-        brew install re2c cmake coreutils openssl libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/php@7.4 python@3.12
+        brew install re2c cmake coreutils openssl libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/php@7.4
         brew link --overwrite --force shivammathur/php/php@7.4
         /usr/local/Frameworks/Python.framework/Versions/3.12/bin/python3.12 -m pip install --upgrade pip --break-system-packages && /usr/local/Frameworks/Python.framework/Versions/3.12/bin/pip3 install jsonschema install --break-system-packages jsonschema
       

builtin-functions/_functions.txt

@@ -1270,6 +1270,7 @@ define('CURLOPT_TCP_KEEPALIVE', 200123);
 define('CURLOPT_TCP_KEEPIDLE', 200124);
 define('CURLOPT_TCP_KEEPINTVL', 200125);
 define('CURLOPT_PRIVATE', 200126);
+define('CURLOPT_ACCEPT_ENCODING', 200127);
 
 define('CURLPROXY_HTTP', 400000);
 define('CURLPROXY_HTTP_1_0', 400001);

cmake/external-libraries.cmake

@@ -56,7 +56,7 @@ endif()
 
 find_library(KPHP_H3 h3)
 if(KPHP_H3)
-    add_library(h3 STATIC IMPORTED ${KPHP_H3})
+    add_library(libh3 STATIC IMPORTED ${KPHP_H3})
 else()
     handle_missing_library("h3")
     FetchContent_Declare(

common/fast-backtrace.cpp

@@ -39,17 +39,20 @@ int fast_backtrace(void **buffer, int size) {
     stack_end = static_cast<char *>(__libc_stack_end);
   }
 #endif
+  return fast_backtrace_by_bp(get_bp(), stack_end, buffer, size);
+}
 
-  auto *bp = static_cast<stack_frame *>(get_bp());
+int fast_backtrace_by_bp(void *bp, void *stack_end_, void **buffer, int size) {
+  stack_frame *current_bp = static_cast<stack_frame *>(bp);
   int i = 0;
-  while (i < size && reinterpret_cast<char *>(bp) <= stack_end && !(reinterpret_cast<long>(bp) & (sizeof(long) - 1))) {
-    void *ip = bp->ip;
+  while (i < size && reinterpret_cast<char *>(current_bp) <= stack_end_ && !(reinterpret_cast<long>(current_bp) & (sizeof(long) - 1))) {
+    void *ip = current_bp->ip;
     buffer[i++] = ip;
-    stack_frame *p = bp->bp;
-    if (p <= bp) {
+    stack_frame *p = current_bp->bp;
+    if (p <= current_bp) {
       break;
     }
-    bp = p;
+    current_bp = p;
   }
   return i;
 }

common/fast-backtrace.h

@@ -14,6 +14,7 @@ extern char *stack_end;
 }
 
 int fast_backtrace (void **buffer, int size) __attribute__ ((noinline));
+int fast_backtrace_by_bp (void *bp, void *stack_end_, void **buffer, int size) __attribute__ ((noinline));
 int fast_backtrace_without_recursions(void **buffer, int size) noexcept;
 int fast_backtrace_without_recursions_by_bp(void *bp, void *stack_end_, void **buffer, int size) noexcept;
 

compiler/compiler-core.cpp

@@ -465,7 +465,7 @@ VarPtr CompilerCore::create_var(const std::string &name, VarData::Type type) {
 
 VarPtr CompilerCore::get_global_var(const std::string &name, VarData::Type type,
                                     VertexPtr init_val, bool *is_new_inserted) {
-  TSHashTable<VarPtr>::HTNode *node = global_vars_ht.at(vk::std_hash(name));
+  auto *node = global_vars_ht.at(vk::std_hash(name));
   VarPtr new_var;
   if (!node->data) {
     AutoLocker<Lockable *> locker(node);

compiler/compiler-core.h

@@ -33,7 +33,7 @@ class CompilerCore {
   TSHashTable<FunctionPtr> functions_ht;
   TSHashTable<ClassPtr> classes_ht;
   TSHashTable<DefinePtr> defines_ht;
-  TSHashTable<VarPtr> global_vars_ht;
+  TSHashTable<VarPtr,2'000'000> global_vars_ht;
   TSHashTable<LibPtr, 1000> libs_ht;
   TSHashTable<ModulitePtr, 1000> modulites_ht;
   TSHashTable<ComposerJsonPtr, 1000> composer_json_ht;

compiler/compiler-settings.cpp

@@ -284,11 +284,13 @@ void CompilerSettings::init() {
 
   remove_extra_spaces(extra_cxx_flags.value_);
   std::stringstream ss;
-  ss << extra_cxx_flags.get();
-  ss << " -iquote" << kphp_src_path.get()
-     << " -iquote " << kphp_src_path.get() << "objs/generated/auto/runtime";
-  ss << " -Wall -fwrapv -Wno-parentheses -Wno-trigraphs";
-  ss << " -fno-strict-aliasing -fno-omit-frame-pointer";
+  ss << "-Wall "
+     << extra_cxx_flags.get()
+     << " -iquote" << kphp_src_path.get()
+     << " -iquote " << kphp_src_path.get()
+     << "objs/generated/auto/runtime"
+     << " -fwrapv -Wno-parentheses -Wno-trigraphs"
+     << " -fno-strict-aliasing -fno-omit-frame-pointer";
 #ifdef __x86_64__
   ss << " -march=sandybridge";
 #elif __aarch64__

runtime/array_decl.inl

@@ -252,6 +252,10 @@ public:
   // we can create true vector
   bool has_no_string_keys() const noexcept;
 
+  static size_t estimate_size(int64_t n, bool is_vector) noexcept {
+    return array_inner::estimate_size(n, is_vector);
+  }
+
   T &operator[](int64_t int_key);
   T &operator[](int32_t key) { return (*this)[int64_t{key}]; }
   T &operator[](const string &s);

runtime/confdata-global-manager.cpp

@@ -95,11 +95,12 @@ ConfdataGlobalManager &ConfdataGlobalManager::get() noexcept {
 
 void ConfdataGlobalManager::init(size_t confdata_memory_limit,
                                  std::unordered_set<vk::string_view> &&predefined_wilrdcards,
-                                 std::unique_ptr<re2::RE2> &&blacklist_pattern) noexcept {
+                                 std::unique_ptr<re2::RE2> &&blacklist_pattern,
+                                 std::forward_list<vk::string_view> &&force_ignore_prefixes) noexcept {
   resource_.init(mmap_shared(confdata_memory_limit), confdata_memory_limit);
   confdata_samples_.init(resource_);
   predefined_wildcards_.set_wildcards(std::move(predefined_wilrdcards));
-  key_blacklist_.set_blacklist(std::move(blacklist_pattern));
+  key_blacklist_.set_blacklist(std::move(blacklist_pattern), std::move(force_ignore_prefixes));
 }
 
 ConfdataGlobalManager::~ConfdataGlobalManager() noexcept {

runtime/confdata-global-manager.h

@@ -51,8 +51,9 @@ class ConfdataGlobalManager : vk::not_copyable {
   static ConfdataGlobalManager &get() noexcept;
 
   void init(size_t confdata_memory_limit,
-    std::unordered_set<vk::string_view> &&predefined_wilrdcards,
-            std::unique_ptr<re2::RE2> &&blacklist_pattern) noexcept;
+            std::unordered_set<vk::string_view> &&predefined_wilrdcards,
+            std::unique_ptr<re2::RE2> &&blacklist_pattern,
+            std::forward_list<vk::string_view> &&force_ignore_prefixes) noexcept;
 
   void force_release_all_resources_acquired_by_this_proc_if_init() noexcept {
     if (is_initialized()) {

runtime/confdata-keys.cpp

@@ -131,3 +131,12 @@ void ConfdataKeyMaker::forcibly_change_first_key_wildcard_dots_from_two_to_one()
   second_key_ = string::make_const_string_on_memory(first_key_end, second_key_len, second_key_buffer_.data(), second_key_buffer_.size());
   first_key_type_ = ConfdataFirstKeyType::one_dot_wildcard;
 }
+
+bool ConfdataKeyBlacklist::is_key_forcibly_ignored_by_prefix(vk::string_view key) const noexcept {
+  for (const vk::string_view &prefix : force_ignore_prefixes_) {
+    if (key.starts_with(prefix)) {
+      return true;
+    }
+  }
+  return false;
+}

runtime/confdata-keys.h

@@ -5,6 +5,7 @@
 #pragma once
 #include <array>
 #include <re2/re2.h>
+#include <forward_list>
 #include <unordered_map>
 #include <unordered_set>
 #include <vector>
@@ -112,18 +113,31 @@ class ConfdataKeyMaker : vk::not_copyable {
 };
 
 class ConfdataKeyBlacklist : vk::not_copyable {
+  bool is_key_forcibly_ignored_by_prefix(vk::string_view key) const noexcept;
+
 public:
-  void set_blacklist(std::unique_ptr<re2::RE2> &&blacklist_pattern) noexcept {
+  void set_blacklist(std::unique_ptr<re2::RE2> &&blacklist_pattern, std::forward_list<vk::string_view> &&force_ignore_prefixes) noexcept {
     blacklist_pattern_ = std::move(blacklist_pattern);
+    force_ignore_prefixes_ = std::move(force_ignore_prefixes);
   }
 
   bool is_blacklisted(vk::string_view key) const noexcept {
-    return blacklist_pattern_ &&
+    // from PHP class KphpConfiguration, e.g. for langs
+    if (blacklist_pattern_ &&
            re2::RE2::FullMatch(
              re2::StringPiece(key.data(), static_cast<uint32_t>(key.size())),
-             *blacklist_pattern_);
+             *blacklist_pattern_)) {
+      return true;
+    }
+    // emergency startup option to disable keys by prefix, e.g. 'highload.vid'
+    if (unlikely(!force_ignore_prefixes_.empty()) &&
+        is_key_forcibly_ignored_by_prefix(key)) {
+      return true;
+    }
+    return false;
   }
 
 private:
   std::unique_ptr<re2::RE2> blacklist_pattern_;
+  std::forward_list<vk::string_view> force_ignore_prefixes_;
 };

runtime/curl.cpp

@@ -583,6 +583,7 @@ bool curl_setopt(EasyContext *easy_context, int64_t option, const mixed &value)
       {CURLOPT_TCP_KEEPIDLE,         long_option_setter},
       {CURLOPT_TCP_KEEPINTVL,        long_option_setter},
       {CURLOPT_PRIVATE,              private_option_setter},
+      {CURLOPT_ACCEPT_ENCODING,      string_option_setter},
     });
 
   constexpr size_t CURLOPT_OPTION_OFFSET = 200000;

server/confdata-binlog-events.h

@@ -90,6 +90,10 @@ struct lev_confdata_store_wrapper : impl_::confdata_event<BASE, impl_::select_st
     return size <= 0 ? mixed{string{}} : mc_get_value(mem, size, get_flags());
   }
 
+  size_t get_data_size() const noexcept {
+    return BASE::data_len;
+  }
+
   vk::string_view get_value_as_string() const noexcept {
     const char *mem = BASE::data + BASE::key_len;
     const int size = BASE::data_len;