relace tlcompiler with tlgen

.github/workflows/macos.yml

@@ -33,10 +33,11 @@ jobs:
       run: |
         brew tap shivammathur/php
         brew update
-        brew install re2c cmake coreutils openssl libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/php@7.4 python@3.12
+        brew install re2c cmake coreutils openssl libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/php@7.4 python@3.12 golang
         brew link --overwrite --force shivammathur/php/php@7.4
         /usr/local/Frameworks/Python.framework/Versions/3.12/bin/python3.12 -m pip install --upgrade pip --break-system-packages && /usr/local/Frameworks/Python.framework/Versions/3.12/bin/pip3 install jsonschema install --break-system-packages jsonschema
-
+        go install github.com/vkcom/tl/cmd/tlgen@latest
+
     - name: Run cmake
       run: cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DDOWNLOAD_MISSING_LIBRARIES=On -S $GITHUB_WORKSPACE -B ${{runner.workspace}}/build
 

Dockerfile

@@ -5,12 +5,13 @@ RUN apt-get update && \
     apt-get install -y --no-install-recommends apt-utils ca-certificates gnupg wget && \
     wget -qO /etc/apt/trusted.gpg.d/vkpartner.asc https://artifactory-external.vkpartner.ru/artifactory/api/gpg/key/public && \
     echo "deb https://artifactory-external.vkpartner.ru/artifactory/kphp focal main" >> /etc/apt/sources.list && \
+    echo "deb https://artifactory.mvk.com/artifactory/debian bullseye-backports main" >> /etc/apt/sources.list && \
     apt-get update && \
-    apt-get install -y --no-install-recommends kphp php7.4-vkext vk-flex-data vk-tl-tools && \
+    apt-get install -y --no-install-recommends kphp php7.4-vkext vk-flex-data vk-tl-tools tlgen && \
     rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /var/www/vkontakte/data/www/vkontakte.com/tl/ && \
-    tl-compiler -e /var/www/vkontakte/data/www/vkontakte.com/tl/scheme.tlo -w 4 \
+    tlgen -ignoreGeneratedCode -tloPath /var/www/vkontakte/data/www/vkontakte.com/tl/scheme.tlo \
     /usr/share/vkontakte/examples/tl-files/common.tl /usr/share/vkontakte/examples/tl-files/tl.tl
 
 RUN useradd -ms /bin/bash kitten

cmake/external-libraries.cmake

@@ -56,7 +56,7 @@ endif()
 
 find_library(KPHP_H3 h3)
 if(KPHP_H3)
-    add_library(h3 STATIC IMPORTED ${KPHP_H3})
+    add_library(libh3 STATIC IMPORTED ${KPHP_H3})
 else()
     handle_missing_library("h3")
     FetchContent_Declare(

common/tl/tl.cmake

@@ -1,8 +1,7 @@
 include(${COMMON_DIR}/tlo-parsing/tlo-parsing.cmake)
-include(${COMMON_DIR}/tl/compiler/tl-compiler.cmake)
 include(${COMMON_DIR}/tl2php/tl2php.cmake)
 
-install(TARGETS tl-compiler tl2php
+install(TARGETS tl2php
         COMPONENT TL_TOOLS
         RUNTIME DESTINATION bin)
 

compiler/compiler-core.cpp

@@ -465,7 +465,7 @@ VarPtr CompilerCore::create_var(const std::string &name, VarData::Type type) {
 
 VarPtr CompilerCore::get_global_var(const std::string &name, VarData::Type type,
                                     VertexPtr init_val, bool *is_new_inserted) {
-  TSHashTable<VarPtr>::HTNode *node = global_vars_ht.at(vk::std_hash(name));
+  auto *node = global_vars_ht.at(vk::std_hash(name));
   VarPtr new_var;
   if (!node->data) {
     AutoLocker<Lockable *> locker(node);

compiler/compiler-core.h

@@ -33,7 +33,7 @@ class CompilerCore {
   TSHashTable<FunctionPtr> functions_ht;
   TSHashTable<ClassPtr> classes_ht;
   TSHashTable<DefinePtr> defines_ht;
-  TSHashTable<VarPtr> global_vars_ht;
+  TSHashTable<VarPtr,2'000'000> global_vars_ht;
   TSHashTable<LibPtr, 1000> libs_ht;
   TSHashTable<ModulitePtr, 1000> modulites_ht;
   TSHashTable<ComposerJsonPtr, 1000> composer_json_ht;

docs/kphp-basics/installation.md

@@ -75,7 +75,7 @@ sudo apt install kphp vk-tl-tools
 sudo apt install php7.4-vkext
 
 sudo mkdir -p /var/www/vkontakte/data/www/vkontakte.com/tl/
-sudo tl-compiler -e /var/www/vkontakte/data/www/vkontakte.com/tl/scheme.tlo /usr/share/vkontakte/examples/tl-files/common.tl /usr/share/vkontakte/examples/tl-files/tl.tl
+sudo tlgen -ignoreGeneratedCode -tloPath /var/www/vkontakte/data/www/vkontakte.com/tl/scheme.tlo /usr/share/vkontakte/examples/tl-files/common.tl /usr/share/vkontakte/examples/tl-files/tl.tl
 ```
 
 

docs/kphp-client/tl-schema-and-rpc/tl2php.md

@@ -9,9 +9,10 @@ A console tool to generate PHP classes based on TL schema. TL schema must be bef
 
 ## How to make the .tlo file from TL schema
 
-*.tlo* file is a "compiled TL schema" file. You use `tl-compiler` to generate it from a set of **.tl* files. It is inside the `vk-tl-tools` package, that had been installed if you followed the installation instructions.
+*.tlo* file is a "compiled TL schema" file. You use `tlgen` to generate it from a set of **.tl* files. It is inside the `tlgen` package, that had been installed if you followed the installation instructions.  
+`-ignoreGeneratedCode` *- omits generated Golang code of (de)serializers. More on that see [here](https://github.com/vkcom/tl).*
 ```bash
-tl-compiler -e /path/to/output.tlo input1.tl input2.tl ...
+tlgen -ignoreGeneratedCode -tloPath /path/to/output.tlo input1.tl input2.tl ...
 ```
 
 

docs/kphp-internals/developing-and-extending-kphp/compiling-kphp-from-sources.md

@@ -176,9 +176,10 @@ sudo cp ../objs/vkext/modules${PHP_VERSION}/vkext.so $(php-config${PHP_VERSION}
 
 **Compiling .tlo for php.ini**
 
-Use the `tl-compiler` executable from `vk-tl-tools` package:
+Use the `tlgen` executable from `tlgen` package:  
+`-ignoreGeneratedCode` *- omits generated Golang code of (de)serializers. More on that see [here](https://github.com/vkcom/tl).*
 ```bash
-tl-compiler -e /path/to/output.tlo input1.tl input2.tl ...
+tlgen -ignoreGeneratedCode -tloPath /path/to/output.tlo input1.tl input2.tl ...
 ```
 
 

tests/python/lib/file_utils.py

@@ -44,10 +44,10 @@ def search_tl_client():
 def search_combined_tlo(working_dir):
     if _ENGINE_INSTALL_PATH:
         return _check_file("combined.tlo", _ENGINE_INSTALL_PATH, os.path.isfile)
-    tl_compiler_path = _check_file("objs/bin/tl-compiler", _KPHP_REPO, _check_bin)
+    tlgen_path = _check_file("tlgen", _KPHP_REPO, _check_bin)
     common_tl = _check_file("common/tl-files/common.tl", _KPHP_REPO, os.path.isfile)
     subprocess.call(
-        ["bash", "-c", "{} -e {} {}".format(tl_compiler_path, working_dir + "/combined.tlo", common_tl)],
+        ["bash", "-c", "{} -ignoreGeneratedCode -tloPath {} {}".format(tlgen_path, working_dir + "/combined.tlo", common_tl)],
         env={"ASAN_OPTIONS": "detect_leaks=0"}
     )
     return _check_file("combined.tlo", working_dir, os.path.isfile)