Add openssl and curl vendoring

Signed-off-by: Petr Shumilov <p.shumilov@vkteam.ru>

.github/workflows/Dockerfile.buster

@@ -22,7 +22,7 @@ RUN apt-get update && \
       git cmake-data=3.18* cmake=3.18* make g++ gperf netcat \
       python3.7 python3-dev libpython3-dev python3-pip python3-setuptools python3-wheel mysql-server libmysqlclient-dev && \
     pip3 install -r /tmp/requirements.txt && \
-    apt-get install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
+    apt-get install -y --no-install-recommends kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
       libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \
       libpq5=14.* postgresql-14 postgresql-server-dev-14 libpq-dev=14.* libnuma-dev composer && \
     rm -rf /var/lib/apt/lists/* && \

.github/workflows/Dockerfile.focal

@@ -15,7 +15,7 @@ RUN apt-get update && \
       git cmake make clang g++ g++-10 gperf netcat \
       python3.7 python3-pip python3.7-distutils python3.7-dev libpython3.7-dev python3-jsonschema python3-setuptools mysql-server libmysqlclient-dev && \
     python3.7 -m pip install pip && python3.7 -m pip install -r /tmp/requirements.txt && \
-    apt-get install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
+    apt-get install -y --no-install-recommends kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
       libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \
       libpq5=14.* postgresql-14 postgresql-server-dev-14 libpq-dev=14.* libnuma-dev composer unzip && \
     rm -rf /var/lib/apt/lists/*

.github/workflows/Dockerfile.jammy

@@ -19,7 +19,7 @@ RUN apt update && \
       git cmake make g++ lld gperf netcat \
       python3.7 python3-pip python3.7-distutils python3.7-dev libpython3.7-dev python3-jsonschema python3-setuptools mysql-server libmysqlclient-dev && \
     python3.7 -m pip install pip && python3.7 -m pip install -r /tmp/requirements.txt && \
-    apt install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
+    apt install -y --no-install-recommends kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
       libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libnuma-dev unzip \
       libldap-dev libkrb5-dev libpq5=14.* postgresql-14 postgresql-server-dev-14 libpq-dev=14.* && \
     rm -rf /var/lib/apt/lists/*

.gitmodules

@@ -7,3 +7,11 @@
 [submodule "third-party/pcre2"]
 	path = third-party/pcre2
 	url = https://github.com/PCRE2Project/pcre2.git
+[submodule "third-party/openssl"]
+	path = third-party/openssl
+	url = https://github.com/openssl/openssl.git
+	branch = OpenSSL_1_1_1-stable
+[submodule "third-party/curl"]
+	path = third-party/curl
+	url = https://github.com/VKCOM/curl.git
+	branch = dpkg-build-7.60.0

CMakeLists.txt

@@ -20,6 +20,7 @@ include(CheckCXXCompilerFlag)
 include(AddFileDependencies)
 include(FetchContent)
 include(GNUInstallDirs)
+include(ExternalProject)
 
 # Global includes must be before all other includes/add_subdirectories
 include(cmake/utils.cmake)
@@ -29,6 +30,10 @@ include(cmake/external-libraries.cmake)
 include(cmake/init-compilation-flags.cmake)
 include(cmake/popular-common.cmake)
 
+update_git_submodules()
+include(${THIRD_PARTY_DIR}/openssl-cmake/openssl.cmake)
+include(${THIRD_PARTY_DIR}/curl-cmake/curl.cmake)
+
 # TODO: use FetchContent_Declare instead of include_directories
 include_directories(.)
 
@@ -85,6 +90,13 @@ else ()
             ARCHIVE DESTINATION ${INSTALL_KPHP_SOURCE}/objs)
 endif ()
 
+install(DIRECTORY ${OBJS_DIR}/include
+        COMPONENT KPHP
+        DESTINATION ${INSTALL_KPHP_SOURCE}/objs)
+install(DIRECTORY ${OBJS_DIR}/lib
+        COMPONENT KPHP
+        DESTINATION ${INSTALL_KPHP_SOURCE}/objs)
+
 install(DIRECTORY ${COMMON_DIR}
                   ${BASE_DIR}/runtime
 		  ${RUNTIME_COMMON_DIR}

cmake/init-compilation-flags.cmake

@@ -49,9 +49,6 @@ else()
     add_link_options(-fno-lto)
 endif()
 
-set(OPENSSL_USE_STATIC_LIBS TRUE)
-find_package(OpenSSL REQUIRED)
-include_directories(${OPENSSL_INCLUDE_DIR})
 
 option(ADDRESS_SANITIZER "Enable address sanitizer")
 if(ADDRESS_SANITIZER)

cmake/popular-common.cmake

@@ -52,4 +52,4 @@ if(COMPILE_RUNTIME_LIGHT)
 endif()
 
 vk_add_library(popular_common OBJECT ${POPULAR_COMMON_SOURCES})
-set_property(TARGET popular_common PROPERTY POSITION_INDEPENDENT_CODE ON)
+#set_property(TARGET popular_common PROPERTY POSITION_INDEPENDENT_CODE ON)

common/crypto/aes256-generic.cpp

@@ -4,9 +4,9 @@
 
 #include "common/crypto/aes256-generic.h"
 
-#include <string.h>
+#include "openssl/aes.h"
 
-#include <openssl/aes.h>
+#include <string.h>
 
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"

common/crypto/aes256.h

@@ -8,7 +8,7 @@
 #include <sys/cdefs.h>
 #include <stdint.h>
 
-#include <openssl/aes.h>
+#include "openssl/aes.h"
 
 #define AES256_KEY_BITS 256
 

common/kfs/kfs-binlog.cpp

@@ -8,7 +8,6 @@
 #include <cstring>
 #include <cstdlib>
 #include <fcntl.h>
-#include <openssl/rand.h>
 #include <sys/stat.h>
 #include <unistd.h>
 

common/sha1.h

@@ -5,7 +5,7 @@
 #ifndef __SHA1_H__
 #define __SHA1_H__
 
-#include <openssl/sha.h>
+#include "openssl/sha.h"
 
 void sha1 (unsigned char *input, int ilen, unsigned char output[20]);
 

common/wrappers/openssl.h

@@ -5,13 +5,14 @@
 #ifndef KDB_COMMON_OPENSSL_H
 #define KDB_COMMON_OPENSSL_H
 
-#include <openssl/crypto.h>
-#include <openssl/engine.h>
-#include <openssl/evp.h>
-#include <openssl/opensslv.h>
 #include <stddef.h>
 #include <string.h>
 
+#include "openssl/crypto.h"
+#include "openssl/engine.h"
+#include "openssl/evp.h"
+#include "openssl/opensslv.h"
+
 #if OPENSSL_VERSION_NUMBER < 0x10100000L
 
 static inline RSA *EVP_PKEY_get0_RSA(EVP_PKEY *pkey) {

compiler/compiler-settings.cpp

@@ -5,10 +5,12 @@
 #include "compiler/compiler-settings.h"
 
 #include <fstream>
-#include <openssl/sha.h>
 #include <sstream>
+#include <string_view>
 #include <unistd.h>
 
+#include "openssl/sha.h"
+
 #include "common/algorithms/contains.h"
 #include "common/algorithms/find.h"
 #include "common/version-string.h"
@@ -143,16 +145,22 @@ void append_if_doesnt_contain(std::string &ld_flags, const T &libs, vk::string_v
   }
 }
 
-void append_curl(std::string &cxx_flags, std::string &ld_flags) noexcept {
+void append_3dparty_headers(std::string &cxx_flags, const std::string &path_to_3dparty, const std::string &libname) noexcept {
+  cxx_flags += " -I" + path_to_3dparty + "include/" + libname;
+}
+
+void append_3dparty_lib(std::string &ld_flags, const std::string &path_to_3dparty, const std::string &libname) noexcept {
+  ld_flags += " " + path_to_3dparty + "lib/lib" + libname + ".a";
+}
+
+void append_curl([[maybe_unused]] std::string &cxx_flags, std::string &ld_flags, const std::string &path_to_3dparty) noexcept {
   if (!contains_lib(ld_flags, "curl")) {
 #if defined(__APPLE__)
-    static_cast<void>(cxx_flags);
     ld_flags += " -lcurl";
 #else
     // TODO make it as an option?
-    const std::string curl_dir = "/opt/curl7600";
-    cxx_flags += " -I" + curl_dir + "/include/";
-    ld_flags += " " + curl_dir + "/lib/libcurl.a";
+    append_3dparty_headers(cxx_flags, path_to_3dparty, "curl");
+    append_3dparty_lib(ld_flags, path_to_3dparty, "curl");
 #endif
   }
 }
@@ -354,10 +362,12 @@ void CompilerSettings::init() {
 
   remove_extra_spaces(extra_ld_flags.value_);
 
+  auto third_party_path = kphp_src_path.get() + "objs/";
+
   ld_flags.value_ = extra_ld_flags.get();
-  append_curl(cxx_default_flags, ld_flags.value_);
+  append_curl(cxx_default_flags, ld_flags.value_, third_party_path);
   append_apple_options(cxx_default_flags, ld_flags.value_);
-  std::vector<vk::string_view> external_static_libs{"pcre", "re2", "yaml-cpp", "h3", "z", "zstd", "nghttp2", "kphp-timelib"};
+  std::vector<vk::string_view> os_installed_libs{"pcre", "re2", "yaml-cpp", "h3", "z", "zstd", "nghttp2", "kphp-timelib"};
 
 #ifdef KPHP_TIMELIB_LIB_DIR
   ld_flags.value_ += " -L" KPHP_TIMELIB_LIB_DIR;
@@ -381,7 +391,7 @@ void CompilerSettings::init() {
 
 #ifdef PDO_DRIVER_MYSQL
 #ifdef PDO_LIBS_STATIC_LINKING
-  external_static_libs.emplace_back("mysqlclient");
+  os_installed_libs.emplace_back("mysqlclient");
 #else
   external_libs.emplace_back("mysqlclient");
 #endif
@@ -390,9 +400,9 @@ void CompilerSettings::init() {
 #ifdef PDO_DRIVER_PGSQL
 #ifdef PDO_LIBS_STATIC_LINKING
   ld_flags.value_ += fmt_format(" -L /usr/lib/postgresql/{}/lib/ ", PDO_DRIVER_PGSQL_VERSION);
-  external_static_libs.emplace_back("pq");
-  external_static_libs.emplace_back("pgcommon");
-  external_static_libs.emplace_back("pgport");
+  os_installed_libs.emplace_back("pq");
+  os_installed_libs.emplace_back("pgcommon");
+  os_installed_libs.emplace_back("pgport");
   // following common libraries are required for libpq.a
   external_libs.emplace_back("ldap");
   external_libs.emplace_back("gssapi_krb5");
@@ -401,18 +411,19 @@ void CompilerSettings::init() {
 #endif
 #endif
 
-  external_static_libs.emplace_back("ssl");
-  external_static_libs.emplace_back("crypto");
+  append_3dparty_headers(cxx_default_flags, third_party_path, "openssl");
+  append_3dparty_lib(ld_flags.value_, third_party_path, "ssl");
+  append_3dparty_lib(ld_flags.value_, third_party_path, "crypto");
 
 #if defined(__APPLE__)
-  append_if_doesnt_contain(ld_flags.value_, external_static_libs, "-l");
+  append_if_doesnt_contain(ld_flags.value_, os_installed_libs, "-l");
   auto flex_prefix = kphp_src_path.value_ + "objs/flex/lib";
   append_if_doesnt_contain(ld_flags.value_, vk::to_array({"vk-flex-data"}), flex_prefix, ".a");
   external_libs.emplace_back("iconv");
 #else
-  external_static_libs.emplace_back("numa");
-  external_static_libs.emplace_back("vk-flex-data");
-  append_if_doesnt_contain(ld_flags.value_, external_static_libs, "-l:lib", ".a");
+  append_3dparty_lib(ld_flags.value_, third_party_path, "vk-flex-data");
+  os_installed_libs.emplace_back("numa");
+  append_if_doesnt_contain(ld_flags.value_, os_installed_libs, "-l:lib", ".a");
   external_libs.emplace_back("rt");
 #endif
   append_if_doesnt_contain(ld_flags.value_, external_libs, "-l");

compiler/kphp2cpp.cpp

@@ -182,7 +182,7 @@ std::string get_default_extra_cxxflags() noexcept {
 }
 
 std::string get_default_extra_ldflags() noexcept {
-  std::string flags{"-L${KPHP_PATH}/objs/flex -ggdb -fno-lto"};
+  std::string flags{"-ggdb -fno-lto"};
 #ifdef KPHP_HAS_NO_PIE
   flags += " " KPHP_HAS_NO_PIE;
 #endif

flex/flex.cmake

@@ -32,6 +32,10 @@ set_target_properties(flex_data_shared flex_data_static
                       PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${OBJS_DIR}/flex
                                  ARCHIVE_OUTPUT_DIRECTORY ${OBJS_DIR}/flex
                                  OUTPUT_NAME vk-flex-data)
+add_custom_command(
+        TARGET flex_data_static POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:flex_data_static> ${OBJS_DIR}/lib/libvk-flex-data.a
+)
 
 install(TARGETS flex_data_shared flex_data_static
         COMPONENT FLEX

net/net-crypto-aes.h

@@ -7,7 +7,7 @@
 
 #include <stdlib.h>
 
-#include <openssl/aes.h>
+#include "openssl/aes.h"
 
 #include "common/pid.h"
 #include "common/crypto/aes256.h"

runtime/curl.cpp

@@ -6,9 +6,10 @@
 
 #include <cstdio>
 #include <cstring>
-#include <curl/curl.h>
-#include <curl/easy.h>
-#include <curl/multi.h>
+
+#include "curl/curl.h"
+#include "curl/easy.h"
+#include "curl/multi.h"
 
 #include "runtime-common/stdlib/tracing/tracing-functions.h"
 #include "runtime/context/runtime-context.h"

runtime/openssl.cpp

@@ -7,20 +7,21 @@
 #include <cerrno>
 #include <memory>
 #include <netdb.h>
-#include <openssl/asn1.h>
-#include <openssl/err.h>
-#include <openssl/evp.h>
-#include <openssl/hmac.h>
-#include <openssl/md5.h>
-#include <openssl/pem.h>
-#include <openssl/rsa.h>
-#include <openssl/sha.h>
-#include <openssl/ssl.h>
-#include <openssl/x509v3.h>
 #include <poll.h>
 #include <sys/time.h>
 #include <unistd.h>
 
+#include "openssl/asn1.h"
+#include "openssl/err.h"
+#include "openssl/evp.h"
+#include "openssl/hmac.h"
+#include "openssl/md5.h"
+#include "openssl/pem.h"
+#include "openssl/rsa.h"
+#include "openssl/sha.h"
+#include "openssl/ssl.h"
+#include "openssl/x509v3.h"
+
 #include "common/crc32.h"
 #include "common/resolver.h"
 #include "common/smart_ptrs/unique_ptr_with_delete_function.h"

runtime/openssl.h

@@ -4,7 +4,7 @@
 
 #pragma once
 
-#include <openssl/pkcs7.h>
+#include "openssl/pkcs7.h"
 
 #include "runtime-common/core/runtime-core.h"
 

runtime/runtime.cmake

@@ -146,18 +146,19 @@ allow_deprecated_declarations(${BASE_DIR}/runtime/allocator.cpp ${BASE_DIR}/runt
 allow_deprecated_declarations_for_apple(${BASE_DIR}/runtime/inter-process-mutex.cpp)
 
 vk_add_library(kphp_runtime OBJECT ${KPHP_RUNTIME_ALL_SOURCES})
-target_include_directories(kphp_runtime PUBLIC ${BASE_DIR} /opt/curl7600/include)
+target_include_directories(kphp_runtime PUBLIC ${BASE_DIR} ${OPENSSL_INCLUDE_DIR} ${CURL_INCLUDE_DIR})
 
-add_dependencies(kphp_runtime kphp-timelib)
+add_dependencies(kphp_runtime kphp-timelib curl)
+add_dependencies(curl openssl)
 
 prepare_cross_platform_libs(RUNTIME_LIBS yaml-cpp re2 zstd h3) # todo: linking between static libs is no-op, is this redundant? do we need to add mysqlclient here?
-set(RUNTIME_LIBS vk::kphp_runtime vk::kphp_server vk::runtime-common vk::popular_common vk::unicode vk::common_src vk::binlog_src vk::net_src ${RUNTIME_LIBS} OpenSSL::Crypto m z pthread)
+set(RUNTIME_LIBS vk::kphp_runtime vk::kphp_server vk::runtime-common vk::popular_common vk::unicode vk::common_src vk::binlog_src vk::net_src ${RUNTIME_LIBS} CURL::curl OpenSSL::SSL OpenSSL::Crypto m z pthread)
 vk_add_library(kphp-full-runtime STATIC)
 target_link_libraries(kphp-full-runtime PUBLIC ${RUNTIME_LIBS})
 set_target_properties(kphp-full-runtime PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${OBJS_DIR})
 
 prepare_cross_platform_libs(RUNTIME_LINK_TEST_LIBS pcre nghttp2 kphp-timelib)
-set(RUNTIME_LINK_TEST_LIBS vk::flex_data_static ${CURL_LIB} OpenSSL::SSL ${NUMA_LIB} ${RUNTIME_LINK_TEST_LIBS} ${EPOLL_SHIM_LIB} ${ICONV_LIB} ${RT_LIB})
+set(RUNTIME_LINK_TEST_LIBS vk::flex_data_static CURL::curl OpenSSL::SSL ${NUMA_LIB} ${RUNTIME_LINK_TEST_LIBS} ${EPOLL_SHIM_LIB} ${ICONV_LIB} ${RT_LIB})
 
 if (PDO_DRIVER_MYSQL)
     list(APPEND RUNTIME_LINK_TEST_LIBS mysqlclient)