From 5c2eb19cc26307d34c79513d6302064917c3c078 Mon Sep 17 00:00:00 2001 From: "Maxim [maxirmx] Samsonov" Date: Tue, 30 Jul 2024 21:09:38 +0300 Subject: [PATCH 1/9] fix: provide support for jemalloc LG_VADDR parameter on MacOS (libdwarfs 0.5.7, https://github.com/tamatebako/tebako/issues/177) --- .github/workflows/macos.yml | 14 +++++------ CMakeLists.txt | 24 +++++++++--------- README.adoc | 34 ++++++++++++++++++++++++++ common.env | 2 +- lib/tebako/packager/patch_libraries.rb | 9 +++---- lib/tebako/version.rb | 2 +- 6 files changed, 58 insertions(+), 27 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 4a5e0172..38c1667e 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -48,7 +48,7 @@ concurrency: cancel-in-progress: true env: - CACHE_VER: 02 + CACHE_VER: 03 VERBOSE: no jobs: @@ -60,9 +60,9 @@ jobs: matrix: env: - { os: macos-12, xcode: 13.1 } - - { os: macos-14, xcode: 14.3.1 } + - { os: macos-14, xcode: 14.3.1, LG_VADDR: 39 } - { os: macos-14, xcode: 15.4 } - ruby_ver: ['3.1.6', '3.3.3'] + ruby_ver: ['3.1.6', '3.3.4'] steps: - name: Checkout tebako packaging environment uses: actions/checkout@v4 @@ -99,11 +99,11 @@ jobs: matrix: env: - { os: macos-12, xcode: 13.1 } - - { os: macos-14, xcode: 14.3.1 } + - { os: macos-14, xcode: 14.3.1, LG_VADDR: 39} package_ruby_ver: [ '3.1.6', '3.2.4' ] ruby_ver: ['3.1.6'] include: - - { env: { os: macos-14, xcode: 15.4 }, ruby_ver: '3.3.3', package_ruby_ver: '3.2.4' } + - { env: { os: macos-14, xcode: 15.4 }, ruby_ver: '3.3.4', package_ruby_ver: '3.3.4' } steps: - name: Checkout tebako packaging environment uses: actions/checkout@v4 @@ -152,11 +152,11 @@ jobs: matrix: env: - { os: macos-12, xcode: 13.1 } - - { os: macos-14, xcode: 14.3.1 } + - { os: macos-14, xcode: 14.3.1, LG_VADDR: 39} package_ruby_ver: [ '3.1.6', '3.2.4' ] ruby_ver: ['3.1.6'] include: - - { env: { os: macos-14, xcode: 15.4 }, ruby_ver: '3.3.3', package_ruby_ver: '3.2.4' } + - { env: { os: macos-14, xcode: 15.4 }, ruby_ver: '3.3.4', package_ruby_ver: '3.3.4' } steps: - name: Checkout tebako packaging environment uses: actions/checkout@v4 diff --git a/CMakeLists.txt b/CMakeLists.txt index e37bf445..f3c79d47 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -193,18 +193,18 @@ set(RUBY_STASH_DIR ${DEPS}/stash_${RUBY_VER}) string(SUBSTRING ${RUBY_VER} 0 3 RUBY_VER_BASE) string(CONCAT RUBY_API_VER ${RUBY_VER_BASE} ".0") -if(DWARFS_PRELOAD) - def_ext_prj_t(LIBDWARFS_WR "0.5.5" "7bf8e5b4432f35b65f6034f614067c2018995c1bebaf935e8cdddc1a3e045c01") - - string(REGEX REPLACE "^([0-9]+)\\.([0-9]+)\\.([0-9]+).*" "\\1;\\2;\\3" LIBDWARFS_WR_VER_COMPONENTS ${LIBDWARFS_WR_VER}) - - list(GET LIBDWARFS_WR_VER_COMPONENTS 0 LIBDWARFS_WR_VER_MAJOR) - list(GET LIBDWARFS_WR_VER_COMPONENTS 1 LIBDWARFS_WR_VER_MINOR) - list(GET LIBDWARFS_WR_VER_COMPONENTS 2 LIBDWARFS_WR_VER_PATCH) - set (LIBDWARFS_WR_VER_M ${LIBDWARFS_WR_VER_MAJOR}.${LIBDWARFS_WR_VER_MINOR}.${LIBDWARFS_WR_VER_PATCH}) -else(DWARFS_PRELOAD) - def_ext_prj_g(DWARFS_WR "v0.5.5") -endif(DWARFS_PRELOAD) +#if(DWARFS_PRELOAD) +# def_ext_prj_t(LIBDWARFS_WR "0.5.5" "7bf8e5b4432f35b65f6034f614067c2018995c1bebaf935e8cdddc1a3e045c01") +# +# string(REGEX REPLACE "^([0-9]+)\\.([0-9]+)\\.([0-9]+).*" "\\1;\\2;\\3" LIBDWARFS_WR_VER_COMPONENTS ${LIBDWARFS_WR_VER}) +# +# list(GET LIBDWARFS_WR_VER_COMPONENTS 0 LIBDWARFS_WR_VER_MAJOR) +# list(GET LIBDWARFS_WR_VER_COMPONENTS 1 LIBDWARFS_WR_VER_MINOR) +# list(GET LIBDWARFS_WR_VER_COMPONENTS 2 LIBDWARFS_WR_VER_PATCH) +# set (LIBDWARFS_WR_VER_M ${LIBDWARFS_WR_VER_MAJOR}.${LIBDWARFS_WR_VER_MINOR}.${LIBDWARFS_WR_VER_PATCH}) +#else(DWARFS_PRELOAD) +def_ext_prj_g(DWARFS_WR "v0.5.7") +#endif(DWARFS_PRELOAD) set(LIBYAML_RUBY_OPTION "") if(${RUBY_VER} VERSION_LESS "3.2.0") diff --git a/README.adoc b/README.adoc index 34740db1..38fde6a9 100644 --- a/README.adoc +++ b/README.adoc @@ -518,6 +518,40 @@ correct version. Run the following command prior to using Tebako, or add it into your shell profile. +====== jemalloc Library Build + +The `libdwarfs` build script creates an additional jemalloc installation on macOS. This is done to satisfy the magic applied by folly during linking but uses a static library. +If the library is created in an emulated environment (QEMU, Rosetta, etc.), there are known issues (link:https://github.com/jemalloc/jemalloc/issues/1997[jemalloc issue #1997]) where jemalloc incorrectly defines the number of significant virtual address bits (lg-vaddr parameter). + +These issues can be fixed by explicitly setting the `--with-lg-vaddr` parameter for the jemalloc build. We decided not to automate this since we do not feel that we can provide reasonable test coverage. Instead, our build script accepts the `LG_VADDR` environment variable and passes it to the jemalloc build as `--with-lg-vaddr=${LG_VADDR}`. + +The `LG_VADDR` parameter specifies the number of significant virtual address bits, which can vary based on the CPU architecture and emulation status. + +Simple script to set `LG_VADDR`. Please note that it is provided for illustration only. + +[source,sh] +---- +#!/bin/bash + +# Check the CPU architecture +ARCH=$(uname -m) + +# Check if running under Rosetta 2 emulation +if [[ "$ARCH" == "x86_64" && $(sysctl -n sysctl.proc_translated) == "1" ]]; then + echo "Running on Apple Silicon under Rosetta 2 emulation" + export LG_VADDR=39 +elif [[ "$ARCH" == "arm64" ]]; then + echo "Running on Apple Silicon" + export LG_VADDR=39 +else + echo "Running on Intel Silicon" + export LG_VADDR=48 +fi + +echo "Setting lg-vaddr to $LG_VADDR" +---- + + [source,sh] ---- export PATH="$(brew --prefix bison)/bin:$PATH" diff --git a/common.env b/common.env index 5e3119a4..de40404d 100644 --- a/common.env +++ b/common.env @@ -1,5 +1,5 @@ BUILD_TYPE=Release DEPS=deps INCBIN_TAG=348e36b -DWARFS_WR_TAG=v0.5.5 +DWARFS_WR_TAG=v0.5.7 RUBY_VER=3.1.6 diff --git a/lib/tebako/packager/patch_libraries.rb b/lib/tebako/packager/patch_libraries.rb index 908bee11..a9ca0c79 100644 --- a/lib/tebako/packager/patch_libraries.rb +++ b/lib/tebako/packager/patch_libraries.rb @@ -34,13 +34,10 @@ module Packager module PatchLibraries class << self # rubocop:disable Style/WordArray - - # NOTE: folly provides build-in implementation of jemalloc - DARWIN_BREW_LIBS = [ - ["zlib", "z"], ["gdbm", "gdbm"], ["readline", "readline"], ["libffi", "ffi"], - ["ncurses", "ncurses"], ["fmt", "fmt"], ["lz4", "lz4"], ["xz", "lzma"], - ["libyaml", "yaml"], ["boost", "boost_chrono"], ["double-conversion", "double-conversion"] + ["zlib", "z"], ["gdbm", "gdbm"], ["readline", "readline"], ["libffi", "ffi"], + ["ncurses", "ncurses"], ["fmt", "fmt"], ["lz4", "lz4"], ["xz", "lzma"], + ["libyaml", "yaml"], ["boost", "boost_chrono"], ["double-conversion", "double-conversion"] ].freeze DARWIN_BREW_LIBS_PRE_31 = [["openssl@1.1", "ssl"], ["openssl@1.1", "crypto"]].freeze diff --git a/lib/tebako/version.rb b/lib/tebako/version.rb index 98589c07..b3deffa1 100644 --- a/lib/tebako/version.rb +++ b/lib/tebako/version.rb @@ -26,5 +26,5 @@ # POSSIBILITY OF SUCH DAMAGE. module Tebako - VERSION = "0.8.0" + VERSION = "0.8.1" end From 616f408ac4fccdf8956320642ec4a6e5ecd9f028 Mon Sep 17 00:00:00 2001 From: Maxim Samsonov Date: Thu, 11 Jul 2024 16:16:57 +0300 Subject: [PATCH 2/9] chore: disable -test- extensions during packaging --- lib/tebako/cli_rubies.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/tebako/cli_rubies.rb b/lib/tebako/cli_rubies.rb index 76e9e928..71896b7b 100644 --- a/lib/tebako/cli_rubies.rb +++ b/lib/tebako/cli_rubies.rb @@ -41,9 +41,9 @@ module CliRubies "2.7.8" => "c2dab63cbc8f2a05526108ad419efa63a67ed4074dbbcf9fc2b1ca664cb45ba0", "3.0.7" => "2a3411977f2850431136b0fab8ad53af09fb74df2ee2f4fb7f11b378fe034388", "3.1.6" => "0d0dafb859e76763432571a3109d1537d976266be3083445651dc68deed25c22", + "3.2.3" => "af7f1757d9ddb630345988139211f1fd570ff5ba830def1cc7c468ae9b65c9ba", "3.2.4" => "c72b3c5c30482dca18b0f868c9075f3f47d8168eaf626d4e682ce5b59c858692", - "3.3.3" => "83c05b2177ee9c335b631b29b8c077b4770166d02fa527f3a9f6a40d13f3cce2", - "3.3.4" => "fe6a30f97d54e029768f2ddf4923699c416cdbc3a6e96db3e2d5716c7db96a34" + "3.3.3" => "83c05b2177ee9c335b631b29b8c077b4770166d02fa527f3a9f6a40d13f3cce2" }.freeze DEFAULT_RUBY_VERSION = "3.1.6" From 0e6ac49d2e5818c737fbd1f0c9cd1e319ee96ea7 Mon Sep 17 00:00:00 2001 From: "Maxim [maxirmx] Samsonov" Date: Mon, 15 Jul 2024 12:14:41 +0300 Subject: [PATCH 3/9] chore: drop support for Ruby 3.1.4, 3.1.5, 3.2.3 --- lib/tebako/cli_rubies.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/tebako/cli_rubies.rb b/lib/tebako/cli_rubies.rb index 71896b7b..d9ee91ab 100644 --- a/lib/tebako/cli_rubies.rb +++ b/lib/tebako/cli_rubies.rb @@ -41,7 +41,6 @@ module CliRubies "2.7.8" => "c2dab63cbc8f2a05526108ad419efa63a67ed4074dbbcf9fc2b1ca664cb45ba0", "3.0.7" => "2a3411977f2850431136b0fab8ad53af09fb74df2ee2f4fb7f11b378fe034388", "3.1.6" => "0d0dafb859e76763432571a3109d1537d976266be3083445651dc68deed25c22", - "3.2.3" => "af7f1757d9ddb630345988139211f1fd570ff5ba830def1cc7c468ae9b65c9ba", "3.2.4" => "c72b3c5c30482dca18b0f868c9075f3f47d8168eaf626d4e682ce5b59c858692", "3.3.3" => "83c05b2177ee9c335b631b29b8c077b4770166d02fa527f3a9f6a40d13f3cce2" }.freeze From 133b9d4980112455acdc3cb887fb170a880da8f8 Mon Sep 17 00:00:00 2001 From: "Maxim [maxirmx] Samsonov" Date: Mon, 15 Jul 2024 00:54:27 +0300 Subject: [PATCH 4/9] feat: Ruby 3.3 on Linux --- lib/tebako/cli_rubies.rb | 3 +- lib/tebako/packager/pass2.rb | 4 +- lib/tebako/packager/patch_buildsystem.rb | 31 +++---------- lib/tebako/packager/patch_libraries.rb | 55 ++++++++++++++---------- 4 files changed, 42 insertions(+), 51 deletions(-) diff --git a/lib/tebako/cli_rubies.rb b/lib/tebako/cli_rubies.rb index d9ee91ab..76e9e928 100644 --- a/lib/tebako/cli_rubies.rb +++ b/lib/tebako/cli_rubies.rb @@ -42,7 +42,8 @@ module CliRubies "3.0.7" => "2a3411977f2850431136b0fab8ad53af09fb74df2ee2f4fb7f11b378fe034388", "3.1.6" => "0d0dafb859e76763432571a3109d1537d976266be3083445651dc68deed25c22", "3.2.4" => "c72b3c5c30482dca18b0f868c9075f3f47d8168eaf626d4e682ce5b59c858692", - "3.3.3" => "83c05b2177ee9c335b631b29b8c077b4770166d02fa527f3a9f6a40d13f3cce2" + "3.3.3" => "83c05b2177ee9c335b631b29b8c077b4770166d02fa527f3a9f6a40d13f3cce2", + "3.3.4" => "fe6a30f97d54e029768f2ddf4923699c416cdbc3a6e96db3e2d5716c7db96a34" }.freeze DEFAULT_RUBY_VERSION = "3.1.6" diff --git a/lib/tebako/packager/pass2.rb b/lib/tebako/packager/pass2.rb index 80655527..da71e3ee 100755 --- a/lib/tebako/packager/pass2.rb +++ b/lib/tebako/packager/pass2.rb @@ -52,7 +52,7 @@ def get_patch_map(ostype, deps_lib_dir, ruby_ver) include Tebako::Packager::PatchBuildsystem include Tebako::Packager::PatchLiterals def extend_patch_map_r33(patch_map, ostype, deps_lib_dir, ruby_ver) - if PatchHelpers.ruby33?(ruby_ver) || PatchHelpers.msys?(ostype) + if PatchHelpers.ruby33?(ruby_ver) patch_map.store("config.status", get_config_status_patch(ostype, deps_lib_dir, ruby_ver)) end @@ -133,7 +133,7 @@ def mlibs_subst(ostype, deps_lib_dir, ruby_ver) { "MAINLIBS = #{yjit_libs}@MAINLIBS@" => "# -- Start of tebako patch -- \n" \ - "MAINLIBS = #{yjit_libs}#{PatchLibraries.mlibs(ostype, deps_lib_dir, ruby_ver, true)}\n" \ + "MAINLIBS = #{yjit_libs}#{PatchLibraries.mlibs(ostype, deps_lib_dir, ruby_ver, true)}" \ "# -- End of tebako patch -- \n" } end diff --git a/lib/tebako/packager/patch_buildsystem.rb b/lib/tebako/packager/patch_buildsystem.rb index 5004b93b..f8197296 100644 --- a/lib/tebako/packager/patch_buildsystem.rb +++ b/lib/tebako/packager/patch_buildsystem.rb @@ -79,21 +79,13 @@ module PatchBuildsystem def template_makefile_in_subst(ostype, ruby_ver) if PatchHelpers.msys?(ostype) - TEMPLATE_MAKEFILE_IN_BASE_PATCH_MSYS + { TEMPLATE_MAKEFILE_IN_BASE_PATTERN => TEMPLATE_MAKEFILE_IN_BASE_PATCH_MSYS } elsif !PatchHelpers.ruby31?(ruby_ver) - TEMPLATE_MAKEFILE_IN_BASE_PATCH_PRE_3_1 - else - TEMPLATE_MAKEFILE_IN_BASE_PATCH - end - end - - def template_makefile_in_patch_two(ostype, ruby_ver) - if !PatchHelpers.ruby31?(ruby_ver) - { TEMPLATE_MAKEFILE_IN_BASE_PATTERN_PRE_3_1 => template_makefile_in_subst(ostype, ruby_ver) } + { TEMPLATE_MAKEFILE_IN_BASE_PATTERN_PRE_3_1 => TEMPLATE_MAKEFILE_IN_BASE_PATCH_PRE_3_1 } elsif !PatchHelpers.ruby33?(ruby_ver) - { TEMPLATE_MAKEFILE_IN_BASE_PATTERN_PRE_3_3 => template_makefile_in_subst(ostype, ruby_ver) } + { TEMPLATE_MAKEFILE_IN_BASE_PATTERN_PRE_3_3 => TEMPLATE_MAKEFILE_IN_BASE_PATCH } else - { TEMPLATE_MAKEFILE_IN_BASE_PATTERN => template_makefile_in_subst(ostype, ruby_ver) } + { TEMPLATE_MAKEFILE_IN_BASE_PATTERN => TEMPLATE_MAKEFILE_IN_BASE_PATCH } end end @@ -108,22 +100,9 @@ def template_makefile_in_patch_two(ostype, ruby_ver) # End of tebako patch SUBST - def get_config_status_pattern(ostype) - case ostype - when /linux-/ - "S[\"MAINLIBS\"]=\"-lz -lrt -lrt -ldl -lcrypt -lm -lpthread \"" - when /darwin/ - "S[\"MAINLIBS\"]=\"-ldl -lobjc -lpthread \"" - when /msys/ - "S[\"MAINLIBS\"]=\"-lshell32 -lws2_32 -liphlpapi -limagehlp -lshlwapi -lbcrypt \"" - else - raise Tebako::Error, "Unknown ostype #{ostype}" - end - end - def get_config_status_patch(ostype, deps_lib_dir, ruby_ver) { - get_config_status_pattern(ostype) => + "S[\"MAINLIBS\"]=\"-lz -lrt -lrt -ldl -lcrypt -lm -lpthread \"" => "S[\"MAINLIBS\"]=\"#{PatchLibraries.mlibs(ostype, deps_lib_dir, ruby_ver, false)}\"" } end diff --git a/lib/tebako/packager/patch_libraries.rb b/lib/tebako/packager/patch_libraries.rb index a9ca0c79..37142373 100644 --- a/lib/tebako/packager/patch_libraries.rb +++ b/lib/tebako/packager/patch_libraries.rb @@ -32,7 +32,7 @@ module Tebako module Packager # Ruby patching definitions (pass2) module PatchLibraries - class << self + class << self # rubocop:disable Metrics/ClassLength # rubocop:disable Style/WordArray DARWIN_BREW_LIBS = [ ["zlib", "z"], ["gdbm", "gdbm"], ["readline", "readline"], ["libffi", "ffi"], @@ -49,7 +49,7 @@ class << self # rubocop:enable Style/WordArray COMMON_LINUX_LIBRARIES = [ - "-l:libdwarfs-wr.a", "-l:libtebako-fs.a", "-l:libdwarfs.a", "LIBCOMPRESSION", + "-l:libtebako-fs.a", "-l:libdwarfs-wr.a", "-l:libdwarfs.a", "LIBCOMPRESSION", "-l:libfolly.a", "-l:libfsst.a", "-l:libmetadata_thrift.a", "-l:libthrift_light.a", "-l:libxxhash.a", "-l:libfmt.a", "-l:libdouble-conversion.a", "-l:libglog.a", "-l:libgflags.a", "-l:libevent.a" @@ -77,17 +77,6 @@ class << self " -l:libunwind.a", "-l:liblzma.a", "-ldl", "-lpthread" ].freeze - MSYS_LIBRARIES = [ - "-l:liblz4.a", "-l:libz.a", "-l:libzstd.a", "-l:liblzma.a", - "-l:libncurses.a", "-l:libunwind.a", "-l:liblzma.a", "-l:libiberty.a", - "LIBYAML", "-l:libffi.a", "-l:libboost_system-mt.a", "-l:libboost_chrono-mt.a", - "-l:libstdc++.a", "-l:libdl.a", "-static-libgcc", "-static-libstdc++", - "-l:libssl.a", "-l:libcrypto.a", "-l:libz.a", "-l:libwinpthread.a", - "-lcrypt32", "-lshlwapi", "-lwsock32", "-liphlpapi", - "-limagehlp", "-lbcrypt", "-lole32", "-loleaut32", - "-luuid", "-lws2_32" - ].freeze - def linux_gnu_libraries(ruby_ver, with_compression) libraries = COMMON_LINUX_LIBRARIES + COMMON_ARCHIEVE_LIBRARIES + LINUX_GNU_LIBRARIES linux_libraries(libraries, ruby_ver, with_compression) @@ -111,12 +100,6 @@ def linux_libraries(libraries, ruby_ver, with_compression) libraries.join(" ") end - def msys_libraries(ruby_ver, with_compression) - libraries = with_compression ? ["-Wl,-Bstatic"] : [] - libraries = libraries + COMMON_LINUX_LIBRARIES + MSYS_LIBRARIES - linux_libraries(libraries, ruby_ver, with_compression) - end - def process_brew_libs!(libs, brew_libs) brew_libs.each { |lib| libs << "#{PatchHelpers.get_prefix_macos(lib[0]).chop}/lib/lib#{lib[1]}.a " } end @@ -146,12 +129,40 @@ def darwin_libraries(deps_lib_dir, ruby_ver, with_compression) # This is fixed by ext/extmk.rb patch [TODO ?] # ..................................................... - def mlibs(ostype, deps_lib_dir, ruby_ver, with_compression) # rubocop:disable Metrics/MethodLength + def linux_common_libs + <<~SUBST + -l:libtebako-fs.a -l:libdwarfs-wr.a -l:libdwarfs.a -Wl,--push-state,--whole-archive -l:libdwarfs_compression.a -Wl,--pop-state -l:libfolly.a -l:libfsst.a \\ + -l:libmetadata_thrift.a -l:libthrift_light.a -l:libxxhash.a -l:libfmt.a -l:libdouble-conversion.a -l:libglog.a -l:libgflags.a -l:libevent.a \\ + SUBST + end + + # Used for mkconfig.rb + def msys_base_libs(ruby_ver) + <<~SUBST + "-l:libtebako-fs.a -l:libdwarfs-wr.a -l:libdwarfs.a -l:libdwarfs_compression.a -l:libfolly.a -l:libfsst.a " \\ + "-l:libmetadata_thrift.a -l:libthrift_light.a -l:libxxhash.a -l:libfmt.a -l:libdouble-conversion.a -l:libglog.a -l:libgflags.a -l:libevent.a " \\ + "-l:liblz4.a -l:libz.a -l:libzstd.a -l:liblzma.a -l:libncurses.a -l:libunwind.a -l:liblzma.a -l:libiberty.a #{PatchHelpers.yaml_reference(ruby_ver)} " \\ + "-l:libffi.a -l:libboost_system-mt.a -l:libboost_chrono-mt.a -l:libstdc++.a -l:libdl.a -static-libgcc -static-libstdc++ -l:libssl.a -l:libcrypto.a " \\ + "-l:libz.a -l:libwinpthread.a -lcrypt32 -lshlwapi -lwsock32 -liphlpapi -limagehlp -lbcrypt -lole32 -loleaut32 -luuid" + SUBST + end + + # Used in Makefile + def msys_libs(ruby_ver) + <<~SUBST + -Wl,-Bstatic #{linux_common_libs} \ + -l:liblz4.a -l:libz.a -l:libzstd.a -l:liblzma.a -l:libncurses.a -l:libunwind.a -l:liblzma.a -l:libiberty.a #{PatchHelpers.yaml_reference(ruby_ver)} \\ + -l:libffi.a -l:libboost_system-mt.a -l:libboost_chrono-mt.a -l:libstdc++.a -l:libdl.a -static-libgcc -static-libstdc++ -l:libssl.a -l:libcrypto.a -l:libz.a \\ + -l:libwinpthread.a -lcrypt32 -lshlwapi -lwsock32 -liphlpapi -limagehlp -lshlwapi -lbcrypt -lws2_32 -lole32 -loleaut32 -luuid + SUBST + end + + def mlibs(ostype, deps_lib_dir, ruby_ver, unquoted) # rubocop:disable Metrics/MethodLength case ostype when /linux-gnu/ - linux_gnu_libraries(ruby_ver, with_compression) + linux_gnu_libraries(ruby_ver, unquoted) when /linux-musl/ - linux_musl_libraries(ruby_ver, with_compression) + linux_musl_libraries(ruby_ver, unquoted) when /darwin/ darwin_libraries(deps_lib_dir, ruby_ver, with_compression) when /msys/ From 7148bec5b074a2a5a532beefa6b3fc497f23614d Mon Sep 17 00:00:00 2001 From: maxirmx Date: Mon, 15 Jul 2024 22:33:39 +0300 Subject: [PATCH 5/9] feat: Ruby 3.3 on MacOS --- lib/tebako/packager/pass2.rb | 1 + lib/tebako/packager/patch_buildsystem.rb | 14 +++++++++++++- lib/tebako/packager/patch_libraries.rb | 6 +++--- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/tebako/packager/pass2.rb b/lib/tebako/packager/pass2.rb index da71e3ee..31f587fd 100755 --- a/lib/tebako/packager/pass2.rb +++ b/lib/tebako/packager/pass2.rb @@ -53,6 +53,7 @@ def get_patch_map(ostype, deps_lib_dir, ruby_ver) include Tebako::Packager::PatchLiterals def extend_patch_map_r33(patch_map, ostype, deps_lib_dir, ruby_ver) if PatchHelpers.ruby33?(ruby_ver) + puts get_config_status_patch(ostype, deps_lib_dir, ruby_ver) patch_map.store("config.status", get_config_status_patch(ostype, deps_lib_dir, ruby_ver)) end diff --git a/lib/tebako/packager/patch_buildsystem.rb b/lib/tebako/packager/patch_buildsystem.rb index f8197296..85af4812 100644 --- a/lib/tebako/packager/patch_buildsystem.rb +++ b/lib/tebako/packager/patch_buildsystem.rb @@ -100,9 +100,21 @@ def template_makefile_in_subst(ostype, ruby_ver) # End of tebako patch SUBST + def get_config_status_pattern(ostype) + case ostype + when /linux-/ + "S[\"MAINLIBS\"]=\"-lz -lrt -lrt -ldl -lcrypt -lm -lpthread \"" + when /darwin/ + "S[\"MAINLIBS\"]=\"-ldl -lobjc -lpthread \"" + # when /msys/ + else + raise Tebako::Error, "Unknown ostype #{ostype}" + end + end + def get_config_status_patch(ostype, deps_lib_dir, ruby_ver) { - "S[\"MAINLIBS\"]=\"-lz -lrt -lrt -ldl -lcrypt -lm -lpthread \"" => + get_config_status_pattern(ostype) => "S[\"MAINLIBS\"]=\"#{PatchLibraries.mlibs(ostype, deps_lib_dir, ruby_ver, false)}\"" } end diff --git a/lib/tebako/packager/patch_libraries.rb b/lib/tebako/packager/patch_libraries.rb index 37142373..fe1098c2 100644 --- a/lib/tebako/packager/patch_libraries.rb +++ b/lib/tebako/packager/patch_libraries.rb @@ -157,12 +157,12 @@ def msys_libs(ruby_ver) SUBST end - def mlibs(ostype, deps_lib_dir, ruby_ver, unquoted) # rubocop:disable Metrics/MethodLength + def mlibs(ostype, deps_lib_dir, ruby_ver, with_compression) # rubocop:disable Metrics/MethodLength case ostype when /linux-gnu/ - linux_gnu_libraries(ruby_ver, unquoted) + linux_gnu_libraries(ruby_ver, with_compression) when /linux-musl/ - linux_musl_libraries(ruby_ver, unquoted) + linux_musl_libraries(ruby_ver, with_compression) when /darwin/ darwin_libraries(deps_lib_dir, ruby_ver, with_compression) when /msys/ From 113d8665c4745f387ddaa01bddb69038c62aeddf Mon Sep 17 00:00:00 2001 From: Maxim Samsonov Date: Sat, 20 Jul 2024 21:05:47 +0300 Subject: [PATCH 6/9] feat: Ruby 3.3 on Windows/MSys --- lib/tebako/packager/pass2.rb | 5 +-- lib/tebako/packager/patch_buildsystem.rb | 19 ++++++--- lib/tebako/packager/patch_libraries.rb | 49 +++++++++--------------- 3 files changed, 35 insertions(+), 38 deletions(-) diff --git a/lib/tebako/packager/pass2.rb b/lib/tebako/packager/pass2.rb index 31f587fd..80655527 100755 --- a/lib/tebako/packager/pass2.rb +++ b/lib/tebako/packager/pass2.rb @@ -52,8 +52,7 @@ def get_patch_map(ostype, deps_lib_dir, ruby_ver) include Tebako::Packager::PatchBuildsystem include Tebako::Packager::PatchLiterals def extend_patch_map_r33(patch_map, ostype, deps_lib_dir, ruby_ver) - if PatchHelpers.ruby33?(ruby_ver) - puts get_config_status_patch(ostype, deps_lib_dir, ruby_ver) + if PatchHelpers.ruby33?(ruby_ver) || PatchHelpers.msys?(ostype) patch_map.store("config.status", get_config_status_patch(ostype, deps_lib_dir, ruby_ver)) end @@ -134,7 +133,7 @@ def mlibs_subst(ostype, deps_lib_dir, ruby_ver) { "MAINLIBS = #{yjit_libs}@MAINLIBS@" => "# -- Start of tebako patch -- \n" \ - "MAINLIBS = #{yjit_libs}#{PatchLibraries.mlibs(ostype, deps_lib_dir, ruby_ver, true)}" \ + "MAINLIBS = #{yjit_libs}#{PatchLibraries.mlibs(ostype, deps_lib_dir, ruby_ver, true)}\n" \ "# -- End of tebako patch -- \n" } end diff --git a/lib/tebako/packager/patch_buildsystem.rb b/lib/tebako/packager/patch_buildsystem.rb index 85af4812..5004b93b 100644 --- a/lib/tebako/packager/patch_buildsystem.rb +++ b/lib/tebako/packager/patch_buildsystem.rb @@ -79,13 +79,21 @@ module PatchBuildsystem def template_makefile_in_subst(ostype, ruby_ver) if PatchHelpers.msys?(ostype) - { TEMPLATE_MAKEFILE_IN_BASE_PATTERN => TEMPLATE_MAKEFILE_IN_BASE_PATCH_MSYS } + TEMPLATE_MAKEFILE_IN_BASE_PATCH_MSYS elsif !PatchHelpers.ruby31?(ruby_ver) - { TEMPLATE_MAKEFILE_IN_BASE_PATTERN_PRE_3_1 => TEMPLATE_MAKEFILE_IN_BASE_PATCH_PRE_3_1 } + TEMPLATE_MAKEFILE_IN_BASE_PATCH_PRE_3_1 + else + TEMPLATE_MAKEFILE_IN_BASE_PATCH + end + end + + def template_makefile_in_patch_two(ostype, ruby_ver) + if !PatchHelpers.ruby31?(ruby_ver) + { TEMPLATE_MAKEFILE_IN_BASE_PATTERN_PRE_3_1 => template_makefile_in_subst(ostype, ruby_ver) } elsif !PatchHelpers.ruby33?(ruby_ver) - { TEMPLATE_MAKEFILE_IN_BASE_PATTERN_PRE_3_3 => TEMPLATE_MAKEFILE_IN_BASE_PATCH } + { TEMPLATE_MAKEFILE_IN_BASE_PATTERN_PRE_3_3 => template_makefile_in_subst(ostype, ruby_ver) } else - { TEMPLATE_MAKEFILE_IN_BASE_PATTERN => TEMPLATE_MAKEFILE_IN_BASE_PATCH } + { TEMPLATE_MAKEFILE_IN_BASE_PATTERN => template_makefile_in_subst(ostype, ruby_ver) } end end @@ -106,7 +114,8 @@ def get_config_status_pattern(ostype) "S[\"MAINLIBS\"]=\"-lz -lrt -lrt -ldl -lcrypt -lm -lpthread \"" when /darwin/ "S[\"MAINLIBS\"]=\"-ldl -lobjc -lpthread \"" - # when /msys/ + when /msys/ + "S[\"MAINLIBS\"]=\"-lshell32 -lws2_32 -liphlpapi -limagehlp -lshlwapi -lbcrypt \"" else raise Tebako::Error, "Unknown ostype #{ostype}" end diff --git a/lib/tebako/packager/patch_libraries.rb b/lib/tebako/packager/patch_libraries.rb index fe1098c2..a9ca0c79 100644 --- a/lib/tebako/packager/patch_libraries.rb +++ b/lib/tebako/packager/patch_libraries.rb @@ -32,7 +32,7 @@ module Tebako module Packager # Ruby patching definitions (pass2) module PatchLibraries - class << self # rubocop:disable Metrics/ClassLength + class << self # rubocop:disable Style/WordArray DARWIN_BREW_LIBS = [ ["zlib", "z"], ["gdbm", "gdbm"], ["readline", "readline"], ["libffi", "ffi"], @@ -49,7 +49,7 @@ class << self # rubocop:disable Metrics/ClassLength # rubocop:enable Style/WordArray COMMON_LINUX_LIBRARIES = [ - "-l:libtebako-fs.a", "-l:libdwarfs-wr.a", "-l:libdwarfs.a", "LIBCOMPRESSION", + "-l:libdwarfs-wr.a", "-l:libtebako-fs.a", "-l:libdwarfs.a", "LIBCOMPRESSION", "-l:libfolly.a", "-l:libfsst.a", "-l:libmetadata_thrift.a", "-l:libthrift_light.a", "-l:libxxhash.a", "-l:libfmt.a", "-l:libdouble-conversion.a", "-l:libglog.a", "-l:libgflags.a", "-l:libevent.a" @@ -77,6 +77,17 @@ class << self # rubocop:disable Metrics/ClassLength " -l:libunwind.a", "-l:liblzma.a", "-ldl", "-lpthread" ].freeze + MSYS_LIBRARIES = [ + "-l:liblz4.a", "-l:libz.a", "-l:libzstd.a", "-l:liblzma.a", + "-l:libncurses.a", "-l:libunwind.a", "-l:liblzma.a", "-l:libiberty.a", + "LIBYAML", "-l:libffi.a", "-l:libboost_system-mt.a", "-l:libboost_chrono-mt.a", + "-l:libstdc++.a", "-l:libdl.a", "-static-libgcc", "-static-libstdc++", + "-l:libssl.a", "-l:libcrypto.a", "-l:libz.a", "-l:libwinpthread.a", + "-lcrypt32", "-lshlwapi", "-lwsock32", "-liphlpapi", + "-limagehlp", "-lbcrypt", "-lole32", "-loleaut32", + "-luuid", "-lws2_32" + ].freeze + def linux_gnu_libraries(ruby_ver, with_compression) libraries = COMMON_LINUX_LIBRARIES + COMMON_ARCHIEVE_LIBRARIES + LINUX_GNU_LIBRARIES linux_libraries(libraries, ruby_ver, with_compression) @@ -100,6 +111,12 @@ def linux_libraries(libraries, ruby_ver, with_compression) libraries.join(" ") end + def msys_libraries(ruby_ver, with_compression) + libraries = with_compression ? ["-Wl,-Bstatic"] : [] + libraries = libraries + COMMON_LINUX_LIBRARIES + MSYS_LIBRARIES + linux_libraries(libraries, ruby_ver, with_compression) + end + def process_brew_libs!(libs, brew_libs) brew_libs.each { |lib| libs << "#{PatchHelpers.get_prefix_macos(lib[0]).chop}/lib/lib#{lib[1]}.a " } end @@ -129,34 +146,6 @@ def darwin_libraries(deps_lib_dir, ruby_ver, with_compression) # This is fixed by ext/extmk.rb patch [TODO ?] # ..................................................... - def linux_common_libs - <<~SUBST - -l:libtebako-fs.a -l:libdwarfs-wr.a -l:libdwarfs.a -Wl,--push-state,--whole-archive -l:libdwarfs_compression.a -Wl,--pop-state -l:libfolly.a -l:libfsst.a \\ - -l:libmetadata_thrift.a -l:libthrift_light.a -l:libxxhash.a -l:libfmt.a -l:libdouble-conversion.a -l:libglog.a -l:libgflags.a -l:libevent.a \\ - SUBST - end - - # Used for mkconfig.rb - def msys_base_libs(ruby_ver) - <<~SUBST - "-l:libtebako-fs.a -l:libdwarfs-wr.a -l:libdwarfs.a -l:libdwarfs_compression.a -l:libfolly.a -l:libfsst.a " \\ - "-l:libmetadata_thrift.a -l:libthrift_light.a -l:libxxhash.a -l:libfmt.a -l:libdouble-conversion.a -l:libglog.a -l:libgflags.a -l:libevent.a " \\ - "-l:liblz4.a -l:libz.a -l:libzstd.a -l:liblzma.a -l:libncurses.a -l:libunwind.a -l:liblzma.a -l:libiberty.a #{PatchHelpers.yaml_reference(ruby_ver)} " \\ - "-l:libffi.a -l:libboost_system-mt.a -l:libboost_chrono-mt.a -l:libstdc++.a -l:libdl.a -static-libgcc -static-libstdc++ -l:libssl.a -l:libcrypto.a " \\ - "-l:libz.a -l:libwinpthread.a -lcrypt32 -lshlwapi -lwsock32 -liphlpapi -limagehlp -lbcrypt -lole32 -loleaut32 -luuid" - SUBST - end - - # Used in Makefile - def msys_libs(ruby_ver) - <<~SUBST - -Wl,-Bstatic #{linux_common_libs} \ - -l:liblz4.a -l:libz.a -l:libzstd.a -l:liblzma.a -l:libncurses.a -l:libunwind.a -l:liblzma.a -l:libiberty.a #{PatchHelpers.yaml_reference(ruby_ver)} \\ - -l:libffi.a -l:libboost_system-mt.a -l:libboost_chrono-mt.a -l:libstdc++.a -l:libdl.a -static-libgcc -static-libstdc++ -l:libssl.a -l:libcrypto.a -l:libz.a \\ - -l:libwinpthread.a -lcrypt32 -lshlwapi -lwsock32 -liphlpapi -limagehlp -lshlwapi -lbcrypt -lws2_32 -lole32 -loleaut32 -luuid - SUBST - end - def mlibs(ostype, deps_lib_dir, ruby_ver, with_compression) # rubocop:disable Metrics/MethodLength case ostype when /linux-gnu/ From 88938914ccc86afb0358ba731c37b8616f22e239 Mon Sep 17 00:00:00 2001 From: "Maxim [maxirmx] Samsonov" Date: Fri, 26 Jul 2024 23:52:04 +0300 Subject: [PATCH 7/9] Branch setup --- lib/tebako/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tebako/version.rb b/lib/tebako/version.rb index b3deffa1..1d4e99e4 100644 --- a/lib/tebako/version.rb +++ b/lib/tebako/version.rb @@ -26,5 +26,5 @@ # POSSIBILITY OF SUCH DAMAGE. module Tebako - VERSION = "0.8.1" + VERSION = "0.9.0" end From 27a5326bfc35bc55cc8b671f3b1ed0d354d0e28a Mon Sep 17 00:00:00 2001 From: "Maxim [maxirmx] Samsonov" Date: Mon, 29 Jul 2024 15:54:59 +0300 Subject: [PATCH 8/9] Updated MacOs patch for bundle_loader --- .tebako.yml | 1 + lib/tebako/packager/pass1.rb | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.tebako.yml b/.tebako.yml index 00b19dff..3f29afac 100644 --- a/.tebako.yml +++ b/.tebako.yml @@ -1 +1,2 @@ prefix: PWD +# Ruby: 3.2.4 diff --git a/lib/tebako/packager/pass1.rb b/lib/tebako/packager/pass1.rb index 9f89dfe0..303d0e49 100755 --- a/lib/tebako/packager/pass1.rb +++ b/lib/tebako/packager/pass1.rb @@ -92,6 +92,10 @@ module Pass1 # "EXTDLDFLAGS=\"-bundle_loader '\\$(BUILTRUBY)'\"" => "" # }.freeze + DARWIN_CONFIGURE_PATCH = { + "elif test \"x$EXTSTATIC\" = x" => "elif true" + }.freeze + OPENSSL_EXTCONF_RB_SUBST = <<~SUBST # Start of tebako patch $defs.push("-DRUBY_EXPORT=1") @@ -137,11 +141,7 @@ def get_patch_map(ostype, mount_point, ruby_ver) patch_map = get_base_patch_map(mount_point) # .................................................... - # Fixing (bypassing) configure script bug where a variable is used before initialization - # On MacOS it generates bod EXTDLDFLAGS WITH -bundle_loader - # However, it loooks likes EXTDLDFLAGS are just redundant for final Ruby linkage, so we are - # just removing it on pass 2 (at least for now) - # patch_map.store("configure", DARWIN_CONFIGURE_PATCH) if ostype =~ /darwin/ + patch_map.store("configure", DARWIN_CONFIGURE_PATCH) if ostype =~ /darwin/ # .................................................... # autoload :OpenSSL, "openssl" From 6cc55b357ec830f2214e9b8545e90766ad70e6d6 Mon Sep 17 00:00:00 2001 From: "Maxim [maxirmx] Samsonov" Date: Sun, 4 Aug 2024 12:53:21 +0300 Subject: [PATCH 9/9] feat: minimal Tebako configuration file 'tebafile' --- .tebako.yml | 5 ++-- README.adoc | 69 ++++++++++++++++++++++++++++++++++----------- lib/tebako/cli.rb | 11 ++++---- src/tebako-main.cpp | 2 ++ 4 files changed, 64 insertions(+), 23 deletions(-) diff --git a/.tebako.yml b/.tebako.yml index 3f29afac..3f75d567 100644 --- a/.tebako.yml +++ b/.tebako.yml @@ -1,2 +1,3 @@ -prefix: PWD -# Ruby: 3.2.4 +options: + prefix: PWD + Ruby: 3.2.4 diff --git a/README.adoc b/README.adoc index 38fde6a9..cbead936 100644 --- a/README.adoc +++ b/README.adoc @@ -86,8 +86,8 @@ with debug information unstripped. You can opt to run 'strip -S' manually, it mo | 2.7.8 | Linux, macOS | 3.0.7 | Linux, macOS -| 3.1.{4,5,6} | Linux, macOS, Windows -| 3.2.{3,4} | Linux, macOS, Windows +| 3.1.6 | Linux, macOS, Windows +| 3.2.4 | Linux, macOS, Windows | 3.3.{3,4} | Linux, macOS, Windows |=== @@ -232,8 +232,6 @@ Displays the Tebako version. Displays the help message. - - == Usage === General @@ -291,9 +289,6 @@ docker run -v :/mnt/w \ tebako {command} {parameters} ---- - - - ==== Packaging from outside the container To package your application from outside the container, just run a single Docker @@ -659,7 +654,8 @@ tebako press \ [-R|--Ruby=] \ [-o|--output=] \ [-l|--log-level=] \ - [-D|--devmode] + [-D|--devmode] \ + [-t|--tebafile=] ---- Where: @@ -668,7 +664,7 @@ Where: the Tebako root folder (see details in the Tebako Root Folder Selection section) `Ruby`:: -This parameter defines Ruby version that will be packaged (optional, defaults to +this parameter defines Ruby version that will be packaged (optional, defaults to `3.1.6`) ``:: @@ -685,11 +681,16 @@ the output file name (optional, defaults to `/] \ [-R|--Ruby=] \ - [-D|--devmode] + [-D|--devmode] \ + [-t|--tebafile=] ---- Where: ``:: the Tebako root folder (see details in the Tebako Root Folder Selection section) -`Ruby` parameter defines Ruby version that will be packaged (optional, defaults to 3.1.6) +`Ruby`:: parameter defines Ruby version that will be packaged (optional, defaults to 3.1.6) + +`tebafile`:: +the tebako configuration file (optional, defaults to `$PWD/.tebako.yml`). +Please refer to the separate section below for tebafile description. -`devmode` flag activates development mode, in which Tebako's cache and packaging consistency checks are relaxed. +`devmode`:: flag activates development mode, in which Tebako's cache and packaging consistency checks are relaxed. Please note that this mode is not intended for production use and should only be used during development. ==== Clean @@ -746,13 +752,18 @@ Normally you do not need to do it since tebako packager optimizes artifacts life [source,sh] ---- $ tebako clean \ - [-p|--prefix=] + [-p|--prefix=] \ + [-t|--tebafile=] ---- Where: ``:: the Tebako root folder (see details in the Tebako Root Folder Selection section) +`tebafile`:: +the tebako configuration file (optional, defaults to `$PWD/.tebako.yml`). +Please refer to the separate section below for tebafile description. + [example] ==== [source,sh] @@ -777,7 +788,9 @@ NOTE: Compiled DwarFS libraries are not cleaned. ---- $ tebako clean_ruby [-p|--prefix=] \ - [-R|--Ruby=] + [-R|--Ruby=] \ + [-t|--tebafile=] + ---- Where: @@ -788,6 +801,10 @@ the Tebako setup folder (optional, defaults to current folder) `Ruby`:: defines Ruby version that will cleaned (optional, cleans all versions by default) +`tebafile`:: +the tebako configuration file (optional, defaults to `$PWD/.tebako.yml`). +Please refer to the separate section below for tebafile description. + [example] ==== [source,sh] @@ -806,6 +823,26 @@ as a cache key in CI/CD pipelines. $ tebako hash ---- +=== Tebako configuration file + +It is possible to provide all or some options for the `tebako setup/press/clean/clean_ruby` commands via Tebako configuration file ('tebafile'). +Tebafile is a YAML file with a single section 'options'. The options are the same as long names for the command line. For, example for the prefix option + +[source] +---- +-p|--prefix= +---- +the key in the YAML file would be 'prefix'. + +Below is an example tebafile that sets values for prefix and Ruby options +[source,yaml] +---- +options: + prefix: /tmp/tebako + Ruby: 3.2.4 +---- + +Please note that the options provided on the command line have preference over tebafile settings. === Image extraction diff --git a/lib/tebako/cli.rb b/lib/tebako/cli.rb index 47db2323..bfbf0b18 100755 --- a/lib/tebako/cli.rb +++ b/lib/tebako/cli.rb @@ -40,7 +40,7 @@ # Tebako - an executable packager # Implementation of tebako command-line interface module Tebako - OPTIONS_FILE = ".tebako.yml" + DEFAULT_TEBAFILE = ".tebako.yml" # Tebako packager front-end class Cli < Thor package_name "Tebako" @@ -48,7 +48,8 @@ class Cli < Thor desc: "A path to tebako packaging environment, '~/.tebako' ('$HOME/.tebako') by default" class_option :devmode, type: :boolean, aliases: "-D", required: false, desc: "Developer mode, please do not use if unsure" - + class_option :tebafile, type: :string, aliases: "-t", required: false, + desc: "tebako configuration file 'tebafile', '$PWD/.tebako.yml' by default" desc "clean", "Clean tebako packaging environment" def clean clean_cache @@ -123,10 +124,10 @@ def initialize(*args) def options original_options = super + tebafile = original_options["tebafile"].nil? ? DEFAULT_TEBAFILE : original_options["tebafile"] + return original_options unless File.exist?(tebafile) - return original_options unless File.exist?(OPTIONS_FILE) - - defaults = ::YAML.load_file(OPTIONS_FILE) || {} + defaults = ::YAML.load_file(tebafile)["options"] || {} Thor::CoreExt::HashWithIndifferentAccess.new(defaults.merge(original_options)) end end diff --git a/src/tebako-main.cpp b/src/tebako-main.cpp index 985577a4..293f7693 100644 --- a/src/tebako-main.cpp +++ b/src/tebako-main.cpp @@ -145,6 +145,8 @@ extern "C" int tebako_main(int* argc, char*** argv) { // Nested error, no recovery :( } } + + // tebako_chdir("/__tebako_memfs__/local"); } return ret; }