diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 4a5e0172..38c1667e 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -48,7 +48,7 @@ concurrency: cancel-in-progress: true env: - CACHE_VER: 02 + CACHE_VER: 03 VERBOSE: no jobs: @@ -60,9 +60,9 @@ jobs: matrix: env: - { os: macos-12, xcode: 13.1 } - - { os: macos-14, xcode: 14.3.1 } + - { os: macos-14, xcode: 14.3.1, LG_VADDR: 39 } - { os: macos-14, xcode: 15.4 } - ruby_ver: ['3.1.6', '3.3.3'] + ruby_ver: ['3.1.6', '3.3.4'] steps: - name: Checkout tebako packaging environment uses: actions/checkout@v4 @@ -99,11 +99,11 @@ jobs: matrix: env: - { os: macos-12, xcode: 13.1 } - - { os: macos-14, xcode: 14.3.1 } + - { os: macos-14, xcode: 14.3.1, LG_VADDR: 39} package_ruby_ver: [ '3.1.6', '3.2.4' ] ruby_ver: ['3.1.6'] include: - - { env: { os: macos-14, xcode: 15.4 }, ruby_ver: '3.3.3', package_ruby_ver: '3.2.4' } + - { env: { os: macos-14, xcode: 15.4 }, ruby_ver: '3.3.4', package_ruby_ver: '3.3.4' } steps: - name: Checkout tebako packaging environment uses: actions/checkout@v4 @@ -152,11 +152,11 @@ jobs: matrix: env: - { os: macos-12, xcode: 13.1 } - - { os: macos-14, xcode: 14.3.1 } + - { os: macos-14, xcode: 14.3.1, LG_VADDR: 39} package_ruby_ver: [ '3.1.6', '3.2.4' ] ruby_ver: ['3.1.6'] include: - - { env: { os: macos-14, xcode: 15.4 }, ruby_ver: '3.3.3', package_ruby_ver: '3.2.4' } + - { env: { os: macos-14, xcode: 15.4 }, ruby_ver: '3.3.4', package_ruby_ver: '3.3.4' } steps: - name: Checkout tebako packaging environment uses: actions/checkout@v4 diff --git a/.tebako.yml b/.tebako.yml index 00b19dff..3f75d567 100644 --- a/.tebako.yml +++ b/.tebako.yml @@ -1 +1,3 @@ -prefix: PWD +options: + prefix: PWD + Ruby: 3.2.4 diff --git a/CMakeLists.txt b/CMakeLists.txt index e37bf445..f3c79d47 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -193,18 +193,18 @@ set(RUBY_STASH_DIR ${DEPS}/stash_${RUBY_VER}) string(SUBSTRING ${RUBY_VER} 0 3 RUBY_VER_BASE) string(CONCAT RUBY_API_VER ${RUBY_VER_BASE} ".0") -if(DWARFS_PRELOAD) - def_ext_prj_t(LIBDWARFS_WR "0.5.5" "7bf8e5b4432f35b65f6034f614067c2018995c1bebaf935e8cdddc1a3e045c01") - - string(REGEX REPLACE "^([0-9]+)\\.([0-9]+)\\.([0-9]+).*" "\\1;\\2;\\3" LIBDWARFS_WR_VER_COMPONENTS ${LIBDWARFS_WR_VER}) - - list(GET LIBDWARFS_WR_VER_COMPONENTS 0 LIBDWARFS_WR_VER_MAJOR) - list(GET LIBDWARFS_WR_VER_COMPONENTS 1 LIBDWARFS_WR_VER_MINOR) - list(GET LIBDWARFS_WR_VER_COMPONENTS 2 LIBDWARFS_WR_VER_PATCH) - set (LIBDWARFS_WR_VER_M ${LIBDWARFS_WR_VER_MAJOR}.${LIBDWARFS_WR_VER_MINOR}.${LIBDWARFS_WR_VER_PATCH}) -else(DWARFS_PRELOAD) - def_ext_prj_g(DWARFS_WR "v0.5.5") -endif(DWARFS_PRELOAD) +#if(DWARFS_PRELOAD) +# def_ext_prj_t(LIBDWARFS_WR "0.5.5" "7bf8e5b4432f35b65f6034f614067c2018995c1bebaf935e8cdddc1a3e045c01") +# +# string(REGEX REPLACE "^([0-9]+)\\.([0-9]+)\\.([0-9]+).*" "\\1;\\2;\\3" LIBDWARFS_WR_VER_COMPONENTS ${LIBDWARFS_WR_VER}) +# +# list(GET LIBDWARFS_WR_VER_COMPONENTS 0 LIBDWARFS_WR_VER_MAJOR) +# list(GET LIBDWARFS_WR_VER_COMPONENTS 1 LIBDWARFS_WR_VER_MINOR) +# list(GET LIBDWARFS_WR_VER_COMPONENTS 2 LIBDWARFS_WR_VER_PATCH) +# set (LIBDWARFS_WR_VER_M ${LIBDWARFS_WR_VER_MAJOR}.${LIBDWARFS_WR_VER_MINOR}.${LIBDWARFS_WR_VER_PATCH}) +#else(DWARFS_PRELOAD) +def_ext_prj_g(DWARFS_WR "v0.5.7") +#endif(DWARFS_PRELOAD) set(LIBYAML_RUBY_OPTION "") if(${RUBY_VER} VERSION_LESS "3.2.0") diff --git a/README.adoc b/README.adoc index 34740db1..cbead936 100644 --- a/README.adoc +++ b/README.adoc @@ -86,8 +86,8 @@ with debug information unstripped. You can opt to run 'strip -S' manually, it mo | 2.7.8 | Linux, macOS | 3.0.7 | Linux, macOS -| 3.1.{4,5,6} | Linux, macOS, Windows -| 3.2.{3,4} | Linux, macOS, Windows +| 3.1.6 | Linux, macOS, Windows +| 3.2.4 | Linux, macOS, Windows | 3.3.{3,4} | Linux, macOS, Windows |=== @@ -232,8 +232,6 @@ Displays the Tebako version. Displays the help message. - - == Usage === General @@ -291,9 +289,6 @@ docker run -v :/mnt/w \ tebako {command} {parameters} ---- - - - ==== Packaging from outside the container To package your application from outside the container, just run a single Docker @@ -518,6 +513,40 @@ correct version. Run the following command prior to using Tebako, or add it into your shell profile. +====== jemalloc Library Build + +The `libdwarfs` build script creates an additional jemalloc installation on macOS. This is done to satisfy the magic applied by folly during linking but uses a static library. +If the library is created in an emulated environment (QEMU, Rosetta, etc.), there are known issues (link:https://github.com/jemalloc/jemalloc/issues/1997[jemalloc issue #1997]) where jemalloc incorrectly defines the number of significant virtual address bits (lg-vaddr parameter). + +These issues can be fixed by explicitly setting the `--with-lg-vaddr` parameter for the jemalloc build. We decided not to automate this since we do not feel that we can provide reasonable test coverage. Instead, our build script accepts the `LG_VADDR` environment variable and passes it to the jemalloc build as `--with-lg-vaddr=${LG_VADDR}`. + +The `LG_VADDR` parameter specifies the number of significant virtual address bits, which can vary based on the CPU architecture and emulation status. + +Simple script to set `LG_VADDR`. Please note that it is provided for illustration only. + +[source,sh] +---- +#!/bin/bash + +# Check the CPU architecture +ARCH=$(uname -m) + +# Check if running under Rosetta 2 emulation +if [[ "$ARCH" == "x86_64" && $(sysctl -n sysctl.proc_translated) == "1" ]]; then + echo "Running on Apple Silicon under Rosetta 2 emulation" + export LG_VADDR=39 +elif [[ "$ARCH" == "arm64" ]]; then + echo "Running on Apple Silicon" + export LG_VADDR=39 +else + echo "Running on Intel Silicon" + export LG_VADDR=48 +fi + +echo "Setting lg-vaddr to $LG_VADDR" +---- + + [source,sh] ---- export PATH="$(brew --prefix bison)/bin:$PATH" @@ -625,7 +654,8 @@ tebako press \ [-R|--Ruby=] \ [-o|--output=] \ [-l|--log-level=] \ - [-D|--devmode] + [-D|--devmode] \ + [-t|--tebafile=] ---- Where: @@ -634,7 +664,7 @@ Where: the Tebako root folder (see details in the Tebako Root Folder Selection section) `Ruby`:: -This parameter defines Ruby version that will be packaged (optional, defaults to +this parameter defines Ruby version that will be packaged (optional, defaults to `3.1.6`) ``:: @@ -651,11 +681,16 @@ the output file name (optional, defaults to `/] \ [-R|--Ruby=] \ - [-D|--devmode] + [-D|--devmode] \ + [-t|--tebafile=] ---- Where: ``:: the Tebako root folder (see details in the Tebako Root Folder Selection section) -`Ruby` parameter defines Ruby version that will be packaged (optional, defaults to 3.1.6) +`Ruby`:: parameter defines Ruby version that will be packaged (optional, defaults to 3.1.6) -`devmode` flag activates development mode, in which Tebako's cache and packaging consistency checks are relaxed. +`tebafile`:: +the tebako configuration file (optional, defaults to `$PWD/.tebako.yml`). +Please refer to the separate section below for tebafile description. + +`devmode`:: flag activates development mode, in which Tebako's cache and packaging consistency checks are relaxed. Please note that this mode is not intended for production use and should only be used during development. ==== Clean @@ -712,13 +752,18 @@ Normally you do not need to do it since tebako packager optimizes artifacts life [source,sh] ---- $ tebako clean \ - [-p|--prefix=] + [-p|--prefix=] \ + [-t|--tebafile=] ---- Where: ``:: the Tebako root folder (see details in the Tebako Root Folder Selection section) +`tebafile`:: +the tebako configuration file (optional, defaults to `$PWD/.tebako.yml`). +Please refer to the separate section below for tebafile description. + [example] ==== [source,sh] @@ -743,7 +788,9 @@ NOTE: Compiled DwarFS libraries are not cleaned. ---- $ tebako clean_ruby [-p|--prefix=] \ - [-R|--Ruby=] + [-R|--Ruby=] \ + [-t|--tebafile=] + ---- Where: @@ -754,6 +801,10 @@ the Tebako setup folder (optional, defaults to current folder) `Ruby`:: defines Ruby version that will cleaned (optional, cleans all versions by default) +`tebafile`:: +the tebako configuration file (optional, defaults to `$PWD/.tebako.yml`). +Please refer to the separate section below for tebafile description. + [example] ==== [source,sh] @@ -772,6 +823,26 @@ as a cache key in CI/CD pipelines. $ tebako hash ---- +=== Tebako configuration file + +It is possible to provide all or some options for the `tebako setup/press/clean/clean_ruby` commands via Tebako configuration file ('tebafile'). +Tebafile is a YAML file with a single section 'options'. The options are the same as long names for the command line. For, example for the prefix option + +[source] +---- +-p|--prefix= +---- +the key in the YAML file would be 'prefix'. + +Below is an example tebafile that sets values for prefix and Ruby options +[source,yaml] +---- +options: + prefix: /tmp/tebako + Ruby: 3.2.4 +---- + +Please note that the options provided on the command line have preference over tebafile settings. === Image extraction diff --git a/common.env b/common.env index 5e3119a4..de40404d 100644 --- a/common.env +++ b/common.env @@ -1,5 +1,5 @@ BUILD_TYPE=Release DEPS=deps INCBIN_TAG=348e36b -DWARFS_WR_TAG=v0.5.5 +DWARFS_WR_TAG=v0.5.7 RUBY_VER=3.1.6 diff --git a/lib/tebako/cli.rb b/lib/tebako/cli.rb index 47db2323..bfbf0b18 100755 --- a/lib/tebako/cli.rb +++ b/lib/tebako/cli.rb @@ -40,7 +40,7 @@ # Tebako - an executable packager # Implementation of tebako command-line interface module Tebako - OPTIONS_FILE = ".tebako.yml" + DEFAULT_TEBAFILE = ".tebako.yml" # Tebako packager front-end class Cli < Thor package_name "Tebako" @@ -48,7 +48,8 @@ class Cli < Thor desc: "A path to tebako packaging environment, '~/.tebako' ('$HOME/.tebako') by default" class_option :devmode, type: :boolean, aliases: "-D", required: false, desc: "Developer mode, please do not use if unsure" - + class_option :tebafile, type: :string, aliases: "-t", required: false, + desc: "tebako configuration file 'tebafile', '$PWD/.tebako.yml' by default" desc "clean", "Clean tebako packaging environment" def clean clean_cache @@ -123,10 +124,10 @@ def initialize(*args) def options original_options = super + tebafile = original_options["tebafile"].nil? ? DEFAULT_TEBAFILE : original_options["tebafile"] + return original_options unless File.exist?(tebafile) - return original_options unless File.exist?(OPTIONS_FILE) - - defaults = ::YAML.load_file(OPTIONS_FILE) || {} + defaults = ::YAML.load_file(tebafile)["options"] || {} Thor::CoreExt::HashWithIndifferentAccess.new(defaults.merge(original_options)) end end diff --git a/lib/tebako/packager/pass1.rb b/lib/tebako/packager/pass1.rb index 9f89dfe0..303d0e49 100755 --- a/lib/tebako/packager/pass1.rb +++ b/lib/tebako/packager/pass1.rb @@ -92,6 +92,10 @@ module Pass1 # "EXTDLDFLAGS=\"-bundle_loader '\\$(BUILTRUBY)'\"" => "" # }.freeze + DARWIN_CONFIGURE_PATCH = { + "elif test \"x$EXTSTATIC\" = x" => "elif true" + }.freeze + OPENSSL_EXTCONF_RB_SUBST = <<~SUBST # Start of tebako patch $defs.push("-DRUBY_EXPORT=1") @@ -137,11 +141,7 @@ def get_patch_map(ostype, mount_point, ruby_ver) patch_map = get_base_patch_map(mount_point) # .................................................... - # Fixing (bypassing) configure script bug where a variable is used before initialization - # On MacOS it generates bod EXTDLDFLAGS WITH -bundle_loader - # However, it loooks likes EXTDLDFLAGS are just redundant for final Ruby linkage, so we are - # just removing it on pass 2 (at least for now) - # patch_map.store("configure", DARWIN_CONFIGURE_PATCH) if ostype =~ /darwin/ + patch_map.store("configure", DARWIN_CONFIGURE_PATCH) if ostype =~ /darwin/ # .................................................... # autoload :OpenSSL, "openssl" diff --git a/lib/tebako/packager/patch_libraries.rb b/lib/tebako/packager/patch_libraries.rb index 908bee11..a9ca0c79 100644 --- a/lib/tebako/packager/patch_libraries.rb +++ b/lib/tebako/packager/patch_libraries.rb @@ -34,13 +34,10 @@ module Packager module PatchLibraries class << self # rubocop:disable Style/WordArray - - # NOTE: folly provides build-in implementation of jemalloc - DARWIN_BREW_LIBS = [ - ["zlib", "z"], ["gdbm", "gdbm"], ["readline", "readline"], ["libffi", "ffi"], - ["ncurses", "ncurses"], ["fmt", "fmt"], ["lz4", "lz4"], ["xz", "lzma"], - ["libyaml", "yaml"], ["boost", "boost_chrono"], ["double-conversion", "double-conversion"] + ["zlib", "z"], ["gdbm", "gdbm"], ["readline", "readline"], ["libffi", "ffi"], + ["ncurses", "ncurses"], ["fmt", "fmt"], ["lz4", "lz4"], ["xz", "lzma"], + ["libyaml", "yaml"], ["boost", "boost_chrono"], ["double-conversion", "double-conversion"] ].freeze DARWIN_BREW_LIBS_PRE_31 = [["openssl@1.1", "ssl"], ["openssl@1.1", "crypto"]].freeze diff --git a/lib/tebako/version.rb b/lib/tebako/version.rb index 98589c07..1d4e99e4 100644 --- a/lib/tebako/version.rb +++ b/lib/tebako/version.rb @@ -26,5 +26,5 @@ # POSSIBILITY OF SUCH DAMAGE. module Tebako - VERSION = "0.8.0" + VERSION = "0.9.0" end diff --git a/src/tebako-main.cpp b/src/tebako-main.cpp index 985577a4..293f7693 100644 --- a/src/tebako-main.cpp +++ b/src/tebako-main.cpp @@ -145,6 +145,8 @@ extern "C" int tebako_main(int* argc, char*** argv) { // Nested error, no recovery :( } } + + // tebako_chdir("/__tebako_memfs__/local"); } return ret; }