From bad7a16756a1d50baadf9a5dc71e3f0aeb8d4f16 Mon Sep 17 00:00:00 2001 From: Jonah Beckford <71855677+jonahbeckford@users.noreply.github.com> Date: Mon, 7 Nov 2022 12:06:05 -0800 Subject: [PATCH] Use dkml-workflows.1.1.0 w/o pin (do upgrade) --- ci/setup-dkml/gh-darwin/post/action.yml | 73 + ci/setup-dkml/gh-darwin/post/dune | 17 + ci/setup-dkml/gh-darwin/pre/action.yml | 1630 ++++++++++++++++ ci/setup-dkml/gh-darwin/pre/dune | 17 + ci/setup-dkml/gh-linux/post/action.yml | 91 + ci/setup-dkml/gh-linux/post/dune | 17 + ci/setup-dkml/gh-linux/pre/action.yml | 1691 +++++++++++++++++ ci/setup-dkml/gh-linux/pre/dune | 17 + ci/setup-dkml/gh-windows/post/action.yml | 102 + ci/setup-dkml/gh-windows/post/dune | 17 + ci/setup-dkml/gh-windows/pre/action.yml | 1864 +++++++++++++++++++ ci/setup-dkml/gh-windows/pre/dune | 17 + ci/setup-dkml/pc/setup-dkml-linux_x86.sh | 1512 +++++++++++++++ ci/setup-dkml/pc/setup-dkml-linux_x86_64.sh | 1512 +++++++++++++++ 14 files changed, 8577 insertions(+) create mode 100644 ci/setup-dkml/gh-darwin/post/action.yml create mode 100644 ci/setup-dkml/gh-darwin/post/dune create mode 100644 ci/setup-dkml/gh-darwin/pre/action.yml create mode 100644 ci/setup-dkml/gh-darwin/pre/dune create mode 100644 ci/setup-dkml/gh-linux/post/action.yml create mode 100644 ci/setup-dkml/gh-linux/post/dune create mode 100644 ci/setup-dkml/gh-linux/pre/action.yml create mode 100644 ci/setup-dkml/gh-linux/pre/dune create mode 100644 ci/setup-dkml/gh-windows/post/action.yml create mode 100644 ci/setup-dkml/gh-windows/post/dune create mode 100644 ci/setup-dkml/gh-windows/pre/action.yml create mode 100644 ci/setup-dkml/gh-windows/pre/dune create mode 100644 ci/setup-dkml/pc/setup-dkml-linux_x86.sh create mode 100644 ci/setup-dkml/pc/setup-dkml-linux_x86_64.sh diff --git a/ci/setup-dkml/gh-darwin/post/action.yml b/ci/setup-dkml/gh-darwin/post/action.yml new file mode 100644 index 0000000..3fb1cf6 --- /dev/null +++ b/ci/setup-dkml/gh-darwin/post/action.yml @@ -0,0 +1,73 @@ +# setup-dkml +# Short form: sd4 + +name: post-dkml +author: Diskuv, Inc. +description: Teardown after building OCaml native executables for Darwin + +runs: + using: "composite" + + steps: + - name: Full matrix variables + shell: bash + # Every matrix variable lookup in this Action should use the output of this step. Even + # the matrix variables that the user must specify (ex. dkml_host_abi) should be + # referenced using [steps.full_matrix_vars.outputs.dkml_host_abi] rather than + # [matrix.dkml_host_abi] so that there is a single place to edit for variable changes. + id: full_matrix_vars + run: | + # Select correct Darwin matrix variables + case "${{ matrix.dkml_host_abi }}" in + + darwin_x86_64) + dkml_host_os='darwin'; + opam_root_cacheable='/Users/runner/.opam'; + abi_pattern='macos-darwin_all'; + gh_os='macos-latest'; + gh_unix_shell='sh'; + bootstrap_opam_version='2.2.0-dkml20220801T155940Z'; + dkml_host_abi='darwin_x86_64'; + opam_root='/Users/runner/.opam' ;; + + *) echo "FATAL: Unsupported dkml_host_abi=$dkml_host_abi in Darwin action.yml"; exit 107 ;; + esac + + add() { + echo "$1=$2" | tee -a $GITHUB_OUTPUT | tee -a $GITHUB_ENV + } + + add dkml_host_abi "$dkml_host_abi" + add abi_pattern "$abi_pattern" + add opam_root "$opam_root" + add opam_root_cacheable "$opam_root_cacheable" + add exe_ext "${exe_ext:-}" + add bootstrap_opam_version "${bootstrap_opam_version:-}" + add ocaml_options "${ocaml_options:-}" + + - name: Teardown DKML build apparatus + shell: bash + env: + _STUB_FOR_AUTOGEN: "ON" # + + # autogen from global_env_vars. + DEFAULT_DKML_COMPILER: '4.12.1-v1.0.2' + PIN_BASE: 'v0.14.3' + PIN_BIGSTRINGAF: '0.8.0' + PIN_CORE_KERNEL: 'v0.14.2' + PIN_CTYPES_FOREIGN: '0.19.2-windowssupport-r4' + PIN_CTYPES: '0.19.2-windowssupport-r4' + PIN_CURLY: '0.2.1-windows-env_r2' + PIN_DIGESTIF: '1.0.1' + PIN_DUNE: '2.9.3+shim.1.0.2~r0' + PIN_DUNE_CONFIGURATOR: '2.9.3' + PIN_DKML_APPS: '1.0.1' + PIN_OCAMLBUILD: '0.14.0' + PIN_OCAMLFIND: '1.9.1' + PIN_OCP_INDENT: '1.8.2-windowssupport' + PIN_PPX_EXPECT: 'v0.14.1' + PIN_PTIME: '0.8.6-msvcsupport' + PIN_TIME_NOW: 'v0.14.0' + PIN_WITH_DKML: '1.0.1' + run: | + sh .ci/sd4/run-teardown-dkml.sh GITHUB_WORKSPACE "$GITHUB_WORKSPACE" diff --git a/ci/setup-dkml/gh-darwin/post/dune b/ci/setup-dkml/gh-darwin/post/dune new file mode 100644 index 0000000..e8c6ac3 --- /dev/null +++ b/ci/setup-dkml/gh-darwin/post/dune @@ -0,0 +1,17 @@ +; DO NOT EDIT THIS FILE. It is auto-generated by generate-setup-dkml-scaffold +; Typical upgrade steps: +; opam upgrade dkml-workflows && opam exec -- generate-setup-dkml-scaffold && dune build '@gen-dkml' --auto-promote + +(rule + (alias gen-dkml) + (target action.gen.yml) + (action + (setenv + OCAMLRUNPARAM + b + (run gh-dkml-action-yml --phase post --output-darwin %{target})))) + +(rule + (alias gen-dkml) + (action + (diff action.yml action.gen.yml))) diff --git a/ci/setup-dkml/gh-darwin/pre/action.yml b/ci/setup-dkml/gh-darwin/pre/action.yml new file mode 100644 index 0000000..0cd8f29 --- /dev/null +++ b/ci/setup-dkml/gh-darwin/pre/action.yml @@ -0,0 +1,1630 @@ +# setup-dkml +# Short form: sd4 + +name: pre-dkml +author: Diskuv, Inc. +description: Setup before building OCaml native executables for Darwin + +inputs: + FDOPEN_OPAMEXE_BOOTSTRAP: + description: "Use opam.exe from fdopen on Windows. Typically only used when bootstrapping Opam for the first time. May be needed to solve '\"create_process\" failed on sleep: Bad file descriptor' which may need https://github.com/ocaml/opam/commit/417b97d8cfada35682a0f4107eb2e4f9e24fba91" + required: false + default: 'false' + CACHE_PREFIX: + description: The prefix of the cache keys. + required: false + default: "v1" + OCAML_COMPILER: + description: "The version of the OCaml compiler, as tagged by the https://github.com/diskuv/dkml-base-compiler repository. If not specified the latest compiler, or the version associated with the 'dkml-compiler' input, is used. Currently only 4.12.1 is supported" + required: false + DKML_COMPILER: + description: "Git branch, tag or commit for dkml-compiler. However if OCAML_COMPILER is specified the 'dkml-base-compiler' version is taken from OCAML_COMPILER." + required: false + default: "" # "@repository@" = Opam ; "" = latest from default branch of git clone + CONF_DKML_CROSS_TOOLCHAIN: + description: "Git branch, tag or commit for conf-dkml-cross-toolchain" + required: false + default: "@repository@" # "@repository@" = Opam ; "" = latest from default branch of git clone + DISKUV_OPAM_REPOSITORY: + description: "Git branch, tag or commit for diskuv-opam-repository" + required: false + default: "" # DEFAULT_DISKUV_OPAM_REPOSITORY_TAG is used as default for empty strings + VERBOSE: + description: "Turn on verbose logging" + required: false + default: 'false' + SECONDARY_SWITCH: + description: "Turn on secondary 'two' switch" + required: false + default: 'false' + +runs: + using: "composite" + + steps: + - name: Full matrix variables + shell: bash + # Every matrix variable lookup in this Action should use the output of this step. Even + # the matrix variables that the user must specify (ex. dkml_host_abi) should be + # referenced using [steps.full_matrix_vars.outputs.dkml_host_abi] rather than + # [matrix.dkml_host_abi] so that there is a single place to edit for variable changes. + id: full_matrix_vars + run: | + # Select correct Darwin matrix variables + case "${{ matrix.dkml_host_abi }}" in + + darwin_x86_64) + dkml_host_os='darwin'; + opam_root_cacheable='/Users/runner/.opam'; + abi_pattern='macos-darwin_all'; + gh_os='macos-latest'; + gh_unix_shell='sh'; + bootstrap_opam_version='2.2.0-dkml20220801T155940Z'; + dkml_host_abi='darwin_x86_64'; + opam_root='/Users/runner/.opam' ;; + + *) echo "FATAL: Unsupported dkml_host_abi=$dkml_host_abi in Darwin action.yml"; exit 107 ;; + esac + + add() { + echo "$1=$2" | tee -a $GITHUB_OUTPUT | tee -a $GITHUB_ENV + } + + add dkml_host_abi "$dkml_host_abi" + add abi_pattern "$abi_pattern" + add opam_root "$opam_root" + add opam_root_cacheable "$opam_root_cacheable" + add exe_ext "${exe_ext:-}" + add bootstrap_opam_version "${bootstrap_opam_version:-}" + add ocaml_options "${ocaml_options:-}" + + - name: Write POSIX shell and AWK scripts + shell: bash + run: | + install -d .ci/sd4 + + cat > .ci/sd4/common-values.sh <<'end_of_script' + #!/bin/sh + + # ------------------------ Log Formatting ------------------------ + + TXT_SECTION="\e[94m" # bright blue + TXT_CLEAR="\e[0m" + + if [ "${GITLAB_CI:-}" = "true" ]; then + # https://docs.gitlab.com/ee/ci/jobs/#expand-and-collapse-job-log-sections + print_section_start() { + print_section_start_NAME=$1 + shift + printf "\e[0Ksection_start:%s:%s[collapsed=true]\r\e[0K" \ + "$(date +%s)" \ + "$print_section_start_NAME" + } + print_section_end() { + print_section_end_NAME=$1 + shift + printf "\e[0Ksection_end:%s:%s\r\e[0K\n" \ + "$(date +%s)" \ + "$print_section_end_NAME" + } + elif [ -n "${GITHUB_ENV:-}" ]; then + # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines + print_section_start() { + print_section_start_NAME=$1 + shift + printf "::group::" + } + print_section_end() { + print_section_end_NAME=$1 + shift + printf "::endgroup::\n" + } + else + print_section_start() { + print_section_start_NAME=$1 + shift + } + print_section_end() { + print_section_end_NAME=$1 + shift + } + fi + + section_begin() { + # https://docs.gitlab.com/ee/ci/yaml/script.html#add-color-codes-to-script-output + section_NAME=$1 + shift + section_HEADER=$1 + shift + print_section_start "$section_NAME" + printf "${TXT_SECTION}%s${TXT_CLEAR}\n" "$section_HEADER" + } + + section_end() { + section_NAME=$1 + shift + print_section_end "$section_NAME" + } + + # ------------------- Other Functions ----------------- + + transfer_dir() { + transfer_dir_SRC=$1 + shift + transfer_dir_DST=$1 + shift + # Remove the destination directory completely, but make sure the parent of the + # destination directory exists so `mv` will work + install -d "$transfer_dir_DST" + rm -rf "$transfer_dir_DST" + # Move + mv "$transfer_dir_SRC" "$transfer_dir_DST" + } + + # Set TEMP variable which is used, among other things, for OCaml's + # [Filename.temp_dir_name] on Win32, and by with-dkml.exe on Windows + export_temp_for_windows() { + if [ -x /usr/bin/cygpath ]; then + if [ -n "${RUNNER_TEMP:-}" ]; then + # GitHub Actions + TEMP=$(cygpath -am "$RUNNER_TEMP") + else + # GitLab CI/CD or desktop + install -d .ci/tmp + TEMP=$(cygpath -am ".ci/tmp") + fi + export TEMP + fi + } + + # Fixup opam_root on Windows to be mixed case. + # On input the following variables must be present: + # - opam_root + # - opam_root_cacheable + # On output the input variables will be modified _and_ the + # following variables will be available: + # - original_opam_root + # - original_opam_root_cacheable + # - unix_opam_root + # - unix_opam_root_cacheable + fixup_opam_root() { + # shellcheck disable=SC2034 + original_opam_root=$opam_root + # shellcheck disable=SC2034 + original_opam_root_cacheable=$opam_root_cacheable + if [ -x /usr/bin/cygpath ]; then + opam_root=$(/usr/bin/cygpath -m "$opam_root") + opam_root_cacheable=$(/usr/bin/cygpath -m "$opam_root_cacheable") + unix_opam_root=$(/usr/bin/cygpath -u "$opam_root") + unix_opam_root_cacheable=$(/usr/bin/cygpath -u "$opam_root_cacheable") + else + # shellcheck disable=SC2034 + unix_opam_root=$opam_root + # shellcheck disable=SC2034 + unix_opam_root_cacheable=$opam_root_cacheable + fi + } + end_of_script + + cat > .ci/sd4/run-checkout-code.sh <<'end_of_script' + #!/bin/sh + + # ================ + # checkout-code.sh + # ================ + # + # Checkouts all of the git source code. + # + # This should be done outside of + # dockcross (used by Linux) since a Docker-in-Docker container can have + # difficulties doing a git checkout (the Git credentials for any private + # repositories are likely not present). We don't care about any private + # repositories for DKML but any code that extends this (ex. DKSDK) may + # need to use private repositories. + + set -euf + + setup_WORKSPACE_VARNAME=$1 + shift + setup_WORKSPACE=$1 + shift + + # ------------------------ Functions ------------------------ + + # shellcheck source=./common-values.sh + . .ci/sd4/common-values.sh + + # Disable automatic garbage collection + git_disable_gc() { + git_disable_gc_NAME=$1 + shift + git -C ".ci/sd4/g/$git_disable_gc_NAME" config --local gc.auto 0 + } + + # Mimic the behavior of GitHub's actions/checkout@v3 + # - the plus symbol in 'git fetch ... origin +REF:refs/tags/v0.0' overrides any existing REF + git_checkout() { + git_checkout_NAME=$1 + shift + git_checkout_URL=$1 + shift + git_checkout_REF=$1 + shift + + if [ -e ".ci/sd4/g/$git_checkout_NAME" ]; then + git_disable_gc "$git_checkout_NAME" + git -C ".ci/sd4/g/$git_checkout_NAME" remote set-url origin "$git_checkout_URL" + git -C ".ci/sd4/g/$git_checkout_NAME" fetch --no-tags --progress --no-recurse-submodules --depth=1 origin "+${git_checkout_REF}:refs/tags/v0.0" + else + install -d ".ci/sd4/g/$git_checkout_NAME" + git -C ".ci/sd4/g/$git_checkout_NAME" -c init.defaultBranch=main init + git_disable_gc "$git_checkout_NAME" + git -C ".ci/sd4/g/$git_checkout_NAME" remote add origin "$git_checkout_URL" + git -C ".ci/sd4/g/$git_checkout_NAME" fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin "+${git_checkout_REF}:refs/tags/v0.0" + fi + git -C ".ci/sd4/g/$git_checkout_NAME" -c advice.detachedHead=false checkout --progress --force refs/tags/v0.0 + git -C ".ci/sd4/g/$git_checkout_NAME" log -1 --format='%H' + } + + # --------------------------------------------------------------------- + + section_begin checkout-info "Summary: code checkout" + + # shellcheck disable=SC2154 + echo " + ================ + checkout-code.sh + ================ + . + --------- + Arguments + --------- + WORKSPACE_VARNAME=$setup_WORKSPACE_VARNAME + WORKSPACE=$setup_WORKSPACE + . + ------ + Inputs + ------ + VERBOSE=${VERBOSE:-} + . + ------ + Matrix + ------ + dkml_host_abi=$dkml_host_abi + . + " + + section_end checkout-info + + install -d .ci/sd4/g + + # dkml-runtime-distribution + + case "$dkml_host_abi" in + windows_*) + section_begin checkout-dkml-runtime-distribution 'Checkout dkml-runtime-distribution' + git_checkout dkml-runtime-distribution https://github.com/diskuv/dkml-runtime-distribution.git "1a3ec82dd851751a95e6a4797387a8163c51520e" + section_end checkout-dkml-runtime-distribution + ;; + esac + + end_of_script + + cat > .ci/sd4/run-setup-dkml.sh <<'end_of_script' + #!/bin/sh + set -euf + + # Constants + SHA512_DEVNULL='cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e' + # Edited by https://gitlab.com/diskuv/diskuv-ocaml/contributors/release.sh + DEFAULT_DISKUV_OPAM_REPOSITORY_TAG=cc9518fa5630bfbe24d4c5e0e2cc29af513037ce + # Constants + # Should be edited by release.sh, but ... + # Can't be 1.0.0 or later until https://github.com/ocaml/opam-repository/pull/21704 ocaml-option-32bit + # can come back in. + DKML_VERSION=0.4.0 + + setup_WORKSPACE_VARNAME=$1 + shift + setup_WORKSPACE=$1 + shift + + # ------------------ Variables and functions ------------------------ + + # shellcheck source=./common-values.sh + . .ci/sd4/common-values.sh + + if [ "${VERBOSE:-}" = "true" ]; then + do_tar_rf() { + tar rvf "$@" + } + else + do_tar_rf() { + tar rf "$@" + } + fi + + # Make the standard input work as an OCaml string. + # This currently only escapes backslashes and double quotes. + escape_arg_as_ocaml_string() { + escape_arg_as_ocaml_string_ARG=$1 + shift + printf "%s" "$escape_arg_as_ocaml_string_ARG" | sed 's#\\#\\\\#g; s#"#\\"#g;' + } + + # Fixup opam_root on Windows to be mixed case. Set original_* and unix_* as well. + fixup_opam_root + + # Set TEMP variable for Windows + export_temp_for_windows + + # Load VS studio environment + if [ -e .ci/sd4/vsenv.sh ]; then + # shellcheck disable=SC1091 + . .ci/sd4/vsenv.sh + fi + + # ------------------------------------------------------------------- + + section_begin setup-info "Summary: setup-dkml" + + # shellcheck disable=SC2154 + echo " + ============= + setup-dkml.sh + ============= + . + --------- + Arguments + --------- + WORKSPACE_VARNAME=$setup_WORKSPACE_VARNAME + WORKSPACE=$setup_WORKSPACE + . + ------ + Inputs + ------ + FDOPEN_OPAMEXE_BOOTSTRAP=${FDOPEN_OPAMEXE_BOOTSTRAP:-} + DISKUV_OPAM_REPOSITORY=${DISKUV_OPAM_REPOSITORY:-} + DKML_COMPILER=${DKML_COMPILER:-} + OCAML_COMPILER=${OCAML_COMPILER:-} + CONF_DKML_CROSS_TOOLCHAIN=${CONF_DKML_CROSS_TOOLCHAIN:-} + SECONDARY_SWITCH=${SECONDARY_SWITCH:-} + MANYLINUX=${MANYLINUX:-} + VERBOSE=${VERBOSE:-} + . + ------------------- + Generated Constants + ------------------- + DKML_VERSION=$DKML_VERSION + DEFAULT_DISKUV_OPAM_REPOSITORY_TAG=$DEFAULT_DISKUV_OPAM_REPOSITORY_TAG + DEFAULT_DKML_COMPILER=$DEFAULT_DKML_COMPILER + . + ------ + Matrix + ------ + dkml_host_abi=$dkml_host_abi + bootstrap_opam_version=$bootstrap_opam_version + abi_pattern=$abi_pattern + opam_root=${opam_root} + opam_root_cacheable=${opam_root_cacheable} + original_opam_root=${original_opam_root} + original_opam_root_cacheable=${original_opam_root_cacheable} + unix_opam_root=${unix_opam_root} + unix_opam_root_cacheable=${unix_opam_root_cacheable} + dockcross_image=${dockcross_image:-} + dockcross_image_custom_prefix=${dockcross_image_custom_prefix:-} + dockcross_run_extra_args=${dockcross_run_extra_args:-} + docker_runner=${docker_runner:-} + in_docker=${in_docker:-} + ocaml_options=${ocaml_options:-} + . + ---- + Pins + ---- + PIN_BASE=${PIN_BASE} + PIN_BIGSTRINGAF=${PIN_BIGSTRINGAF} + PIN_CORE_KERNEL=${PIN_CORE_KERNEL} + PIN_CTYPES=${PIN_CTYPES} + PIN_CTYPES_FOREIGN=${PIN_CTYPES_FOREIGN} + PIN_CURLY=${PIN_CURLY} + PIN_DIGESTIF=${PIN_DIGESTIF} + PIN_DKML_APPS=${PIN_DKML_APPS} + PIN_DUNE=${PIN_DUNE} + PIN_DUNE_CONFIGURATOR=${PIN_DUNE_CONFIGURATOR} + PIN_OCAMLBUILD=${PIN_OCAMLBUILD} + PIN_OCAMLFIND=${PIN_OCAMLFIND} + PIN_OCP_INDENT=${PIN_OCP_INDENT} + PIN_PPX_EXPECT=${PIN_PPX_EXPECT} + PIN_PTIME=${PIN_PTIME} + PIN_TIME_NOW=${PIN_TIME_NOW} + PIN_WITH_DKML=${PIN_WITH_DKML} + . + " + case "$dkml_host_abi" in + windows_*) + # shellcheck disable=SC2153 + echo " + ------------- + Visual Studio + ------------- + VS_DIR=$VS_DIR + VS_VCVARSVER=$VS_VCVARSVER + VS_WINSDKVER=$VS_WINSDKVER + VS_MSVSPREFERENCE=$VS_MSVSPREFERENCE + VS_CMAKEGENERATOR=$VS_CMAKEGENERATOR + . + " + ;; + esac + section_end setup-info + + do_bootstrap() { + # Bootstrap from historical release + runit_BOOTSTRAPPED=0 + + # Bootstrap opam from fdopen (Windows) + if [ "$runit_BOOTSTRAPPED" = 0 ] && [ "${FDOPEN_OPAMEXE_BOOTSTRAP:-}" = "true" ]; then + if [ -e .ci/sd4/opam64/bin/opam.exe ] && [ -e .ci/sd4/opam64/bin/opam-installer.exe ]; then + runit_BOOTSTRAPPED=1 + else + case "$dkml_host_abi" in + windows_*) + echo 'Bootstrap opam from fdopen (Windows) ...' + install -d .ci/sd4/bs/bin + wget -O "$setup_WORKSPACE"/.ci/sd4/opam64.tar.xz https://github.com/fdopen/opam-repository-mingw/releases/download/0.0.0.2/opam64.tar.xz + + # this stalls: tar xvCfJ "$setup_WORKSPACE"/.ci/sd4 "$setup_WORKSPACE"/.ci/sd4/opam64.tar.xz + xz -v -d "$setup_WORKSPACE"/.ci/sd4/opam64.tar.xz + tar xvCf .ci/sd4 .ci/sd4/opam64.tar + + rm -rf "$setup_WORKSPACE"/.ci/sd4/bs/bin/Opam.Runtime.amd64 + mv -v "$setup_WORKSPACE"/.ci/sd4/opam64/bin/Opam.Runtime.amd64/ "$setup_WORKSPACE"/.ci/sd4/bs/bin/ + mv -v "$setup_WORKSPACE"/.ci/sd4/opam64/bin/opam.exe "$setup_WORKSPACE"/.ci/sd4/bs/bin/ + mv -v "$setup_WORKSPACE"/.ci/sd4/opam64/bin/opam-installer.exe "$setup_WORKSPACE"/.ci/sd4/bs/bin/ + + # diagnostics + ldd "$setup_WORKSPACE"/.ci/sd4/bs/bin/opam.exe + ldd "$setup_WORKSPACE"/.ci/sd4/bs/bin/opam-installer.exe + + runit_BOOTSTRAPPED=1 + ;; + esac + fi + fi + + # Bootstrap from historical release + if [ "$runit_BOOTSTRAPPED" = 0 ] && [ "$bootstrap_opam_version" != "os" ]; then + install -d .ci/sd4/bs + cd .ci/sd4/bs + + if [ ! -e version ] || [ "$(cat version)" != "$bootstrap_opam_version" ]; then + echo 'Bootstrap opam from historical release (non-Windows; Windows non-fdopen) ...' + if command -v curl; then + curl -L -o opam.tar.gz "https://github.com/diskuv/dkml-component-opam/releases/download/v${bootstrap_opam_version}/dkml-component-staging-opam.tar.gz" + else + wget -O opam.tar.gz "https://github.com/diskuv/dkml-component-opam/releases/download/v${bootstrap_opam_version}/dkml-component-staging-opam.tar.gz" + fi + tar tvfz opam.tar.gz + tar xfz opam.tar.gz "./staging-files/${dkml_host_abi}/" + rm -rf bin/ + mv "staging-files/${dkml_host_abi}/bin" . + rm -rf "${abi_pattern}" + printf "%s" "${bootstrap_opam_version}" >version + fi + + rm -f opam.tar.gz + cd ../../.. + + runit_BOOTSTRAPPED=1 + fi + + # Bootstrap from package manager or GitHub ocaml/opam release + case "$runit_BOOTSTRAPPED,$bootstrap_opam_version,$dkml_host_abi" in + 0,os,darwin_*) + if ! command -v opam; then + echo 'Bootstrap opam from package manager (macOS) ...' + brew install gpatch + brew install opam + fi + runit_BOOTSTRAPPED=1 + ;; + 0,os,linux_x86) + if [ ! -x .ci/sd4/bs/bin/opam ]; then + echo 'Bootstrap opam from GitHub ocaml/opam release (Linux x86) ...' + install -d .ci/sd4/bs/bin + wget -O .ci/sd4/bs/bin/opam.tmp https://github.com/ocaml/opam/releases/download/2.1.2/opam-2.1.2-i686-linux + sha512_check=$(openssl sha512 2>&1 /dev/null | cut -f 2 -d ' ') + check="85a480d60e09a7d37fa0d0434ed97a3187434772ceb4e7e8faa5b06bc18423d004af3ad5849c7d35e72dca155103257fd6b1178872df8291583929eb8f884b6a" + test "$sha512" = "$check" + chmod +x .ci/sd4/bs/bin/opam.tmp + mv .ci/sd4/bs/bin/opam.tmp .ci/sd4/bs/bin/opam + else + echo "openssl 512 option unsupported." + exit 61 + fi + fi + runit_BOOTSTRAPPED=1 + ;; + 0,os,linux_x86_64) + if [ ! -x .ci/sd4/bs/bin/opam ]; then + echo 'Bootstrap opam from GitHub ocaml/opam release (Linux x86_64) ...' + install -d .ci/sd4/bs/bin + wget -O .ci/sd4/bs/bin/opam.tmp https://github.com/ocaml/opam/releases/download/2.1.2/opam-2.1.2-x86_64-linux + sha512_check=$(openssl sha512 2>&1 /dev/null | cut -f 2 -d ' ') + check="c0657ecbd4dc212587a4da70c5ff0402df95d148867be0e1eb1be8863a2851015f191437c3c99b7c2b153fcaa56cac99169c76ec94c5787750d7a59cd1fbb68b" + test "$sha512" = "$check" + chmod +x .ci/sd4/bs/bin/opam.tmp + mv .ci/sd4/bs/bin/opam.tmp .ci/sd4/bs/bin/opam + else + echo "openssl 512 option unsupported." + exit 61 + fi + fi + runit_BOOTSTRAPPED=1 + ;; + esac + } + section_begin bootstrap-opam 'Bootstrap opam' + do_bootstrap + section_end bootstrap-opam + + # Start environment distribution tarball + # We use .tar rather than .tar.gz/.tar.bz2 because we can repeatedly add to an uncompressed .tar. But we need to + # start with an empty tarball since some tar programs will only add ('tar rf xyz.tar') to an existing .tar. + install -d .ci/sd4/dist + tar cf .ci/sd4/dist/opam-with-env.tar -T /dev/null + + do_get_dockcross() { + if [ -n "${dockcross_image:-}" ]; then + # The dockcross script is super-slow + section_begin get-dockcross 'Get dockcross binary (ManyLinux)' + install -d .ci/sd4 + # shellcheck disable=SC2086 + docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_custom_prefix:-}${dockcross_image:-}" >.ci/sd4/dockcross.gen + + # PROBLEM 1 + # --------- + # Super-annoying stderr output from dockcross at line: + # tty -s && [ -z "$MSYS" ] && TTY_ARGS=-ti + # When there is no tty, get: + # tty: ignoring all arguments + # not a tty + # So replace 'tty -s &&' with 'false &&' + sed 's/tty -s &&/false \&\&/' .ci/sd4/dockcross.gen >.ci/sd4/dockcross-real + rm -f .ci/sd4/dockcross.gen + chmod +x .ci/sd4/dockcross-real + + # PROBLEM 2 + # --------- + # By default dockcross for ManyLinux will chown -R all python packages; super-slow (~10 seconds)! + # Confer: https://github.com/dockcross/dockcross/blob/master/manylinux-common/pre_exec.sh + # That kills speed for any repetitive dockcross invocation. + # + # BUT it is unnecessary to chown -R when the current user is root, because inside the Docker container + # the files are already root! + # + # The chown -R (within pre_exec.sh) is not run when the user ids are not passed in. + # Confer: https://github.com/dockcross/dockcross/blob/96d87416f639af0204bdd42553e4b99315ca8476/imagefiles/entrypoint.sh#L21-L53 + # + # So explicitly call the entrypoint if root! + case "$dkml_host_abi" in + # https://github.com/dockcross/dockcross/blob/master/linux-x86/linux32-entrypoint.sh + linux_x86) dockcross_entrypoint=/dockcross/linux32-entrypoint.sh ;; + *) dockcross_entrypoint=/dockcross/entrypoint.sh ;; + esac + cat > .ci/sd4/dockcross <&2 "Missing functionality (\${f}) (in cygwin)." ; exit 1 ; } ; + done ; + HOST_PWD="\$( cygpath -w "\$( readlink -f "\$( pwd ;)" ; )" ; )" ; + else + HOST_PWD=\$PWD + [ -L \$HOST_PWD ] && HOST_PWD=\$(readlink \$HOST_PWD) + fi + + # ---------- End of dockcross script snippet ------- + + # Handle: dockcross --args "-v X:Y --platform P" + ARGS= + if [ "\$#" -ge 1 ] && [ "\$1" = "--args" ]; then + shift + ARGS=\$1 + shift + fi + + # Directly invoke entrypoint + exec docker run --entrypoint /bin/bash \ + --rm \ + \${ARGS:-} \ + -v "\$HOST_PWD":/work \ + ${dockcross_image_custom_prefix:-}${dockcross_image:-} ${dockcross_entrypoint} "\$@" + else + HERE=\$(dirname "\$0") + HERE=\$(cd "\$HERE" && pwd) + exec "\$HERE/dockcross-real" "\$@" + fi + EOF + chmod +x .ci/sd4/dockcross + + # Bundle for consumers of setup-dkml.yml + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/dockcross .ci/sd4/dockcross-real + + section_end get-dockcross + fi + } + do_get_dockcross + + if [ -n "${dockcross_image:-}" ]; then + # rsync needs to be available, even after Docker container disappears + if [ ! -e .ci/sd4/bs/bin/rsync ]; then + section_begin get-opam-prereqs-in-dockcross 'Get Opam prerequisites (ManyLinux)' + install -d .ci/sd4/bs/bin + # shellcheck disable=SC2016 + .ci/sd4/dockcross --args "${dockcross_run_extra_args:-}" sh -c 'sudo yum install -y rsync && install $(command -v rsync) .ci/sd4/bs/bin' + section_end get-opam-prereqs-in-dockcross + fi + fi + + # Opam prerequisites for using opam (not for installing opam) + + { + if [ -n "${docker_runner:-}" ]; then + # rsync needs to be available, even after Docker container disappears + if [ ! -e .ci/sd4/bs/bin/rsync.deps ]; then + section_begin get-opam-prereqs-in-docker 'Get Opam prerequisites (Linux Docker)' + install -d .ci/sd4/bs/bin + ${docker_runner} sh -c ' + apt-get update && + apt-get install -y rsync && + ldd /usr/bin/rsync && + ls -l /lib/i386-linux-gnu/libpopt.so.0 /lib/i386-linux-gnu/libacl.so.1 /lib/i386-linux-gnu/libattr.so.1 && + tar cCfhz / /work/.ci/sd4/bs/bin/deps.tar.gz /usr/bin/rsync /lib/i386-linux-gnu/libpopt.so.0 + ' + touch .ci/sd4/bs/bin/rsync.deps + section_end get-opam-prereqs-in-docker + fi + fi + + # Bundle Opam prerequisites (ManyLinux or Linux Docker) + if [ -n "${docker_runner:-}" ] || [ -n "${dockcross_image:-}" ]; then + # Bundle for consumers of setup-dkml.yml + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/bs/bin/rsync + fi + } + + # Get Opam Cache + do_get_opam_cache() { + if [ "$unix_opam_root_cacheable" = "$unix_opam_root" ]; then return; fi + if [ ! -e "$unix_opam_root_cacheable" ]; then return; fi + section_begin get-opam-cache "Transferring Opam cache to $original_opam_root_cacheable" + echo Starting transfer # need some output or GitLab CI will not display the section duration + transfer_dir "$unix_opam_root_cacheable" "$unix_opam_root" + echo Finished transfer + section_end get-opam-cache + } + do_get_opam_cache + + # Setup Opam + + do_write_opam_scripts() { + case "${FDOPEN_OPAMEXE_BOOTSTRAP:-},$dkml_host_abi" in + true,windows_*) + # With fdopen's opam.exe, 'os-distribution = "cygwinports"'. But native Windows opam.exe has 'os-distribution = "win32"'. + # But on Windows we always want MSYS2 or native Windows libraries, not Cygwin. If cygwinports then + # code like https://github.com/ocaml/opam-repository/blob/08cbb8258bd4bf30cd6f307c958911a29d537b54/packages/conf-pkg-config/conf-pkg-config.2/opam#L36 + # will fail. So always set 'os-distribution = "win32"' on Windows. + PATCH_OS_DISTRIBUTION_WIN32=true + # With fdopen's opam.exe, no 'exe = ".exe"' is set because Cygwin does not need file extensions. + # Native Windows requires a .exe extension. + PATCH_EXE_WIN32=true + ;; + *) + PATCH_OS_DISTRIBUTION_WIN32=false + PATCH_EXE_WIN32=false + ;; + esac + + # --------------------- + # Empty opam repository + # --------------------- + + install -d .ci/sd4/eor + cat >.ci/sd4/eor/repo <.ci/sd4/troubleshoot-opam.sh <&2 + find "\$OPAMROOT"/log -mindepth 1 -maxdepth 1 -name "*.out" ! -name "log-*.out" ! -name "ocaml-variants-*.out" | while read -r dump_on_error_LOG; do + dump_on_error_BLOG=\$(basename "\$dump_on_error_LOG") + printf "\n\n========= [TROUBLESHOOTING] %s ===========\n\n" "\$dump_on_error_BLOG" >&2 + awk -v BLOG="\$dump_on_error_BLOG" '{print "[" BLOG "]", \$0}' "\$dump_on_error_LOG" >&2 + done + printf "\nScroll up to see the [TROUBLESHOOTING] logs that begin at the [START OF TROUBLESHOOTING] line\n" >&2 + EOF + + chmod +x .ci/sd4/troubleshoot-opam.sh + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/troubleshoot-opam.sh + + # --------------- + # Create Opam support scripts (not needed for all platforms) + # The PATH to find opam must work internally in setup-dkml.yml (sd4/bs/bin) and + # by consumers of setup-dkml.yml (sd4/opamexe) + # --------------- + + USER_ID=$(id -u) + GROUP_ID=$(id -g) + USER_NAME=$(id -un) + GROUP_NAME=$(id -gn) + + case "${opam_root}" in + /* | ?:*) # /a/b/c or C:\Windows + validate_supports_docker() { + echo "Docker only supported with relative paths for the opam root, not: ${opam_root}" >&2 + exit 3 + } + ;; + *) # relative path + validate_supports_docker() { + true + } + cat >.ci/sd4/opam-in-docker <&2 + set +e + opam "\$@" + exitcode=\$? + if [ \$troubleshooting = 1 ]; then + [ \$exitcode = 0 ] || "/work/.ci/sd4/troubleshoot-opam.sh" \$OPAMROOT + fi + exit \$exitcode + EOF + chmod +x .ci/sd4/opam-in-docker + ;; + esac + + cat >.ci/sd4/deescalate <.ci/sd4/opam-with-env <&2 + cat .ci/sd4/opam-in-docker >&2 + echo '________________________' >&2 + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/opam-with-env .ci/sd4/opam-in-docker .ci/sd4/edr + + elif [ -n "${docker_runner:-}" ]; then + + cat >.ci/sd4/opam-with-env <&2 + cat .ci/sd4/opam-in-docker >&2 + echo '________________________' >&2 + echo '__ deescalate __' >&2 + cat .ci/sd4/deescalate >&2 + echo '________________' >&2 + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/opam-with-env .ci/sd4/opam-in-docker .ci/sd4/deescalate + + else + + cat >.ci/sd4/opam-with-env <&2 + set +e + opam "\$@" + exitcode=\$? + if [ \$troubleshooting = 1 ]; then + [ \$exitcode = 0 ] || "\${PROJECT_DIR}/.ci/sd4/troubleshoot-opam.sh" \$OPAMROOT + fi + exit \$exitcode + EOF + chmod +x .ci/sd4/opam-with-env + + # Bundle for consumers of setup-dkml.yml + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/opam-with-env + + fi + echo '__ opam-with-env __' >&2 + cat .ci/sd4/opam-with-env >&2 + echo '___________________' >&2 + + # ------- + # opamrun + # ------- + + install -d .ci/sd4/opamrun + cat >.ci/sd4/opamrun/opamrun <>"$GITHUB_PATH" + # Special case: GITHUB_PATH does not influence msys2.CMD of msys2/setup-msys2@v2, so place in real MSYS2 PATH + if [ -n "${MSYSTEM:-}" ]; then + install -d /usr/local/bin + install .ci/sd4/opamrun/opamrun /usr/local/bin/opamrun + fi + fi + + # Place opamrun in the immediate PATH + PATH="$setup_WORKSPACE/.ci/sd4/opamrun:$PATH" + + # Complicated Opam sequence is because: + # 1. Opam's default curl does not work on Windows, + # and `opam init` does not provide a way to change it (TODO: need + # a PR!). + # 2. We have to separate the Opam download cache from the other Opam + # caches + if [ ! -s "$opam_root/.ci.root-init" ]; then # non-empty init file so can be cached irrespective of existence + section_begin opam-init 'Initialize opam root' + + # Clear any partial previous attempt + rm -rf "$opam_root" + + case "$dkml_host_abi,${in_docker:-}" in + windows_*,*) + eor=$(cygpath -am "$setup_WORKSPACE"/.ci/sd4/eor) + opamrun init --disable-sandboxing --no-setup --kind local --bare "$eor" + case "$(opamrun --version)" in + 2.0.*) echo 'download-command: wget' >>"$opam_root/config" ;; + *) opamrun option --yes --global download-command=wget ;; + esac + ;; + *,true) + opamrun init --disable-sandboxing --no-setup --kind local --bare "/work/.ci/sd4/eor" + ;; + *) + opamrun init --disable-sandboxing --no-setup --kind local --bare "$setup_WORKSPACE/.ci/sd4/eor" + ;; + esac + echo yes > "$opam_root/.ci.root-init" + + section_end opam-init + fi + + section_begin opam-vars "Summary: opam global variables" + opamrun --no-troubleshooting var --global || true + section_end opam-vars + + # Build OCaml + + do_switch_create() { + do_switch_create_NAME=$1 + shift + + section_begin "switch-create-$do_switch_create_NAME" "Create opam switch '$do_switch_create_NAME'" + # Create, or recreate, the Opam switch. The Opam switch should not be + # cached except for the compiler (confer docs for setup-ocaml GitHub + # Action) which is the 'dkml' switch (or the 'two' switch). + # Check if the switch name is present in the Opam root (which may come from cache) + NOMINALLY_PRESENT=false + if opamrun switch list --short | grep "^${do_switch_create_NAME}\$"; then NOMINALLY_PRESENT=true; fi + + # Check if the switch is actually present in case of cache incoherence + # or corrupt Opam state that could result in: + # Error: No config file found for switch dkml. Switch broken? + if [ $NOMINALLY_PRESENT = true ] && [ ! -e "$opam_root/$do_switch_create_NAME/.opam-switch/switch-config" ]; then + # Remove the switch name from Opam root, and any partial switch state. + # Ignore inevitable warnings/failure about missing switch. + opamrun --no-troubleshooting switch remove "$do_switch_create_NAME" --yes || true + rm -rf "${opam_root:?}/$do_switch_create_NAME" + NOMINALLY_PRESENT=false + fi + + if [ $NOMINALLY_PRESENT = false ]; then + opamrun switch create "$do_switch_create_NAME" --empty --yes + fi + section_end "switch-create-$do_switch_create_NAME" + } + do_switch_create dkml + if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_switch_create two + else + section_begin "switch-create-two" "Create empty opam switch 'two'" + # Always create a secondary switch ... just empty. Avoid problems with cache content missing + # and idempotency. + opamrun --no-troubleshooting switch remove two --yes || true + rm -rf "$opam_root/two" + opamrun switch create two --empty --yes + section_end "switch-create-two" + fi + + do_switch_active() { + section_begin "switch-active" "Set dkml as active switch" + opamrun switch set dkml --yes + section_end "switch-active" + } + do_switch_active + + do_opam_repositories_add() { + section_begin "opam-repo-add" "Add 'diskuv' opam repository" + if ! opamrun --no-troubleshooting repository list -s | grep '^diskuv'; then + opamrun repository add diskuv "git+https://github.com/diskuv/diskuv-opam-repository.git#${DISKUV_OPAM_REPOSITORY:-$DEFAULT_DISKUV_OPAM_REPOSITORY_TAG}" --yes --dont-select + fi + section_end "opam-repo-add" + } + do_opam_repositories_add + + do_opam_repositories_config() { + do_opam_repositories_config_NAME=$1 + shift + + section_begin "opam-repo-$do_opam_repositories_config_NAME" "Attach repositories to $do_opam_repositories_config_NAME" + + if [ ! -s "$opam_root/.ci.$do_opam_repositories_config_NAME.repo-init" ]; then # non-empty init file so can be cached irrespective of existence + opamrun --no-troubleshooting repository remove default --switch "$do_opam_repositories_config_NAME" --yes || true + opamrun --no-troubleshooting repository remove diskuv --switch "$do_opam_repositories_config_NAME" --yes || true + opamrun repository add default --switch "$do_opam_repositories_config_NAME" --yes + opamrun repository add diskuv --switch "$do_opam_repositories_config_NAME" --yes + echo yes > "$opam_root/.ci.$do_opam_repositories_config_NAME.repo-init" + fi + + section_end "opam-repo-$do_opam_repositories_config_NAME" + } + do_opam_repositories_config dkml + if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_opam_repositories_config two + fi + + do_opam_repositories_update() { + section_begin "opam-repo-update" "Update opam repositories" + # The default repository may be the initial 'eor' (empty) repository + opamrun repository set-url default https://opam.ocaml.org --yes + # Always set the `diskuv` repository url since it can change + opamrun repository set-url diskuv "git+https://github.com/diskuv/diskuv-opam-repository.git#${DISKUV_OPAM_REPOSITORY:-$DEFAULT_DISKUV_OPAM_REPOSITORY_TAG}" --yes --dont-select + # Update both `default` and `diskuv` Opam repositories + opamrun update default diskuv + section_end "opam-repo-update" + } + do_opam_repositories_update + + do_pins() { + do_pins_NAME=$1 + shift + + # dkml-base-compiler + + if [ "${DKML_COMPILER:-}" != '@repository@' ] && [ -z "${DKML_COMPILER:-}" ] && [ -z "${OCAML_COMPILER:-}" ]; then + section_begin checkout-dkml-base-compiler "Pin dkml-base-compiler to default ${DEFAULT_DKML_COMPILER} (neither dkml-base-compiler nor OCAML_COMPILER specified) for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action dkml-base-compiler "https://github.com/diskuv/dkml-compiler.git#${DEFAULT_DKML_COMPILER}" + section_end checkout-dkml-base-compiler + elif [ "${DKML_COMPILER:-}" != '@repository@' ] && [ -n "${DKML_COMPILER:-}" ] && [ -z "${OCAML_COMPILER:-}" ]; then + section_begin checkout-dkml-base-compiler "Pin dkml-base-compiler to $DKML_COMPILER (dkml-base-compiler specified; no OCAML_COMPILER specified) for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action dkml-base-compiler "https://github.com/diskuv/dkml-compiler.git#${DKML_COMPILER}" + section_end checkout-dkml-base-compiler + elif [ -n "${OCAML_COMPILER:-}" ]; then + # Validate OCAML_COMPILER (OCAML_COMPILER specified) + case "${OCAML_COMPILER:-}" in + 4.12.1) true ;; + *) + echo "OCAML_COMPILER version ${OCAML_COMPILER:-} is not supported" >&2 + exit 109 + ;; + esac + + section_begin checkout-dkml-base-compiler "Pin dkml-base-compiler (OCAML_COMPILER specified) for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action dkml-base-compiler "https://github.com/diskuv/dkml-compiler.git#${OCAML_COMPILER}-v${DKML_VERSION}" + section_end checkout-dkml-base-compiler + fi + + # conf-dkml-cross-toolchain + + if [ "${CONF_DKML_CROSS_TOOLCHAIN:-}" != '@repository@' ]; then + section_begin checkout-conf-dkml-cross-toolchain "Pin conf-dkml-cross-toolchain for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action conf-dkml-cross-toolchain "https://github.com/diskuv/conf-dkml-cross-toolchain.git#$CONF_DKML_CROSS_TOOLCHAIN" + section_end checkout-conf-dkml-cross-toolchain + fi + + # patches necessary for Windows in diskuv-opam-repository + # + # - ocamlfind and ocamlbuild + # + # - dune-configurator (and hence Dune) + # Dune 2.9.1 and 3.0.2 will fail to build jst-config.v0.14.1 because for jst-config/discover/discover.ml Dune does: + # cl -nologo -O2 -Gy- -MD -I Z:/.opam_root-cached-8/installer-ocaml/lib/ocaml -o C:\Users\beckf\AppData\Local\Temp\build_f18aec_dune\ocaml-configurator4d3858\c-test-31\test.obj -c C:\Users\beckf\AppData\Local\Temp\build_f18aec_dune\ocaml-configurator4d3858\c-test-31\test.c advapi32.lib ws2_32.lib version.lib + # instead of + # cl -nologo -O2 -Gy- -MD -I Z:/.opam_root-cached-8/installer-ocaml/lib/ocaml /FoC:\Users\beckf\AppData\Local\Temp\build_f18aec_dune\ocaml-configurator4d3858\c-test-31\test.obj -c C:\Users\beckf\AppData\Local\Temp\build_f18aec_dune\ocaml-configurator4d3858\c-test-31\test.c advapi32.lib ws2_32.lib version.lib + # with the (irrelevant) test.c file: + # #include + # #include + # + # #ifdef ARCH_BIG_ENDIAN + # const char *s0 = "BEGIN-0-true-END"; + # #else + # const char *s0 = "BEGIN-0-false-END"; + # #endif + # + # #ifdef ARCH_SIXTYFOUR + # const char *s1 = "BEGIN-1-true-END"; + # #else + # const char *s1 = "BEGIN-1-false-END"; + # #endif + # + # The actual problem is dune-configurator ... we only have patches in Diskuv + # repository up until 2.9.3. Need to upstream fix the problem. + # + # - ppx_expect; only patch is for v0.14.1. Need to upstream fix the problem. + # - base; patches for v0.14.1/2/3. Need to upstream fix the problem. + section_begin "opam-pins-$do_pins_NAME" "Opam pins for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version base "${PIN_BASE}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version bigstringaf "${PIN_BIGSTRINGAF}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version core_kernel "${PIN_CORE_KERNEL}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ctypes "${PIN_CTYPES}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ctypes-foreign "${PIN_CTYPES_FOREIGN}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version curly "${PIN_CURLY}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version digestif "${PIN_DIGESTIF}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version dkml-apps "${PIN_DKML_APPS}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version dune "${PIN_DUNE}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version dune-configurator "${PIN_DUNE_CONFIGURATOR}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ocamlbuild "${PIN_OCAMLBUILD}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ocamlfind "${PIN_OCAMLFIND}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ocp-indent "${PIN_OCP_INDENT}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ppx_expect "${PIN_PPX_EXPECT}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ptime "${PIN_PTIME}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version time_now "${PIN_TIME_NOW}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version with-dkml "${PIN_WITH_DKML}" + section_end "opam-pins-$do_pins_NAME" + } + + do_pins dkml + if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_pins two + fi + + do_use_vsstudio() { + do_use_vsstudio_NAME=$1 + shift + case "$dkml_host_abi" in + windows_*) + section_begin "use-vsstudio-$do_use_vsstudio_NAME" "Use Visual Studio in dkml-* Opam packages (Windows) for $do_use_vsstudio_NAME switch" + + # shellcheck disable=SC2153 + E_VS_DIR=$(escape_arg_as_ocaml_string "$VS_DIR") + # shellcheck disable=SC2153 + E_VS_VCVARSVER=$(escape_arg_as_ocaml_string "$VS_VCVARSVER") + # shellcheck disable=SC2153 + E_VS_WINSDKVER=$(escape_arg_as_ocaml_string "$VS_WINSDKVER") + # shellcheck disable=SC2153 + E_VS_MSVSPREFERENCE=$(escape_arg_as_ocaml_string "$VS_MSVSPREFERENCE") + # shellcheck disable=SC2153 + E_VS_CMAKEGENERATOR=$(escape_arg_as_ocaml_string "$VS_CMAKEGENERATOR") + + case "$(opamrun --version)" in + 2.0.*) + if [ "${in_docker}" = "true" ]; then + echo Opam 2.0 support in dockcross to use a portable opam var prefix not yet implemented + exit 67 + fi + OP=$(opamrun var prefix --switch "$do_use_vsstudio_NAME") + OPSC=$OP/.opam-switch/switch-config + if grep setenv: "$OPSC"; then + echo "INFO: Updating switch-config. Old was:" + awk '{print ">> " $0}' "$OPSC" + + awk '$1=="setenv:"{x=1} x==0{print} x==1 && $0=="]"{x=0}' "$OPSC" >"$OPSC".trimmed + mv "$OPSC".trimmed "$OPSC" + fi + echo 'setenv: [' >>"$OPSC" + echo ' [DKML_COMPILE_SPEC = "1"]' >>"$OPSC" + echo ' [DKML_COMPILE_TYPE = "VS"]' >>"$OPSC" + echo " [DKML_COMPILE_VS_DIR = \"$E_VS_DIR\"]" >>"$OPSC" + echo " [DKML_COMPILE_VS_VCVARSVER = \"$E_VS_VCVARSVER\"]" >>"$OPSC" + echo " [DKML_COMPILE_VS_WINSDKVER = \"$E_VS_WINSDKVER\"]" >>"$OPSC" + echo " [DKML_COMPILE_VS_MSVSPREFERENCE = \"$E_VS_MSVSPREFERENCE\"]" >>"$OPSC" + echo " [DKML_COMPILE_VS_CMAKEGENERATOR = \"$E_VS_CMAKEGENERATOR\"]" >>"$OPSC" + echo " [DKML_HOST_ABI = \"${dkml_host_abi}\"]" >>"$OPSC" + echo ']' >>"$OPSC" + cat "$OPSC" >&2 # print + ;; + *) + opamrun option --switch "$do_use_vsstudio_NAME" setenv= # reset + opamrun option --switch "$do_use_vsstudio_NAME" setenv+='DKML_COMPILE_SPEC = "1"' + opamrun option --switch "$do_use_vsstudio_NAME" setenv+='DKML_COMPILE_TYPE = "VS"' + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_DIR = \"$E_VS_DIR\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_VCVARSVER = \"$E_VS_VCVARSVER\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_WINSDKVER = \"$E_VS_WINSDKVER\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_MSVSPREFERENCE = \"$E_VS_MSVSPREFERENCE\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_CMAKEGENERATOR = \"$E_VS_CMAKEGENERATOR\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_HOST_ABI = \"${dkml_host_abi}\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv # print + ;; + esac + + # shellcheck disable=SC2016 + opamrun exec --switch "$do_use_vsstudio_NAME" -- sh -c 'echo $VCToolsRedistDir' + + section_end "use-vsstudio-$do_use_vsstudio_NAME" + ;; + esac + } + if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_use_vsstudio two + fi + do_use_vsstudio dkml + + # Because dune.X.Y.Z+shim (and any user DKML packages) requires DKML installed (after all, it is just + # a with-dkml.exe shim), we need either dkmlvars-v2.sexp or DKML environment + # variables. Confer: Dkml_runtimelib.Dkml_context.get_dkmlversion + # + # grep matches either: + # [... [DiskuvOCamlVersion = "1.0.1"] ...] + # DiskuvOCamlVersion = "1.0.1" + do_setenv() { + do_setenv_SWITCH=$1 + shift + opamrun option --switch "$do_setenv_SWITCH" setenv > ".ci/sd4/setenv.$do_setenv_SWITCH.txt" + if ! grep -q '\(^|\[\)DiskuvOCamlVarsVersion ' ".ci/sd4/setenv.$do_setenv_SWITCH.txt"; then + opamrun option --switch "$do_setenv_SWITCH" setenv+='DiskuvOCamlVarsVersion = "2"' + fi + if ! grep -q '\(^|\[\)DiskuvOCamlVersion ' ".ci/sd4/setenv.$do_setenv_SWITCH.txt"; then + opamrun option --switch "$do_setenv_SWITCH" setenv+="DiskuvOCamlVersion = \"$DKML_VERSION\"" + fi + case "${dkml_host_abi}" in + windows_*) + if ! grep -q '\(^|\[\)DiskuvOCamlMSYS2Dir ' ".ci/sd4/setenv.$do_setenv_SWITCH.txt"; then + if [ -x /usr/bin/cygpath ]; then + MSYS2_DIR_NATIVE=$(/usr/bin/cygpath -aw /) + else + # If we are already inside MSYS2 then MSYSTEM_PREFIX should be set. But cygpath should be there as well!! + echo "FATAL: Could not locate MSYS2: there was no cygpath" >&2 + exit 3 + fi + MSYS2_DIR_NATIVE_ESCAPED=$(printf "%s" "$MSYS2_DIR_NATIVE" | sed 's/\\/\\\\/g') + opamrun option --switch "$do_setenv_SWITCH" setenv+="DiskuvOCamlMSYS2Dir = \"$MSYS2_DIR_NATIVE_ESCAPED\"" + fi + esac + } + if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_setenv two + fi + do_setenv dkml + + do_install_compiler() { + do_install_compiler_NAME=$1 + shift + section_begin "install-compiler-$do_install_compiler_NAME" "Install OCaml compiler for $do_install_compiler_NAME switch" + opamrun pin list --switch "$do_install_compiler_NAME" + # shellcheck disable=SC2086 + opamrun upgrade --switch "$do_install_compiler_NAME" --yes dkml-base-compiler conf-dkml-cross-toolchain ${ocaml_options:-} + section_end "install-compiler-$do_install_compiler_NAME" + } + do_install_compiler dkml + if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_install_compiler two + fi + + do_summary() { + do_summary_NAME=$1 + shift + section_begin "summary-$do_summary_NAME" "Summary: $do_summary_NAME switch" + opamrun var --switch "$do_summary_NAME" + opamrun exec --switch "$do_summary_NAME" -- ocamlc -config + section_end "summary-$do_summary_NAME" + } + do_summary dkml + if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_summary two + fi + + end_of_script + + cat > .ci/sd4/run-teardown-dkml.sh <<'end_of_script' + #!/bin/sh + set -euf + + teardown_WORKSPACE_VARNAME=$1 + shift + teardown_WORKSPACE=$1 + shift + + # ------------------ Variables and functions ------------------------ + + # shellcheck source=./common-values.sh + . .ci/sd4/common-values.sh + + # Fixup opam_root on Windows to be mixed case. Set original_* and unix_* as well. + fixup_opam_root + + # Set TEMP variable for Windows + export_temp_for_windows + + # ------------------------------------------------------------------- + + section_begin teardown-info "Summary: teardown-dkml" + + # shellcheck disable=SC2154 + echo " + ================ + teardown-dkml.sh + ================ + . + --------- + Arguments + --------- + WORKSPACE_VARNAME=$teardown_WORKSPACE_VARNAME + WORKSPACE=$teardown_WORKSPACE + . + ------ + Inputs + ------ + VERBOSE=${VERBOSE:-} + . + ------ + Matrix + ------ + dkml_host_abi=$dkml_host_abi + opam_root=${opam_root} + opam_root_cacheable=${opam_root_cacheable} + original_opam_root=${original_opam_root} + original_opam_root_cacheable=${original_opam_root_cacheable} + unix_opam_root=${unix_opam_root} + unix_opam_root_cacheable=${unix_opam_root_cacheable} + . + " + section_end teardown-info + + # Done with Opam cache! + do_save_opam_cache() { + if [ "$unix_opam_root_cacheable" = "$unix_opam_root" ]; then return; fi + section_begin save-opam-cache "Transfer Opam cache to $original_opam_root" + echo Starting transfer # need some output or GitLab CI will not display the section duration + transfer_dir "$unix_opam_root" "$unix_opam_root_cacheable" + echo Finished transfer + section_end save-opam-cache + } + do_save_opam_cache + + do_fill_skipped_cache_entries() { + section_begin fill-skipped-cache-entries "Populate skipped cache entries" + + # Needed to stop GitLab CI/CD cache warnings 'no matching files', etc. + if [ ! -e .ci/sd4/vsenv.sh ]; then + install -d .ci/sd4 + rm -f .ci/sd4/vsenv.sh + touch .ci/sd4/vsenv.sh + chmod +x .ci/sd4/vsenv.sh + echo "Created empty vsenv.sh" + else + echo "Found vsenv.sh" + fi + + if [ ! -e msys64 ]; then + rm -rf msys64 + install -d msys64 + touch msys64/.keep + echo "Created empty msys64" + else + echo "Found msys64/" + fi + + if [ -s "$unix_opam_root_cacheable/.ci.dkml.repo-init" ]; then + echo "Found non-empty $unix_opam_root_cacheable/.ci.dkml.repo-init" + else + touch "$unix_opam_root_cacheable/.ci.dkml.repo-init" + echo "Created empty $unix_opam_root_cacheable/.ci.dkml.repo-init" + fi + + if [ -s "$unix_opam_root_cacheable/.ci.two.repo-init" ]; then + echo "Found non-empty $unix_opam_root_cacheable/.ci.two.repo-init" + else + touch "$unix_opam_root_cacheable/.ci.two.repo-init" + echo "Created empty $unix_opam_root_cacheable/.ci.two.repo-init" + fi + + section_end fill-skipped-cache-entries + } + do_fill_skipped_cache_entries + + do_at_least_one_artifact() { + install -d dist + find dist -mindepth 1 -maxdepth 1 >.ci/dist.files + if [ ! -s .ci/dist.files ]; then + section_begin one-artifact "Create empty artifact file" + + # Avoid confusing "ERROR: No files to upload" in GitLab CI + touch dist/.keep + echo "Created dist/.keep" + + section_end one-artifact + fi + } + do_at_least_one_artifact + + end_of_script + + - name: Prepare cache keys + # An undocumented bug with GitHub Actions is that a space in the key + # will "succeed" but it never gets restored. So we will hash a + # user-friendly file instead. + shell: bash + run: | + set -x + echo '317446-${{ inputs.FDOPEN_OPAMEXE_BOOTSTRAP }}-${{ steps.full_matrix_vars.outputs.dkml_host_abi }}-${{ steps.full_matrix_vars.outputs.opam_abi }}-${{ steps.full_matrix_vars.outputs.bootstrap_opam_version }}' > .ci/sd4/cachekey.opam.binaries + echo '${{ steps.full_matrix_vars.outputs.abi_pattern }}-${{ steps.full_matrix_vars.outputs.vsstudio_arch }}-${{ steps.full_matrix_vars.outputs.vsstudio_hostarch }}-${{ steps.full_matrix_vars.outputs.vsstudio_dir }}-${{ steps.full_matrix_vars.outputs.vsstudio_vcvarsver }}-${{ steps.full_matrix_vars.outputs.vsstudio_winsdkver }}-${{ steps.full_matrix_vars.outputs.vsstudio_msvspreference }}-${{ steps.full_matrix_vars.outputs.vsstudio_cmakegenerator }}' > .ci/sd4/cachekey.vsstudio + echo '${{ inputs.OCAML_COMPILER }}-${{ inputs.DISKUV_OPAM_REPOSITORY }}-${{ inputs.DKML_COMPILER }}-${{ inputs.CONF_DKML_CROSS_TOOLCHAIN }}' > .ci/sd4/cachekey.ci.inputs + + # Bootstrapping Opam + + - name: Cache Opam bootstrap by OS + uses: actions/cache@v3 + id: cache-sd4-bs + with: + path: .ci/sd4/bs + key: + "1_${{ inputs.CACHE_PREFIX }}_${{ runner.os }}-sd4-bs-${{ hashFiles('.ci/sd4/cachekey.opam.binaries') }}" + + - name: Cache Opam root by (host,target) + uses: actions/cache@v3 + id: cache-sd4-opamroot + with: + path: | + ${{ steps.full_matrix_vars.outputs.opam_root_cacheable }}/config + ${{ steps.full_matrix_vars.outputs.opam_root_cacheable }}/dkml + ${{ steps.full_matrix_vars.outputs.opam_root_cacheable }}/two + ${{ steps.full_matrix_vars.outputs.opam_root_cacheable }}/repo + ${{ steps.full_matrix_vars.outputs.opam_root_cacheable }}/download-cache + ${{ steps.full_matrix_vars.outputs.opam_root_cacheable }}/.ci.root-init + ${{ steps.full_matrix_vars.outputs.opam_root_cacheable }}/.ci.dkml.repo-init + ${{ steps.full_matrix_vars.outputs.opam_root_cacheable }}/.ci.two.repo-init + key: + "1_${{ inputs.CACHE_PREFIX }}_${{ env.DKML_VERSION }}_${{ steps.full_matrix_vars.outputs.abi_pattern }}-sd4-opamroot-${{ hashFiles('.ci/sd4/cachekey.opam.binaries') }}-${{ hashFiles('cachekey.ci.inputs') }}" + + # The action/checkout steps used for pinning will re-use existing Git objects + # because of caching + - name: Cache Git checkouts of Opam pins by OS + uses: actions/cache@v3 + id: cache-sd4-git + with: + path: .ci/sd4/g + key: + "1_${{ inputs.CACHE_PREFIX }}_${{ runner.os }}-sd4-git-${{ hashFiles('.ci/sd4/cachekey.ci.inputs') }}" + + # Checkout code + + - name: Checkout DKML code + shell: bash + env: + # Push down inputs variables for GitHub/GitLab portable scripts + VERBOSE: "${{ inputs.VERBOSE }}" + run: 'sh .ci/sd4/run-checkout-code.sh GITHUB_WORKSPACE "$GITHUB_WORKSPACE"' + + # Setup C compiler + + - name: Compile 'dkml' switch + shell: bash + env: + # Push down inputs variables for GitHub/GitLab portable scripts + # When non-empty, instead of building the standard components from the + # central Opam repository, use the github/gitlab development repositories + # directly. + FDOPEN_OPAMEXE_BOOTSTRAP: "${{ inputs.FDOPEN_OPAMEXE_BOOTSTRAP }}" + CONF_DKML_CROSS_TOOLCHAIN: "${{ inputs.CONF_DKML_CROSS_TOOLCHAIN }}" + DISKUV_OPAM_REPOSITORY: "${{ inputs.DISKUV_OPAM_REPOSITORY }}" + DKML_COMPILER: "${{ inputs.DKML_COMPILER }}" + OCAML_COMPILER: "${{ inputs.OCAML_COMPILER }}" + SECONDARY_SWITCH: "${{ inputs.SECONDARY_SWITCH }}" + VERBOSE: "${{ inputs.VERBOSE }}" + + # autogen from global_env_vars. + DEFAULT_DKML_COMPILER: '4.12.1-v1.0.2' + PIN_BASE: 'v0.14.3' + PIN_BIGSTRINGAF: '0.8.0' + PIN_CORE_KERNEL: 'v0.14.2' + PIN_CTYPES_FOREIGN: '0.19.2-windowssupport-r4' + PIN_CTYPES: '0.19.2-windowssupport-r4' + PIN_CURLY: '0.2.1-windows-env_r2' + PIN_DIGESTIF: '1.0.1' + PIN_DUNE: '2.9.3+shim.1.0.2~r0' + PIN_DUNE_CONFIGURATOR: '2.9.3' + PIN_DKML_APPS: '1.0.1' + PIN_OCAMLBUILD: '0.14.0' + PIN_OCAMLFIND: '1.9.1' + PIN_OCP_INDENT: '1.8.2-windowssupport' + PIN_PPX_EXPECT: 'v0.14.1' + PIN_PTIME: '0.8.6-msvcsupport' + PIN_TIME_NOW: 'v0.14.0' + PIN_WITH_DKML: '1.0.1' + run: | + sh .ci/sd4/run-setup-dkml.sh GITHUB_WORKSPACE "$GITHUB_WORKSPACE" + + # Expose opamrun to remaining GitHub Action steps + opamrunabs="$GITHUB_WORKSPACE/.ci/sd4/opamrun" + if [ -x /usr/bin/cygpath ]; then opamrunabs=$(/usr/bin/cygpath -aw "$opamrunabs"); fi + echo "$opamrunabs" >> $GITHUB_PATH diff --git a/ci/setup-dkml/gh-darwin/pre/dune b/ci/setup-dkml/gh-darwin/pre/dune new file mode 100644 index 0000000..a0e1f8e --- /dev/null +++ b/ci/setup-dkml/gh-darwin/pre/dune @@ -0,0 +1,17 @@ +; DO NOT EDIT THIS FILE. It is auto-generated by generate-setup-dkml-scaffold +; Typical upgrade steps: +; opam upgrade dkml-workflows && opam exec -- generate-setup-dkml-scaffold && dune build '@gen-dkml' --auto-promote + +(rule + (alias gen-dkml) + (target action.gen.yml) + (action + (setenv + OCAMLRUNPARAM + b + (run gh-dkml-action-yml --phase pre --output-darwin %{target})))) + +(rule + (alias gen-dkml) + (action + (diff action.yml action.gen.yml))) diff --git a/ci/setup-dkml/gh-linux/post/action.yml b/ci/setup-dkml/gh-linux/post/action.yml new file mode 100644 index 0000000..7123c1e --- /dev/null +++ b/ci/setup-dkml/gh-linux/post/action.yml @@ -0,0 +1,91 @@ +# setup-dkml +# Short form: sd4 + +name: post-dkml +author: Diskuv, Inc. +description: Teardown after building OCaml native executables for Linux + +runs: + using: "composite" + + steps: + - name: Full matrix variables + shell: bash + # Every matrix variable lookup in this Action should use the output of this step. Even + # the matrix variables that the user must specify (ex. dkml_host_abi) should be + # referenced using [steps.full_matrix_vars.outputs.dkml_host_abi] rather than + # [matrix.dkml_host_abi] so that there is a single place to edit for variable changes. + id: full_matrix_vars + run: | + # Select correct Linux matrix variables + case "${{ matrix.dkml_host_abi }}" in + + linux_x86) + dkml_host_os='linux'; + opam_root_cacheable='.ci/o'; + abi_pattern='manylinux2014-linux_x86'; + gh_os='ubuntu-latest'; + comment='(CentOS 7, etc.)'; + gh_unix_shell='sh'; + bootstrap_opam_version='2.2.0-dkml20220801T155940Z'; + dkml_host_abi='linux_x86'; + opam_root='.ci/o'; + in_docker='true'; + dockcross_image='dockcross/manylinux2014-x86'; + dockcross_run_extra_args='--platform linux/386' ;; + + linux_x86_64) + dkml_host_os='linux'; + opam_root_cacheable='.ci/o'; + abi_pattern='manylinux2014-linux_x86_64'; + gh_os='ubuntu-latest'; + comment='(CentOS 7, etc.)'; + gh_unix_shell='sh'; + bootstrap_opam_version='2.2.0-dkml20220801T155940Z'; + dkml_host_abi='linux_x86_64'; + opam_root='.ci/o'; + dockcross_image='dockcross/manylinux2014-x64'; + dockcross_run_extra_args='--platform linux/amd64'; + in_docker='true' ;; + + *) echo "FATAL: Unsupported dkml_host_abi=$dkml_host_abi in Linux action.yml"; exit 107 ;; + esac + + add() { + echo "$1=$2" | tee -a $GITHUB_OUTPUT | tee -a $GITHUB_ENV + } + + add dkml_host_abi "$dkml_host_abi" + add abi_pattern "$abi_pattern" + add opam_root "$opam_root" + add opam_root_cacheable "$opam_root_cacheable" + add exe_ext "${exe_ext:-}" + add bootstrap_opam_version "${bootstrap_opam_version:-}" + add ocaml_options "${ocaml_options:-}" + + - name: Teardown DKML build apparatus + shell: bash + env: + _STUB_FOR_AUTOGEN: "ON" # + + # autogen from global_env_vars. + DEFAULT_DKML_COMPILER: '4.12.1-v1.0.2' + PIN_BASE: 'v0.14.3' + PIN_BIGSTRINGAF: '0.8.0' + PIN_CORE_KERNEL: 'v0.14.2' + PIN_CTYPES_FOREIGN: '0.19.2-windowssupport-r4' + PIN_CTYPES: '0.19.2-windowssupport-r4' + PIN_CURLY: '0.2.1-windows-env_r2' + PIN_DIGESTIF: '1.0.1' + PIN_DUNE: '2.9.3+shim.1.0.2~r0' + PIN_DUNE_CONFIGURATOR: '2.9.3' + PIN_DKML_APPS: '1.0.1' + PIN_OCAMLBUILD: '0.14.0' + PIN_OCAMLFIND: '1.9.1' + PIN_OCP_INDENT: '1.8.2-windowssupport' + PIN_PPX_EXPECT: 'v0.14.1' + PIN_PTIME: '0.8.6-msvcsupport' + PIN_TIME_NOW: 'v0.14.0' + PIN_WITH_DKML: '1.0.1' + run: | + sh .ci/sd4/run-teardown-dkml.sh GITHUB_WORKSPACE "$GITHUB_WORKSPACE" diff --git a/ci/setup-dkml/gh-linux/post/dune b/ci/setup-dkml/gh-linux/post/dune new file mode 100644 index 0000000..e4e55e6 --- /dev/null +++ b/ci/setup-dkml/gh-linux/post/dune @@ -0,0 +1,17 @@ +; DO NOT EDIT THIS FILE. It is auto-generated by generate-setup-dkml-scaffold +; Typical upgrade steps: +; opam upgrade dkml-workflows && opam exec -- generate-setup-dkml-scaffold && dune build '@gen-dkml' --auto-promote + +(rule + (alias gen-dkml) + (target action.gen.yml) + (action + (setenv + OCAMLRUNPARAM + b + (run gh-dkml-action-yml --phase post --output-linux %{target})))) + +(rule + (alias gen-dkml) + (action + (diff action.yml action.gen.yml))) diff --git a/ci/setup-dkml/gh-linux/pre/action.yml b/ci/setup-dkml/gh-linux/pre/action.yml new file mode 100644 index 0000000..dfc9732 --- /dev/null +++ b/ci/setup-dkml/gh-linux/pre/action.yml @@ -0,0 +1,1691 @@ +# setup-dkml +# Short form: sd4 + +name: pre-dkml +author: Diskuv, Inc. +description: Setup before building OCaml native executables for Linux + +inputs: + FDOPEN_OPAMEXE_BOOTSTRAP: + description: "Use opam.exe from fdopen on Windows. Typically only used when bootstrapping Opam for the first time. May be needed to solve '\"create_process\" failed on sleep: Bad file descriptor' which may need https://github.com/ocaml/opam/commit/417b97d8cfada35682a0f4107eb2e4f9e24fba91" + required: false + default: 'false' + CACHE_PREFIX: + description: The prefix of the cache keys. + required: false + default: "v1" + OCAML_COMPILER: + description: "The version of the OCaml compiler, as tagged by the https://github.com/diskuv/dkml-base-compiler repository. If not specified the latest compiler, or the version associated with the 'dkml-compiler' input, is used. Currently only 4.12.1 is supported" + required: false + DKML_COMPILER: + description: "Git branch, tag or commit for dkml-compiler. However if OCAML_COMPILER is specified the 'dkml-base-compiler' version is taken from OCAML_COMPILER." + required: false + default: "" # "@repository@" = Opam ; "" = latest from default branch of git clone + CONF_DKML_CROSS_TOOLCHAIN: + description: "Git branch, tag or commit for conf-dkml-cross-toolchain" + required: false + default: "@repository@" # "@repository@" = Opam ; "" = latest from default branch of git clone + DISKUV_OPAM_REPOSITORY: + description: "Git branch, tag or commit for diskuv-opam-repository" + required: false + default: "" # DEFAULT_DISKUV_OPAM_REPOSITORY_TAG is used as default for empty strings + VERBOSE: + description: "Turn on verbose logging" + required: false + default: 'false' + SECONDARY_SWITCH: + description: "Turn on secondary 'two' switch" + required: false + default: 'false' + MANYLINUX: + description: "Use a ManyLinux dockcross Docker container to generate relatively portable Linux executables without static linking. Docker is required" + required: false + default: 'true' + +runs: + using: "composite" + + steps: + - name: Full matrix variables + shell: bash + # Every matrix variable lookup in this Action should use the output of this step. Even + # the matrix variables that the user must specify (ex. dkml_host_abi) should be + # referenced using [steps.full_matrix_vars.outputs.dkml_host_abi] rather than + # [matrix.dkml_host_abi] so that there is a single place to edit for variable changes. + id: full_matrix_vars + run: | + # Select correct Linux matrix variables + case "${{ matrix.dkml_host_abi }}" in + + linux_x86) + dkml_host_os='linux'; + opam_root_cacheable='.ci/o'; + abi_pattern='manylinux2014-linux_x86'; + gh_os='ubuntu-latest'; + comment='(CentOS 7, etc.)'; + gh_unix_shell='sh'; + bootstrap_opam_version='2.2.0-dkml20220801T155940Z'; + dkml_host_abi='linux_x86'; + opam_root='.ci/o'; + in_docker='true'; + dockcross_image='dockcross/manylinux2014-x86'; + dockcross_run_extra_args='--platform linux/386' ;; + + linux_x86_64) + dkml_host_os='linux'; + opam_root_cacheable='.ci/o'; + abi_pattern='manylinux2014-linux_x86_64'; + gh_os='ubuntu-latest'; + comment='(CentOS 7, etc.)'; + gh_unix_shell='sh'; + bootstrap_opam_version='2.2.0-dkml20220801T155940Z'; + dkml_host_abi='linux_x86_64'; + opam_root='.ci/o'; + dockcross_image='dockcross/manylinux2014-x64'; + dockcross_run_extra_args='--platform linux/amd64'; + in_docker='true' ;; + + *) echo "FATAL: Unsupported dkml_host_abi=$dkml_host_abi in Linux action.yml"; exit 107 ;; + esac + + add() { + echo "$1=$2" | tee -a $GITHUB_OUTPUT | tee -a $GITHUB_ENV + } + + add dkml_host_abi "$dkml_host_abi" + add abi_pattern "$abi_pattern" + add opam_root "$opam_root" + add opam_root_cacheable "$opam_root_cacheable" + add exe_ext "${exe_ext:-}" + add bootstrap_opam_version "${bootstrap_opam_version:-}" + add ocaml_options "${ocaml_options:-}" + + add dockcross_image "${dockcross_image:-}" + add dockcross_run_extra_args "${dockcross_run_extra_args:-}" + add docker_runner "${docker_runner:-}" + add in_docker "${in_docker:-}" + + # GitHub Actions uses Docker Buildx, so use custom 'dkml-dockcross/*' cached image which speeds up builds + add dockcross_image_custom_prefix "dkml-" + + - name: Write POSIX shell and AWK scripts + shell: bash + run: | + install -d .ci/sd4 + + cat > .ci/sd4/common-values.sh <<'end_of_script' + #!/bin/sh + + # ------------------------ Log Formatting ------------------------ + + TXT_SECTION="\e[94m" # bright blue + TXT_CLEAR="\e[0m" + + if [ "${GITLAB_CI:-}" = "true" ]; then + # https://docs.gitlab.com/ee/ci/jobs/#expand-and-collapse-job-log-sections + print_section_start() { + print_section_start_NAME=$1 + shift + printf "\e[0Ksection_start:%s:%s[collapsed=true]\r\e[0K" \ + "$(date +%s)" \ + "$print_section_start_NAME" + } + print_section_end() { + print_section_end_NAME=$1 + shift + printf "\e[0Ksection_end:%s:%s\r\e[0K\n" \ + "$(date +%s)" \ + "$print_section_end_NAME" + } + elif [ -n "${GITHUB_ENV:-}" ]; then + # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines + print_section_start() { + print_section_start_NAME=$1 + shift + printf "::group::" + } + print_section_end() { + print_section_end_NAME=$1 + shift + printf "::endgroup::\n" + } + else + print_section_start() { + print_section_start_NAME=$1 + shift + } + print_section_end() { + print_section_end_NAME=$1 + shift + } + fi + + section_begin() { + # https://docs.gitlab.com/ee/ci/yaml/script.html#add-color-codes-to-script-output + section_NAME=$1 + shift + section_HEADER=$1 + shift + print_section_start "$section_NAME" + printf "${TXT_SECTION}%s${TXT_CLEAR}\n" "$section_HEADER" + } + + section_end() { + section_NAME=$1 + shift + print_section_end "$section_NAME" + } + + # ------------------- Other Functions ----------------- + + transfer_dir() { + transfer_dir_SRC=$1 + shift + transfer_dir_DST=$1 + shift + # Remove the destination directory completely, but make sure the parent of the + # destination directory exists so `mv` will work + install -d "$transfer_dir_DST" + rm -rf "$transfer_dir_DST" + # Move + mv "$transfer_dir_SRC" "$transfer_dir_DST" + } + + # Set TEMP variable which is used, among other things, for OCaml's + # [Filename.temp_dir_name] on Win32, and by with-dkml.exe on Windows + export_temp_for_windows() { + if [ -x /usr/bin/cygpath ]; then + if [ -n "${RUNNER_TEMP:-}" ]; then + # GitHub Actions + TEMP=$(cygpath -am "$RUNNER_TEMP") + else + # GitLab CI/CD or desktop + install -d .ci/tmp + TEMP=$(cygpath -am ".ci/tmp") + fi + export TEMP + fi + } + + # Fixup opam_root on Windows to be mixed case. + # On input the following variables must be present: + # - opam_root + # - opam_root_cacheable + # On output the input variables will be modified _and_ the + # following variables will be available: + # - original_opam_root + # - original_opam_root_cacheable + # - unix_opam_root + # - unix_opam_root_cacheable + fixup_opam_root() { + # shellcheck disable=SC2034 + original_opam_root=$opam_root + # shellcheck disable=SC2034 + original_opam_root_cacheable=$opam_root_cacheable + if [ -x /usr/bin/cygpath ]; then + opam_root=$(/usr/bin/cygpath -m "$opam_root") + opam_root_cacheable=$(/usr/bin/cygpath -m "$opam_root_cacheable") + unix_opam_root=$(/usr/bin/cygpath -u "$opam_root") + unix_opam_root_cacheable=$(/usr/bin/cygpath -u "$opam_root_cacheable") + else + # shellcheck disable=SC2034 + unix_opam_root=$opam_root + # shellcheck disable=SC2034 + unix_opam_root_cacheable=$opam_root_cacheable + fi + } + end_of_script + + cat > .ci/sd4/run-checkout-code.sh <<'end_of_script' + #!/bin/sh + + # ================ + # checkout-code.sh + # ================ + # + # Checkouts all of the git source code. + # + # This should be done outside of + # dockcross (used by Linux) since a Docker-in-Docker container can have + # difficulties doing a git checkout (the Git credentials for any private + # repositories are likely not present). We don't care about any private + # repositories for DKML but any code that extends this (ex. DKSDK) may + # need to use private repositories. + + set -euf + + setup_WORKSPACE_VARNAME=$1 + shift + setup_WORKSPACE=$1 + shift + + # ------------------------ Functions ------------------------ + + # shellcheck source=./common-values.sh + . .ci/sd4/common-values.sh + + # Disable automatic garbage collection + git_disable_gc() { + git_disable_gc_NAME=$1 + shift + git -C ".ci/sd4/g/$git_disable_gc_NAME" config --local gc.auto 0 + } + + # Mimic the behavior of GitHub's actions/checkout@v3 + # - the plus symbol in 'git fetch ... origin +REF:refs/tags/v0.0' overrides any existing REF + git_checkout() { + git_checkout_NAME=$1 + shift + git_checkout_URL=$1 + shift + git_checkout_REF=$1 + shift + + if [ -e ".ci/sd4/g/$git_checkout_NAME" ]; then + git_disable_gc "$git_checkout_NAME" + git -C ".ci/sd4/g/$git_checkout_NAME" remote set-url origin "$git_checkout_URL" + git -C ".ci/sd4/g/$git_checkout_NAME" fetch --no-tags --progress --no-recurse-submodules --depth=1 origin "+${git_checkout_REF}:refs/tags/v0.0" + else + install -d ".ci/sd4/g/$git_checkout_NAME" + git -C ".ci/sd4/g/$git_checkout_NAME" -c init.defaultBranch=main init + git_disable_gc "$git_checkout_NAME" + git -C ".ci/sd4/g/$git_checkout_NAME" remote add origin "$git_checkout_URL" + git -C ".ci/sd4/g/$git_checkout_NAME" fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin "+${git_checkout_REF}:refs/tags/v0.0" + fi + git -C ".ci/sd4/g/$git_checkout_NAME" -c advice.detachedHead=false checkout --progress --force refs/tags/v0.0 + git -C ".ci/sd4/g/$git_checkout_NAME" log -1 --format='%H' + } + + # --------------------------------------------------------------------- + + section_begin checkout-info "Summary: code checkout" + + # shellcheck disable=SC2154 + echo " + ================ + checkout-code.sh + ================ + . + --------- + Arguments + --------- + WORKSPACE_VARNAME=$setup_WORKSPACE_VARNAME + WORKSPACE=$setup_WORKSPACE + . + ------ + Inputs + ------ + VERBOSE=${VERBOSE:-} + . + ------ + Matrix + ------ + dkml_host_abi=$dkml_host_abi + . + " + + section_end checkout-info + + install -d .ci/sd4/g + + # dkml-runtime-distribution + + case "$dkml_host_abi" in + windows_*) + section_begin checkout-dkml-runtime-distribution 'Checkout dkml-runtime-distribution' + git_checkout dkml-runtime-distribution https://github.com/diskuv/dkml-runtime-distribution.git "1a3ec82dd851751a95e6a4797387a8163c51520e" + section_end checkout-dkml-runtime-distribution + ;; + esac + + end_of_script + + cat > .ci/sd4/run-setup-dkml.sh <<'end_of_script' + #!/bin/sh + set -euf + + # Constants + SHA512_DEVNULL='cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e' + # Edited by https://gitlab.com/diskuv/diskuv-ocaml/contributors/release.sh + DEFAULT_DISKUV_OPAM_REPOSITORY_TAG=cc9518fa5630bfbe24d4c5e0e2cc29af513037ce + # Constants + # Should be edited by release.sh, but ... + # Can't be 1.0.0 or later until https://github.com/ocaml/opam-repository/pull/21704 ocaml-option-32bit + # can come back in. + DKML_VERSION=0.4.0 + + setup_WORKSPACE_VARNAME=$1 + shift + setup_WORKSPACE=$1 + shift + + # ------------------ Variables and functions ------------------------ + + # shellcheck source=./common-values.sh + . .ci/sd4/common-values.sh + + if [ "${VERBOSE:-}" = "true" ]; then + do_tar_rf() { + tar rvf "$@" + } + else + do_tar_rf() { + tar rf "$@" + } + fi + + # Make the standard input work as an OCaml string. + # This currently only escapes backslashes and double quotes. + escape_arg_as_ocaml_string() { + escape_arg_as_ocaml_string_ARG=$1 + shift + printf "%s" "$escape_arg_as_ocaml_string_ARG" | sed 's#\\#\\\\#g; s#"#\\"#g;' + } + + # Fixup opam_root on Windows to be mixed case. Set original_* and unix_* as well. + fixup_opam_root + + # Set TEMP variable for Windows + export_temp_for_windows + + # Load VS studio environment + if [ -e .ci/sd4/vsenv.sh ]; then + # shellcheck disable=SC1091 + . .ci/sd4/vsenv.sh + fi + + # ------------------------------------------------------------------- + + section_begin setup-info "Summary: setup-dkml" + + # shellcheck disable=SC2154 + echo " + ============= + setup-dkml.sh + ============= + . + --------- + Arguments + --------- + WORKSPACE_VARNAME=$setup_WORKSPACE_VARNAME + WORKSPACE=$setup_WORKSPACE + . + ------ + Inputs + ------ + FDOPEN_OPAMEXE_BOOTSTRAP=${FDOPEN_OPAMEXE_BOOTSTRAP:-} + DISKUV_OPAM_REPOSITORY=${DISKUV_OPAM_REPOSITORY:-} + DKML_COMPILER=${DKML_COMPILER:-} + OCAML_COMPILER=${OCAML_COMPILER:-} + CONF_DKML_CROSS_TOOLCHAIN=${CONF_DKML_CROSS_TOOLCHAIN:-} + SECONDARY_SWITCH=${SECONDARY_SWITCH:-} + MANYLINUX=${MANYLINUX:-} + VERBOSE=${VERBOSE:-} + . + ------------------- + Generated Constants + ------------------- + DKML_VERSION=$DKML_VERSION + DEFAULT_DISKUV_OPAM_REPOSITORY_TAG=$DEFAULT_DISKUV_OPAM_REPOSITORY_TAG + DEFAULT_DKML_COMPILER=$DEFAULT_DKML_COMPILER + . + ------ + Matrix + ------ + dkml_host_abi=$dkml_host_abi + bootstrap_opam_version=$bootstrap_opam_version + abi_pattern=$abi_pattern + opam_root=${opam_root} + opam_root_cacheable=${opam_root_cacheable} + original_opam_root=${original_opam_root} + original_opam_root_cacheable=${original_opam_root_cacheable} + unix_opam_root=${unix_opam_root} + unix_opam_root_cacheable=${unix_opam_root_cacheable} + dockcross_image=${dockcross_image:-} + dockcross_image_custom_prefix=${dockcross_image_custom_prefix:-} + dockcross_run_extra_args=${dockcross_run_extra_args:-} + docker_runner=${docker_runner:-} + in_docker=${in_docker:-} + ocaml_options=${ocaml_options:-} + . + ---- + Pins + ---- + PIN_BASE=${PIN_BASE} + PIN_BIGSTRINGAF=${PIN_BIGSTRINGAF} + PIN_CORE_KERNEL=${PIN_CORE_KERNEL} + PIN_CTYPES=${PIN_CTYPES} + PIN_CTYPES_FOREIGN=${PIN_CTYPES_FOREIGN} + PIN_CURLY=${PIN_CURLY} + PIN_DIGESTIF=${PIN_DIGESTIF} + PIN_DKML_APPS=${PIN_DKML_APPS} + PIN_DUNE=${PIN_DUNE} + PIN_DUNE_CONFIGURATOR=${PIN_DUNE_CONFIGURATOR} + PIN_OCAMLBUILD=${PIN_OCAMLBUILD} + PIN_OCAMLFIND=${PIN_OCAMLFIND} + PIN_OCP_INDENT=${PIN_OCP_INDENT} + PIN_PPX_EXPECT=${PIN_PPX_EXPECT} + PIN_PTIME=${PIN_PTIME} + PIN_TIME_NOW=${PIN_TIME_NOW} + PIN_WITH_DKML=${PIN_WITH_DKML} + . + " + case "$dkml_host_abi" in + windows_*) + # shellcheck disable=SC2153 + echo " + ------------- + Visual Studio + ------------- + VS_DIR=$VS_DIR + VS_VCVARSVER=$VS_VCVARSVER + VS_WINSDKVER=$VS_WINSDKVER + VS_MSVSPREFERENCE=$VS_MSVSPREFERENCE + VS_CMAKEGENERATOR=$VS_CMAKEGENERATOR + . + " + ;; + esac + section_end setup-info + + do_bootstrap() { + # Bootstrap from historical release + runit_BOOTSTRAPPED=0 + + # Bootstrap opam from fdopen (Windows) + if [ "$runit_BOOTSTRAPPED" = 0 ] && [ "${FDOPEN_OPAMEXE_BOOTSTRAP:-}" = "true" ]; then + if [ -e .ci/sd4/opam64/bin/opam.exe ] && [ -e .ci/sd4/opam64/bin/opam-installer.exe ]; then + runit_BOOTSTRAPPED=1 + else + case "$dkml_host_abi" in + windows_*) + echo 'Bootstrap opam from fdopen (Windows) ...' + install -d .ci/sd4/bs/bin + wget -O "$setup_WORKSPACE"/.ci/sd4/opam64.tar.xz https://github.com/fdopen/opam-repository-mingw/releases/download/0.0.0.2/opam64.tar.xz + + # this stalls: tar xvCfJ "$setup_WORKSPACE"/.ci/sd4 "$setup_WORKSPACE"/.ci/sd4/opam64.tar.xz + xz -v -d "$setup_WORKSPACE"/.ci/sd4/opam64.tar.xz + tar xvCf .ci/sd4 .ci/sd4/opam64.tar + + rm -rf "$setup_WORKSPACE"/.ci/sd4/bs/bin/Opam.Runtime.amd64 + mv -v "$setup_WORKSPACE"/.ci/sd4/opam64/bin/Opam.Runtime.amd64/ "$setup_WORKSPACE"/.ci/sd4/bs/bin/ + mv -v "$setup_WORKSPACE"/.ci/sd4/opam64/bin/opam.exe "$setup_WORKSPACE"/.ci/sd4/bs/bin/ + mv -v "$setup_WORKSPACE"/.ci/sd4/opam64/bin/opam-installer.exe "$setup_WORKSPACE"/.ci/sd4/bs/bin/ + + # diagnostics + ldd "$setup_WORKSPACE"/.ci/sd4/bs/bin/opam.exe + ldd "$setup_WORKSPACE"/.ci/sd4/bs/bin/opam-installer.exe + + runit_BOOTSTRAPPED=1 + ;; + esac + fi + fi + + # Bootstrap from historical release + if [ "$runit_BOOTSTRAPPED" = 0 ] && [ "$bootstrap_opam_version" != "os" ]; then + install -d .ci/sd4/bs + cd .ci/sd4/bs + + if [ ! -e version ] || [ "$(cat version)" != "$bootstrap_opam_version" ]; then + echo 'Bootstrap opam from historical release (non-Windows; Windows non-fdopen) ...' + if command -v curl; then + curl -L -o opam.tar.gz "https://github.com/diskuv/dkml-component-opam/releases/download/v${bootstrap_opam_version}/dkml-component-staging-opam.tar.gz" + else + wget -O opam.tar.gz "https://github.com/diskuv/dkml-component-opam/releases/download/v${bootstrap_opam_version}/dkml-component-staging-opam.tar.gz" + fi + tar tvfz opam.tar.gz + tar xfz opam.tar.gz "./staging-files/${dkml_host_abi}/" + rm -rf bin/ + mv "staging-files/${dkml_host_abi}/bin" . + rm -rf "${abi_pattern}" + printf "%s" "${bootstrap_opam_version}" >version + fi + + rm -f opam.tar.gz + cd ../../.. + + runit_BOOTSTRAPPED=1 + fi + + # Bootstrap from package manager or GitHub ocaml/opam release + case "$runit_BOOTSTRAPPED,$bootstrap_opam_version,$dkml_host_abi" in + 0,os,darwin_*) + if ! command -v opam; then + echo 'Bootstrap opam from package manager (macOS) ...' + brew install gpatch + brew install opam + fi + runit_BOOTSTRAPPED=1 + ;; + 0,os,linux_x86) + if [ ! -x .ci/sd4/bs/bin/opam ]; then + echo 'Bootstrap opam from GitHub ocaml/opam release (Linux x86) ...' + install -d .ci/sd4/bs/bin + wget -O .ci/sd4/bs/bin/opam.tmp https://github.com/ocaml/opam/releases/download/2.1.2/opam-2.1.2-i686-linux + sha512_check=$(openssl sha512 2>&1 /dev/null | cut -f 2 -d ' ') + check="85a480d60e09a7d37fa0d0434ed97a3187434772ceb4e7e8faa5b06bc18423d004af3ad5849c7d35e72dca155103257fd6b1178872df8291583929eb8f884b6a" + test "$sha512" = "$check" + chmod +x .ci/sd4/bs/bin/opam.tmp + mv .ci/sd4/bs/bin/opam.tmp .ci/sd4/bs/bin/opam + else + echo "openssl 512 option unsupported." + exit 61 + fi + fi + runit_BOOTSTRAPPED=1 + ;; + 0,os,linux_x86_64) + if [ ! -x .ci/sd4/bs/bin/opam ]; then + echo 'Bootstrap opam from GitHub ocaml/opam release (Linux x86_64) ...' + install -d .ci/sd4/bs/bin + wget -O .ci/sd4/bs/bin/opam.tmp https://github.com/ocaml/opam/releases/download/2.1.2/opam-2.1.2-x86_64-linux + sha512_check=$(openssl sha512 2>&1 /dev/null | cut -f 2 -d ' ') + check="c0657ecbd4dc212587a4da70c5ff0402df95d148867be0e1eb1be8863a2851015f191437c3c99b7c2b153fcaa56cac99169c76ec94c5787750d7a59cd1fbb68b" + test "$sha512" = "$check" + chmod +x .ci/sd4/bs/bin/opam.tmp + mv .ci/sd4/bs/bin/opam.tmp .ci/sd4/bs/bin/opam + else + echo "openssl 512 option unsupported." + exit 61 + fi + fi + runit_BOOTSTRAPPED=1 + ;; + esac + } + section_begin bootstrap-opam 'Bootstrap opam' + do_bootstrap + section_end bootstrap-opam + + # Start environment distribution tarball + # We use .tar rather than .tar.gz/.tar.bz2 because we can repeatedly add to an uncompressed .tar. But we need to + # start with an empty tarball since some tar programs will only add ('tar rf xyz.tar') to an existing .tar. + install -d .ci/sd4/dist + tar cf .ci/sd4/dist/opam-with-env.tar -T /dev/null + + do_get_dockcross() { + if [ -n "${dockcross_image:-}" ]; then + # The dockcross script is super-slow + section_begin get-dockcross 'Get dockcross binary (ManyLinux)' + install -d .ci/sd4 + # shellcheck disable=SC2086 + docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_custom_prefix:-}${dockcross_image:-}" >.ci/sd4/dockcross.gen + + # PROBLEM 1 + # --------- + # Super-annoying stderr output from dockcross at line: + # tty -s && [ -z "$MSYS" ] && TTY_ARGS=-ti + # When there is no tty, get: + # tty: ignoring all arguments + # not a tty + # So replace 'tty -s &&' with 'false &&' + sed 's/tty -s &&/false \&\&/' .ci/sd4/dockcross.gen >.ci/sd4/dockcross-real + rm -f .ci/sd4/dockcross.gen + chmod +x .ci/sd4/dockcross-real + + # PROBLEM 2 + # --------- + # By default dockcross for ManyLinux will chown -R all python packages; super-slow (~10 seconds)! + # Confer: https://github.com/dockcross/dockcross/blob/master/manylinux-common/pre_exec.sh + # That kills speed for any repetitive dockcross invocation. + # + # BUT it is unnecessary to chown -R when the current user is root, because inside the Docker container + # the files are already root! + # + # The chown -R (within pre_exec.sh) is not run when the user ids are not passed in. + # Confer: https://github.com/dockcross/dockcross/blob/96d87416f639af0204bdd42553e4b99315ca8476/imagefiles/entrypoint.sh#L21-L53 + # + # So explicitly call the entrypoint if root! + case "$dkml_host_abi" in + # https://github.com/dockcross/dockcross/blob/master/linux-x86/linux32-entrypoint.sh + linux_x86) dockcross_entrypoint=/dockcross/linux32-entrypoint.sh ;; + *) dockcross_entrypoint=/dockcross/entrypoint.sh ;; + esac + cat > .ci/sd4/dockcross <&2 "Missing functionality (\${f}) (in cygwin)." ; exit 1 ; } ; + done ; + HOST_PWD="\$( cygpath -w "\$( readlink -f "\$( pwd ;)" ; )" ; )" ; + else + HOST_PWD=\$PWD + [ -L \$HOST_PWD ] && HOST_PWD=\$(readlink \$HOST_PWD) + fi + + # ---------- End of dockcross script snippet ------- + + # Handle: dockcross --args "-v X:Y --platform P" + ARGS= + if [ "\$#" -ge 1 ] && [ "\$1" = "--args" ]; then + shift + ARGS=\$1 + shift + fi + + # Directly invoke entrypoint + exec docker run --entrypoint /bin/bash \ + --rm \ + \${ARGS:-} \ + -v "\$HOST_PWD":/work \ + ${dockcross_image_custom_prefix:-}${dockcross_image:-} ${dockcross_entrypoint} "\$@" + else + HERE=\$(dirname "\$0") + HERE=\$(cd "\$HERE" && pwd) + exec "\$HERE/dockcross-real" "\$@" + fi + EOF + chmod +x .ci/sd4/dockcross + + # Bundle for consumers of setup-dkml.yml + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/dockcross .ci/sd4/dockcross-real + + section_end get-dockcross + fi + } + do_get_dockcross + + if [ -n "${dockcross_image:-}" ]; then + # rsync needs to be available, even after Docker container disappears + if [ ! -e .ci/sd4/bs/bin/rsync ]; then + section_begin get-opam-prereqs-in-dockcross 'Get Opam prerequisites (ManyLinux)' + install -d .ci/sd4/bs/bin + # shellcheck disable=SC2016 + .ci/sd4/dockcross --args "${dockcross_run_extra_args:-}" sh -c 'sudo yum install -y rsync && install $(command -v rsync) .ci/sd4/bs/bin' + section_end get-opam-prereqs-in-dockcross + fi + fi + + # Opam prerequisites for using opam (not for installing opam) + + { + if [ -n "${docker_runner:-}" ]; then + # rsync needs to be available, even after Docker container disappears + if [ ! -e .ci/sd4/bs/bin/rsync.deps ]; then + section_begin get-opam-prereqs-in-docker 'Get Opam prerequisites (Linux Docker)' + install -d .ci/sd4/bs/bin + ${docker_runner} sh -c ' + apt-get update && + apt-get install -y rsync && + ldd /usr/bin/rsync && + ls -l /lib/i386-linux-gnu/libpopt.so.0 /lib/i386-linux-gnu/libacl.so.1 /lib/i386-linux-gnu/libattr.so.1 && + tar cCfhz / /work/.ci/sd4/bs/bin/deps.tar.gz /usr/bin/rsync /lib/i386-linux-gnu/libpopt.so.0 + ' + touch .ci/sd4/bs/bin/rsync.deps + section_end get-opam-prereqs-in-docker + fi + fi + + # Bundle Opam prerequisites (ManyLinux or Linux Docker) + if [ -n "${docker_runner:-}" ] || [ -n "${dockcross_image:-}" ]; then + # Bundle for consumers of setup-dkml.yml + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/bs/bin/rsync + fi + } + + # Get Opam Cache + do_get_opam_cache() { + if [ "$unix_opam_root_cacheable" = "$unix_opam_root" ]; then return; fi + if [ ! -e "$unix_opam_root_cacheable" ]; then return; fi + section_begin get-opam-cache "Transferring Opam cache to $original_opam_root_cacheable" + echo Starting transfer # need some output or GitLab CI will not display the section duration + transfer_dir "$unix_opam_root_cacheable" "$unix_opam_root" + echo Finished transfer + section_end get-opam-cache + } + do_get_opam_cache + + # Setup Opam + + do_write_opam_scripts() { + case "${FDOPEN_OPAMEXE_BOOTSTRAP:-},$dkml_host_abi" in + true,windows_*) + # With fdopen's opam.exe, 'os-distribution = "cygwinports"'. But native Windows opam.exe has 'os-distribution = "win32"'. + # But on Windows we always want MSYS2 or native Windows libraries, not Cygwin. If cygwinports then + # code like https://github.com/ocaml/opam-repository/blob/08cbb8258bd4bf30cd6f307c958911a29d537b54/packages/conf-pkg-config/conf-pkg-config.2/opam#L36 + # will fail. So always set 'os-distribution = "win32"' on Windows. + PATCH_OS_DISTRIBUTION_WIN32=true + # With fdopen's opam.exe, no 'exe = ".exe"' is set because Cygwin does not need file extensions. + # Native Windows requires a .exe extension. + PATCH_EXE_WIN32=true + ;; + *) + PATCH_OS_DISTRIBUTION_WIN32=false + PATCH_EXE_WIN32=false + ;; + esac + + # --------------------- + # Empty opam repository + # --------------------- + + install -d .ci/sd4/eor + cat >.ci/sd4/eor/repo <.ci/sd4/troubleshoot-opam.sh <&2 + find "\$OPAMROOT"/log -mindepth 1 -maxdepth 1 -name "*.out" ! -name "log-*.out" ! -name "ocaml-variants-*.out" | while read -r dump_on_error_LOG; do + dump_on_error_BLOG=\$(basename "\$dump_on_error_LOG") + printf "\n\n========= [TROUBLESHOOTING] %s ===========\n\n" "\$dump_on_error_BLOG" >&2 + awk -v BLOG="\$dump_on_error_BLOG" '{print "[" BLOG "]", \$0}' "\$dump_on_error_LOG" >&2 + done + printf "\nScroll up to see the [TROUBLESHOOTING] logs that begin at the [START OF TROUBLESHOOTING] line\n" >&2 + EOF + + chmod +x .ci/sd4/troubleshoot-opam.sh + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/troubleshoot-opam.sh + + # --------------- + # Create Opam support scripts (not needed for all platforms) + # The PATH to find opam must work internally in setup-dkml.yml (sd4/bs/bin) and + # by consumers of setup-dkml.yml (sd4/opamexe) + # --------------- + + USER_ID=$(id -u) + GROUP_ID=$(id -g) + USER_NAME=$(id -un) + GROUP_NAME=$(id -gn) + + case "${opam_root}" in + /* | ?:*) # /a/b/c or C:\Windows + validate_supports_docker() { + echo "Docker only supported with relative paths for the opam root, not: ${opam_root}" >&2 + exit 3 + } + ;; + *) # relative path + validate_supports_docker() { + true + } + cat >.ci/sd4/opam-in-docker <&2 + set +e + opam "\$@" + exitcode=\$? + if [ \$troubleshooting = 1 ]; then + [ \$exitcode = 0 ] || "/work/.ci/sd4/troubleshoot-opam.sh" \$OPAMROOT + fi + exit \$exitcode + EOF + chmod +x .ci/sd4/opam-in-docker + ;; + esac + + cat >.ci/sd4/deescalate <.ci/sd4/opam-with-env <&2 + cat .ci/sd4/opam-in-docker >&2 + echo '________________________' >&2 + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/opam-with-env .ci/sd4/opam-in-docker .ci/sd4/edr + + elif [ -n "${docker_runner:-}" ]; then + + cat >.ci/sd4/opam-with-env <&2 + cat .ci/sd4/opam-in-docker >&2 + echo '________________________' >&2 + echo '__ deescalate __' >&2 + cat .ci/sd4/deescalate >&2 + echo '________________' >&2 + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/opam-with-env .ci/sd4/opam-in-docker .ci/sd4/deescalate + + else + + cat >.ci/sd4/opam-with-env <&2 + set +e + opam "\$@" + exitcode=\$? + if [ \$troubleshooting = 1 ]; then + [ \$exitcode = 0 ] || "\${PROJECT_DIR}/.ci/sd4/troubleshoot-opam.sh" \$OPAMROOT + fi + exit \$exitcode + EOF + chmod +x .ci/sd4/opam-with-env + + # Bundle for consumers of setup-dkml.yml + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/opam-with-env + + fi + echo '__ opam-with-env __' >&2 + cat .ci/sd4/opam-with-env >&2 + echo '___________________' >&2 + + # ------- + # opamrun + # ------- + + install -d .ci/sd4/opamrun + cat >.ci/sd4/opamrun/opamrun <>"$GITHUB_PATH" + # Special case: GITHUB_PATH does not influence msys2.CMD of msys2/setup-msys2@v2, so place in real MSYS2 PATH + if [ -n "${MSYSTEM:-}" ]; then + install -d /usr/local/bin + install .ci/sd4/opamrun/opamrun /usr/local/bin/opamrun + fi + fi + + # Place opamrun in the immediate PATH + PATH="$setup_WORKSPACE/.ci/sd4/opamrun:$PATH" + + # Complicated Opam sequence is because: + # 1. Opam's default curl does not work on Windows, + # and `opam init` does not provide a way to change it (TODO: need + # a PR!). + # 2. We have to separate the Opam download cache from the other Opam + # caches + if [ ! -s "$opam_root/.ci.root-init" ]; then # non-empty init file so can be cached irrespective of existence + section_begin opam-init 'Initialize opam root' + + # Clear any partial previous attempt + rm -rf "$opam_root" + + case "$dkml_host_abi,${in_docker:-}" in + windows_*,*) + eor=$(cygpath -am "$setup_WORKSPACE"/.ci/sd4/eor) + opamrun init --disable-sandboxing --no-setup --kind local --bare "$eor" + case "$(opamrun --version)" in + 2.0.*) echo 'download-command: wget' >>"$opam_root/config" ;; + *) opamrun option --yes --global download-command=wget ;; + esac + ;; + *,true) + opamrun init --disable-sandboxing --no-setup --kind local --bare "/work/.ci/sd4/eor" + ;; + *) + opamrun init --disable-sandboxing --no-setup --kind local --bare "$setup_WORKSPACE/.ci/sd4/eor" + ;; + esac + echo yes > "$opam_root/.ci.root-init" + + section_end opam-init + fi + + section_begin opam-vars "Summary: opam global variables" + opamrun --no-troubleshooting var --global || true + section_end opam-vars + + # Build OCaml + + do_switch_create() { + do_switch_create_NAME=$1 + shift + + section_begin "switch-create-$do_switch_create_NAME" "Create opam switch '$do_switch_create_NAME'" + # Create, or recreate, the Opam switch. The Opam switch should not be + # cached except for the compiler (confer docs for setup-ocaml GitHub + # Action) which is the 'dkml' switch (or the 'two' switch). + # Check if the switch name is present in the Opam root (which may come from cache) + NOMINALLY_PRESENT=false + if opamrun switch list --short | grep "^${do_switch_create_NAME}\$"; then NOMINALLY_PRESENT=true; fi + + # Check if the switch is actually present in case of cache incoherence + # or corrupt Opam state that could result in: + # Error: No config file found for switch dkml. Switch broken? + if [ $NOMINALLY_PRESENT = true ] && [ ! -e "$opam_root/$do_switch_create_NAME/.opam-switch/switch-config" ]; then + # Remove the switch name from Opam root, and any partial switch state. + # Ignore inevitable warnings/failure about missing switch. + opamrun --no-troubleshooting switch remove "$do_switch_create_NAME" --yes || true + rm -rf "${opam_root:?}/$do_switch_create_NAME" + NOMINALLY_PRESENT=false + fi + + if [ $NOMINALLY_PRESENT = false ]; then + opamrun switch create "$do_switch_create_NAME" --empty --yes + fi + section_end "switch-create-$do_switch_create_NAME" + } + do_switch_create dkml + if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_switch_create two + else + section_begin "switch-create-two" "Create empty opam switch 'two'" + # Always create a secondary switch ... just empty. Avoid problems with cache content missing + # and idempotency. + opamrun --no-troubleshooting switch remove two --yes || true + rm -rf "$opam_root/two" + opamrun switch create two --empty --yes + section_end "switch-create-two" + fi + + do_switch_active() { + section_begin "switch-active" "Set dkml as active switch" + opamrun switch set dkml --yes + section_end "switch-active" + } + do_switch_active + + do_opam_repositories_add() { + section_begin "opam-repo-add" "Add 'diskuv' opam repository" + if ! opamrun --no-troubleshooting repository list -s | grep '^diskuv'; then + opamrun repository add diskuv "git+https://github.com/diskuv/diskuv-opam-repository.git#${DISKUV_OPAM_REPOSITORY:-$DEFAULT_DISKUV_OPAM_REPOSITORY_TAG}" --yes --dont-select + fi + section_end "opam-repo-add" + } + do_opam_repositories_add + + do_opam_repositories_config() { + do_opam_repositories_config_NAME=$1 + shift + + section_begin "opam-repo-$do_opam_repositories_config_NAME" "Attach repositories to $do_opam_repositories_config_NAME" + + if [ ! -s "$opam_root/.ci.$do_opam_repositories_config_NAME.repo-init" ]; then # non-empty init file so can be cached irrespective of existence + opamrun --no-troubleshooting repository remove default --switch "$do_opam_repositories_config_NAME" --yes || true + opamrun --no-troubleshooting repository remove diskuv --switch "$do_opam_repositories_config_NAME" --yes || true + opamrun repository add default --switch "$do_opam_repositories_config_NAME" --yes + opamrun repository add diskuv --switch "$do_opam_repositories_config_NAME" --yes + echo yes > "$opam_root/.ci.$do_opam_repositories_config_NAME.repo-init" + fi + + section_end "opam-repo-$do_opam_repositories_config_NAME" + } + do_opam_repositories_config dkml + if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_opam_repositories_config two + fi + + do_opam_repositories_update() { + section_begin "opam-repo-update" "Update opam repositories" + # The default repository may be the initial 'eor' (empty) repository + opamrun repository set-url default https://opam.ocaml.org --yes + # Always set the `diskuv` repository url since it can change + opamrun repository set-url diskuv "git+https://github.com/diskuv/diskuv-opam-repository.git#${DISKUV_OPAM_REPOSITORY:-$DEFAULT_DISKUV_OPAM_REPOSITORY_TAG}" --yes --dont-select + # Update both `default` and `diskuv` Opam repositories + opamrun update default diskuv + section_end "opam-repo-update" + } + do_opam_repositories_update + + do_pins() { + do_pins_NAME=$1 + shift + + # dkml-base-compiler + + if [ "${DKML_COMPILER:-}" != '@repository@' ] && [ -z "${DKML_COMPILER:-}" ] && [ -z "${OCAML_COMPILER:-}" ]; then + section_begin checkout-dkml-base-compiler "Pin dkml-base-compiler to default ${DEFAULT_DKML_COMPILER} (neither dkml-base-compiler nor OCAML_COMPILER specified) for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action dkml-base-compiler "https://github.com/diskuv/dkml-compiler.git#${DEFAULT_DKML_COMPILER}" + section_end checkout-dkml-base-compiler + elif [ "${DKML_COMPILER:-}" != '@repository@' ] && [ -n "${DKML_COMPILER:-}" ] && [ -z "${OCAML_COMPILER:-}" ]; then + section_begin checkout-dkml-base-compiler "Pin dkml-base-compiler to $DKML_COMPILER (dkml-base-compiler specified; no OCAML_COMPILER specified) for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action dkml-base-compiler "https://github.com/diskuv/dkml-compiler.git#${DKML_COMPILER}" + section_end checkout-dkml-base-compiler + elif [ -n "${OCAML_COMPILER:-}" ]; then + # Validate OCAML_COMPILER (OCAML_COMPILER specified) + case "${OCAML_COMPILER:-}" in + 4.12.1) true ;; + *) + echo "OCAML_COMPILER version ${OCAML_COMPILER:-} is not supported" >&2 + exit 109 + ;; + esac + + section_begin checkout-dkml-base-compiler "Pin dkml-base-compiler (OCAML_COMPILER specified) for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action dkml-base-compiler "https://github.com/diskuv/dkml-compiler.git#${OCAML_COMPILER}-v${DKML_VERSION}" + section_end checkout-dkml-base-compiler + fi + + # conf-dkml-cross-toolchain + + if [ "${CONF_DKML_CROSS_TOOLCHAIN:-}" != '@repository@' ]; then + section_begin checkout-conf-dkml-cross-toolchain "Pin conf-dkml-cross-toolchain for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action conf-dkml-cross-toolchain "https://github.com/diskuv/conf-dkml-cross-toolchain.git#$CONF_DKML_CROSS_TOOLCHAIN" + section_end checkout-conf-dkml-cross-toolchain + fi + + # patches necessary for Windows in diskuv-opam-repository + # + # - ocamlfind and ocamlbuild + # + # - dune-configurator (and hence Dune) + # Dune 2.9.1 and 3.0.2 will fail to build jst-config.v0.14.1 because for jst-config/discover/discover.ml Dune does: + # cl -nologo -O2 -Gy- -MD -I Z:/.opam_root-cached-8/installer-ocaml/lib/ocaml -o C:\Users\beckf\AppData\Local\Temp\build_f18aec_dune\ocaml-configurator4d3858\c-test-31\test.obj -c C:\Users\beckf\AppData\Local\Temp\build_f18aec_dune\ocaml-configurator4d3858\c-test-31\test.c advapi32.lib ws2_32.lib version.lib + # instead of + # cl -nologo -O2 -Gy- -MD -I Z:/.opam_root-cached-8/installer-ocaml/lib/ocaml /FoC:\Users\beckf\AppData\Local\Temp\build_f18aec_dune\ocaml-configurator4d3858\c-test-31\test.obj -c C:\Users\beckf\AppData\Local\Temp\build_f18aec_dune\ocaml-configurator4d3858\c-test-31\test.c advapi32.lib ws2_32.lib version.lib + # with the (irrelevant) test.c file: + # #include + # #include + # + # #ifdef ARCH_BIG_ENDIAN + # const char *s0 = "BEGIN-0-true-END"; + # #else + # const char *s0 = "BEGIN-0-false-END"; + # #endif + # + # #ifdef ARCH_SIXTYFOUR + # const char *s1 = "BEGIN-1-true-END"; + # #else + # const char *s1 = "BEGIN-1-false-END"; + # #endif + # + # The actual problem is dune-configurator ... we only have patches in Diskuv + # repository up until 2.9.3. Need to upstream fix the problem. + # + # - ppx_expect; only patch is for v0.14.1. Need to upstream fix the problem. + # - base; patches for v0.14.1/2/3. Need to upstream fix the problem. + section_begin "opam-pins-$do_pins_NAME" "Opam pins for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version base "${PIN_BASE}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version bigstringaf "${PIN_BIGSTRINGAF}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version core_kernel "${PIN_CORE_KERNEL}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ctypes "${PIN_CTYPES}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ctypes-foreign "${PIN_CTYPES_FOREIGN}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version curly "${PIN_CURLY}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version digestif "${PIN_DIGESTIF}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version dkml-apps "${PIN_DKML_APPS}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version dune "${PIN_DUNE}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version dune-configurator "${PIN_DUNE_CONFIGURATOR}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ocamlbuild "${PIN_OCAMLBUILD}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ocamlfind "${PIN_OCAMLFIND}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ocp-indent "${PIN_OCP_INDENT}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ppx_expect "${PIN_PPX_EXPECT}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ptime "${PIN_PTIME}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version time_now "${PIN_TIME_NOW}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version with-dkml "${PIN_WITH_DKML}" + section_end "opam-pins-$do_pins_NAME" + } + + do_pins dkml + if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_pins two + fi + + do_use_vsstudio() { + do_use_vsstudio_NAME=$1 + shift + case "$dkml_host_abi" in + windows_*) + section_begin "use-vsstudio-$do_use_vsstudio_NAME" "Use Visual Studio in dkml-* Opam packages (Windows) for $do_use_vsstudio_NAME switch" + + # shellcheck disable=SC2153 + E_VS_DIR=$(escape_arg_as_ocaml_string "$VS_DIR") + # shellcheck disable=SC2153 + E_VS_VCVARSVER=$(escape_arg_as_ocaml_string "$VS_VCVARSVER") + # shellcheck disable=SC2153 + E_VS_WINSDKVER=$(escape_arg_as_ocaml_string "$VS_WINSDKVER") + # shellcheck disable=SC2153 + E_VS_MSVSPREFERENCE=$(escape_arg_as_ocaml_string "$VS_MSVSPREFERENCE") + # shellcheck disable=SC2153 + E_VS_CMAKEGENERATOR=$(escape_arg_as_ocaml_string "$VS_CMAKEGENERATOR") + + case "$(opamrun --version)" in + 2.0.*) + if [ "${in_docker}" = "true" ]; then + echo Opam 2.0 support in dockcross to use a portable opam var prefix not yet implemented + exit 67 + fi + OP=$(opamrun var prefix --switch "$do_use_vsstudio_NAME") + OPSC=$OP/.opam-switch/switch-config + if grep setenv: "$OPSC"; then + echo "INFO: Updating switch-config. Old was:" + awk '{print ">> " $0}' "$OPSC" + + awk '$1=="setenv:"{x=1} x==0{print} x==1 && $0=="]"{x=0}' "$OPSC" >"$OPSC".trimmed + mv "$OPSC".trimmed "$OPSC" + fi + echo 'setenv: [' >>"$OPSC" + echo ' [DKML_COMPILE_SPEC = "1"]' >>"$OPSC" + echo ' [DKML_COMPILE_TYPE = "VS"]' >>"$OPSC" + echo " [DKML_COMPILE_VS_DIR = \"$E_VS_DIR\"]" >>"$OPSC" + echo " [DKML_COMPILE_VS_VCVARSVER = \"$E_VS_VCVARSVER\"]" >>"$OPSC" + echo " [DKML_COMPILE_VS_WINSDKVER = \"$E_VS_WINSDKVER\"]" >>"$OPSC" + echo " [DKML_COMPILE_VS_MSVSPREFERENCE = \"$E_VS_MSVSPREFERENCE\"]" >>"$OPSC" + echo " [DKML_COMPILE_VS_CMAKEGENERATOR = \"$E_VS_CMAKEGENERATOR\"]" >>"$OPSC" + echo " [DKML_HOST_ABI = \"${dkml_host_abi}\"]" >>"$OPSC" + echo ']' >>"$OPSC" + cat "$OPSC" >&2 # print + ;; + *) + opamrun option --switch "$do_use_vsstudio_NAME" setenv= # reset + opamrun option --switch "$do_use_vsstudio_NAME" setenv+='DKML_COMPILE_SPEC = "1"' + opamrun option --switch "$do_use_vsstudio_NAME" setenv+='DKML_COMPILE_TYPE = "VS"' + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_DIR = \"$E_VS_DIR\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_VCVARSVER = \"$E_VS_VCVARSVER\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_WINSDKVER = \"$E_VS_WINSDKVER\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_MSVSPREFERENCE = \"$E_VS_MSVSPREFERENCE\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_CMAKEGENERATOR = \"$E_VS_CMAKEGENERATOR\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_HOST_ABI = \"${dkml_host_abi}\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv # print + ;; + esac + + # shellcheck disable=SC2016 + opamrun exec --switch "$do_use_vsstudio_NAME" -- sh -c 'echo $VCToolsRedistDir' + + section_end "use-vsstudio-$do_use_vsstudio_NAME" + ;; + esac + } + if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_use_vsstudio two + fi + do_use_vsstudio dkml + + # Because dune.X.Y.Z+shim (and any user DKML packages) requires DKML installed (after all, it is just + # a with-dkml.exe shim), we need either dkmlvars-v2.sexp or DKML environment + # variables. Confer: Dkml_runtimelib.Dkml_context.get_dkmlversion + # + # grep matches either: + # [... [DiskuvOCamlVersion = "1.0.1"] ...] + # DiskuvOCamlVersion = "1.0.1" + do_setenv() { + do_setenv_SWITCH=$1 + shift + opamrun option --switch "$do_setenv_SWITCH" setenv > ".ci/sd4/setenv.$do_setenv_SWITCH.txt" + if ! grep -q '\(^|\[\)DiskuvOCamlVarsVersion ' ".ci/sd4/setenv.$do_setenv_SWITCH.txt"; then + opamrun option --switch "$do_setenv_SWITCH" setenv+='DiskuvOCamlVarsVersion = "2"' + fi + if ! grep -q '\(^|\[\)DiskuvOCamlVersion ' ".ci/sd4/setenv.$do_setenv_SWITCH.txt"; then + opamrun option --switch "$do_setenv_SWITCH" setenv+="DiskuvOCamlVersion = \"$DKML_VERSION\"" + fi + case "${dkml_host_abi}" in + windows_*) + if ! grep -q '\(^|\[\)DiskuvOCamlMSYS2Dir ' ".ci/sd4/setenv.$do_setenv_SWITCH.txt"; then + if [ -x /usr/bin/cygpath ]; then + MSYS2_DIR_NATIVE=$(/usr/bin/cygpath -aw /) + else + # If we are already inside MSYS2 then MSYSTEM_PREFIX should be set. But cygpath should be there as well!! + echo "FATAL: Could not locate MSYS2: there was no cygpath" >&2 + exit 3 + fi + MSYS2_DIR_NATIVE_ESCAPED=$(printf "%s" "$MSYS2_DIR_NATIVE" | sed 's/\\/\\\\/g') + opamrun option --switch "$do_setenv_SWITCH" setenv+="DiskuvOCamlMSYS2Dir = \"$MSYS2_DIR_NATIVE_ESCAPED\"" + fi + esac + } + if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_setenv two + fi + do_setenv dkml + + do_install_compiler() { + do_install_compiler_NAME=$1 + shift + section_begin "install-compiler-$do_install_compiler_NAME" "Install OCaml compiler for $do_install_compiler_NAME switch" + opamrun pin list --switch "$do_install_compiler_NAME" + # shellcheck disable=SC2086 + opamrun upgrade --switch "$do_install_compiler_NAME" --yes dkml-base-compiler conf-dkml-cross-toolchain ${ocaml_options:-} + section_end "install-compiler-$do_install_compiler_NAME" + } + do_install_compiler dkml + if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_install_compiler two + fi + + do_summary() { + do_summary_NAME=$1 + shift + section_begin "summary-$do_summary_NAME" "Summary: $do_summary_NAME switch" + opamrun var --switch "$do_summary_NAME" + opamrun exec --switch "$do_summary_NAME" -- ocamlc -config + section_end "summary-$do_summary_NAME" + } + do_summary dkml + if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_summary two + fi + + end_of_script + + cat > .ci/sd4/run-teardown-dkml.sh <<'end_of_script' + #!/bin/sh + set -euf + + teardown_WORKSPACE_VARNAME=$1 + shift + teardown_WORKSPACE=$1 + shift + + # ------------------ Variables and functions ------------------------ + + # shellcheck source=./common-values.sh + . .ci/sd4/common-values.sh + + # Fixup opam_root on Windows to be mixed case. Set original_* and unix_* as well. + fixup_opam_root + + # Set TEMP variable for Windows + export_temp_for_windows + + # ------------------------------------------------------------------- + + section_begin teardown-info "Summary: teardown-dkml" + + # shellcheck disable=SC2154 + echo " + ================ + teardown-dkml.sh + ================ + . + --------- + Arguments + --------- + WORKSPACE_VARNAME=$teardown_WORKSPACE_VARNAME + WORKSPACE=$teardown_WORKSPACE + . + ------ + Inputs + ------ + VERBOSE=${VERBOSE:-} + . + ------ + Matrix + ------ + dkml_host_abi=$dkml_host_abi + opam_root=${opam_root} + opam_root_cacheable=${opam_root_cacheable} + original_opam_root=${original_opam_root} + original_opam_root_cacheable=${original_opam_root_cacheable} + unix_opam_root=${unix_opam_root} + unix_opam_root_cacheable=${unix_opam_root_cacheable} + . + " + section_end teardown-info + + # Done with Opam cache! + do_save_opam_cache() { + if [ "$unix_opam_root_cacheable" = "$unix_opam_root" ]; then return; fi + section_begin save-opam-cache "Transfer Opam cache to $original_opam_root" + echo Starting transfer # need some output or GitLab CI will not display the section duration + transfer_dir "$unix_opam_root" "$unix_opam_root_cacheable" + echo Finished transfer + section_end save-opam-cache + } + do_save_opam_cache + + do_fill_skipped_cache_entries() { + section_begin fill-skipped-cache-entries "Populate skipped cache entries" + + # Needed to stop GitLab CI/CD cache warnings 'no matching files', etc. + if [ ! -e .ci/sd4/vsenv.sh ]; then + install -d .ci/sd4 + rm -f .ci/sd4/vsenv.sh + touch .ci/sd4/vsenv.sh + chmod +x .ci/sd4/vsenv.sh + echo "Created empty vsenv.sh" + else + echo "Found vsenv.sh" + fi + + if [ ! -e msys64 ]; then + rm -rf msys64 + install -d msys64 + touch msys64/.keep + echo "Created empty msys64" + else + echo "Found msys64/" + fi + + if [ -s "$unix_opam_root_cacheable/.ci.dkml.repo-init" ]; then + echo "Found non-empty $unix_opam_root_cacheable/.ci.dkml.repo-init" + else + touch "$unix_opam_root_cacheable/.ci.dkml.repo-init" + echo "Created empty $unix_opam_root_cacheable/.ci.dkml.repo-init" + fi + + if [ -s "$unix_opam_root_cacheable/.ci.two.repo-init" ]; then + echo "Found non-empty $unix_opam_root_cacheable/.ci.two.repo-init" + else + touch "$unix_opam_root_cacheable/.ci.two.repo-init" + echo "Created empty $unix_opam_root_cacheable/.ci.two.repo-init" + fi + + section_end fill-skipped-cache-entries + } + do_fill_skipped_cache_entries + + do_at_least_one_artifact() { + install -d dist + find dist -mindepth 1 -maxdepth 1 >.ci/dist.files + if [ ! -s .ci/dist.files ]; then + section_begin one-artifact "Create empty artifact file" + + # Avoid confusing "ERROR: No files to upload" in GitLab CI + touch dist/.keep + echo "Created dist/.keep" + + section_end one-artifact + fi + } + do_at_least_one_artifact + + end_of_script + + - name: Prepare cache keys + # An undocumented bug with GitHub Actions is that a space in the key + # will "succeed" but it never gets restored. So we will hash a + # user-friendly file instead. + shell: bash + run: | + set -x + echo '317446-${{ inputs.FDOPEN_OPAMEXE_BOOTSTRAP }}-${{ steps.full_matrix_vars.outputs.dkml_host_abi }}-${{ steps.full_matrix_vars.outputs.opam_abi }}-${{ steps.full_matrix_vars.outputs.bootstrap_opam_version }}' > .ci/sd4/cachekey.opam.binaries + echo '${{ steps.full_matrix_vars.outputs.abi_pattern }}-${{ steps.full_matrix_vars.outputs.vsstudio_arch }}-${{ steps.full_matrix_vars.outputs.vsstudio_hostarch }}-${{ steps.full_matrix_vars.outputs.vsstudio_dir }}-${{ steps.full_matrix_vars.outputs.vsstudio_vcvarsver }}-${{ steps.full_matrix_vars.outputs.vsstudio_winsdkver }}-${{ steps.full_matrix_vars.outputs.vsstudio_msvspreference }}-${{ steps.full_matrix_vars.outputs.vsstudio_cmakegenerator }}' > .ci/sd4/cachekey.vsstudio + echo '${{ inputs.OCAML_COMPILER }}-${{ inputs.DISKUV_OPAM_REPOSITORY }}-${{ inputs.DKML_COMPILER }}-${{ inputs.CONF_DKML_CROSS_TOOLCHAIN }}' > .ci/sd4/cachekey.ci.inputs + + # Bootstrapping Opam + + - name: Cache Opam bootstrap by OS + uses: actions/cache@v3 + id: cache-sd4-bs + with: + path: .ci/sd4/bs + key: + "1_${{ inputs.CACHE_PREFIX }}_${{ runner.os }}-sd4-bs-${{ hashFiles('.ci/sd4/cachekey.opam.binaries') }}" + + # ------ + # BEGIN Docker (ManyLinux) + # We use buildx which can cache into GitHub Actions Cache (type=gha) + + - name: Set up Docker Buildx (ManyLinux) + if: steps.full_matrix_vars.outputs.dockcross_image != '' + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Configure Docker (ManyLinux) + if: steps.full_matrix_vars.outputs.dockcross_image != '' + shell: bash + run: | + install -d .ci/sd4/dockcross-ctx + printf 'FROM %s\nRUN echo done fetching image' '${{ steps.full_matrix_vars.outputs.dockcross_image }}' > .ci/sd4/dockcross-ctx/Dockerfile + + - name: Fetch dockcross image (ManyLinux) + if: steps.full_matrix_vars.outputs.dockcross_image != '' + uses: docker/build-push-action@v3 + with: + context: .ci/sd4/dockcross-ctx + builder: "${{ steps.buildx.outputs.name }}" + load: true + tags: dkml-${{ steps.full_matrix_vars.outputs.dockcross_image }} + cache-from: type=gha,scope=${{ github.workflow }} + cache-to: type=gha,mode=max,scope=${{ github.workflow }} + + # END Docker (ManyLinux) + # ------ + + - name: Cache Opam root by (host,target) + uses: actions/cache@v3 + id: cache-sd4-opamroot + with: + path: | + ${{ steps.full_matrix_vars.outputs.opam_root_cacheable }}/config + ${{ steps.full_matrix_vars.outputs.opam_root_cacheable }}/dkml + ${{ steps.full_matrix_vars.outputs.opam_root_cacheable }}/two + ${{ steps.full_matrix_vars.outputs.opam_root_cacheable }}/repo + ${{ steps.full_matrix_vars.outputs.opam_root_cacheable }}/download-cache + ${{ steps.full_matrix_vars.outputs.opam_root_cacheable }}/.ci.root-init + ${{ steps.full_matrix_vars.outputs.opam_root_cacheable }}/.ci.dkml.repo-init + ${{ steps.full_matrix_vars.outputs.opam_root_cacheable }}/.ci.two.repo-init + key: + "1_${{ inputs.CACHE_PREFIX }}_${{ env.DKML_VERSION }}_${{ steps.full_matrix_vars.outputs.abi_pattern }}-sd4-opamroot-${{ hashFiles('.ci/sd4/cachekey.opam.binaries') }}-${{ hashFiles('cachekey.ci.inputs') }}" + + # The action/checkout steps used for pinning will re-use existing Git objects + # because of caching + - name: Cache Git checkouts of Opam pins by OS + uses: actions/cache@v3 + id: cache-sd4-git + with: + path: .ci/sd4/g + key: + "1_${{ inputs.CACHE_PREFIX }}_${{ runner.os }}-sd4-git-${{ hashFiles('.ci/sd4/cachekey.ci.inputs') }}" + + # Checkout code + + - name: Checkout DKML code + shell: bash + env: + # Push down inputs variables for GitHub/GitLab portable scripts + VERBOSE: "${{ inputs.VERBOSE }}" + run: 'sh .ci/sd4/run-checkout-code.sh GITHUB_WORKSPACE "$GITHUB_WORKSPACE"' + + # Setup C compiler + + - name: Compile 'dkml' switch + shell: bash + env: + # Push down inputs variables for GitHub/GitLab portable scripts + # When non-empty, instead of building the standard components from the + # central Opam repository, use the github/gitlab development repositories + # directly. + FDOPEN_OPAMEXE_BOOTSTRAP: "${{ inputs.FDOPEN_OPAMEXE_BOOTSTRAP }}" + CONF_DKML_CROSS_TOOLCHAIN: "${{ inputs.CONF_DKML_CROSS_TOOLCHAIN }}" + DISKUV_OPAM_REPOSITORY: "${{ inputs.DISKUV_OPAM_REPOSITORY }}" + DKML_COMPILER: "${{ inputs.DKML_COMPILER }}" + OCAML_COMPILER: "${{ inputs.OCAML_COMPILER }}" + SECONDARY_SWITCH: "${{ inputs.SECONDARY_SWITCH }}" + VERBOSE: "${{ inputs.VERBOSE }}" + MANYLINUX: "${{ inputs.MANYLINUX }}" + + # autogen from global_env_vars. + DEFAULT_DKML_COMPILER: '4.12.1-v1.0.2' + PIN_BASE: 'v0.14.3' + PIN_BIGSTRINGAF: '0.8.0' + PIN_CORE_KERNEL: 'v0.14.2' + PIN_CTYPES_FOREIGN: '0.19.2-windowssupport-r4' + PIN_CTYPES: '0.19.2-windowssupport-r4' + PIN_CURLY: '0.2.1-windows-env_r2' + PIN_DIGESTIF: '1.0.1' + PIN_DUNE: '2.9.3+shim.1.0.2~r0' + PIN_DUNE_CONFIGURATOR: '2.9.3' + PIN_DKML_APPS: '1.0.1' + PIN_OCAMLBUILD: '0.14.0' + PIN_OCAMLFIND: '1.9.1' + PIN_OCP_INDENT: '1.8.2-windowssupport' + PIN_PPX_EXPECT: 'v0.14.1' + PIN_PTIME: '0.8.6-msvcsupport' + PIN_TIME_NOW: 'v0.14.0' + PIN_WITH_DKML: '1.0.1' + run: | + sh .ci/sd4/run-setup-dkml.sh GITHUB_WORKSPACE "$GITHUB_WORKSPACE" + + # Expose opamrun to remaining GitHub Action steps + opamrunabs="$GITHUB_WORKSPACE/.ci/sd4/opamrun" + if [ -x /usr/bin/cygpath ]; then opamrunabs=$(/usr/bin/cygpath -aw "$opamrunabs"); fi + echo "$opamrunabs" >> $GITHUB_PATH diff --git a/ci/setup-dkml/gh-linux/pre/dune b/ci/setup-dkml/gh-linux/pre/dune new file mode 100644 index 0000000..51d87fe --- /dev/null +++ b/ci/setup-dkml/gh-linux/pre/dune @@ -0,0 +1,17 @@ +; DO NOT EDIT THIS FILE. It is auto-generated by generate-setup-dkml-scaffold +; Typical upgrade steps: +; opam upgrade dkml-workflows && opam exec -- generate-setup-dkml-scaffold && dune build '@gen-dkml' --auto-promote + +(rule + (alias gen-dkml) + (target action.gen.yml) + (action + (setenv + OCAMLRUNPARAM + b + (run gh-dkml-action-yml --phase pre --output-linux %{target})))) + +(rule + (alias gen-dkml) + (action + (diff action.yml action.gen.yml))) diff --git a/ci/setup-dkml/gh-windows/post/action.yml b/ci/setup-dkml/gh-windows/post/action.yml new file mode 100644 index 0000000..df09539 --- /dev/null +++ b/ci/setup-dkml/gh-windows/post/action.yml @@ -0,0 +1,102 @@ +# setup-dkml +# Short form: sd4 + +# Any GitHub Job that includes this action must be in a strategy matrix. +# The matrix variables must include: +# - gh_os: windows-2019 +# abi_pattern: win32-windows_x86 +# dkml_host_abi: windows_x86 + +name: post-dkml +author: Diskuv, Inc. +description: Teardown after building OCaml native executables for Windows + +runs: + using: "composite" + + steps: + - name: Full matrix variables + shell: bash # bash on Windows is Git Bash (an non-upgradable MSYS2 system) + # Every matrix variable lookup in this Action should use the output of this step. Even + # the matrix variables that the user must specify (ex. dkml_host_abi) should be + # referenced using [steps.full_matrix_vars.outputs.dkml_host_abi] rather than + # [matrix.dkml_host_abi] so that there is a single place to edit for variable changes. + id: full_matrix_vars + run: | + # Select correct Windows matrix variables + case "${{ matrix.dkml_host_abi }}" in + + windows_x86) + dkml_host_os='windows'; + opam_root_cacheable='D:/.opam'; + abi_pattern='win32-windows_x86'; + gh_os='windows-2019'; + gh_unix_shell='msys2 {0}'; + msys2_system='MINGW32'; + msys2_packages='mingw-w64-i686-pkg-config'; + exe_ext='.exe'; + bootstrap_opam_version='2.2.0-dkml20220801T155940Z'; + opam_abi='windows_x86'; + dkml_host_abi='windows_x86'; + opam_root='D:/.opam'; + vsstudio_hostarch='x64'; + vsstudio_arch='x86'; + ocaml_options='ocaml-option-32bit' ;; + + windows_x86_64) + dkml_host_os='windows'; + opam_root_cacheable='D:/.opam'; + abi_pattern='win32-windows_x86_64'; + gh_os='windows-2019'; + gh_unix_shell='msys2 {0}'; + msys2_system='CLANG64'; + msys2_packages='mingw-w64-clang-x86_64-pkg-config'; + exe_ext='.exe'; + bootstrap_opam_version='2.2.0-dkml20220801T155940Z'; + opam_abi='windows_x86_64'; + dkml_host_abi='windows_x86_64'; + opam_root='D:/.opam'; + vsstudio_hostarch='x64'; + vsstudio_arch='x64' ;; + + *) echo "FATAL: Unsupported dkml_host_abi=$dkml_host_abi in Windows action.yml"; exit 107 ;; + esac + + add() { + echo "$1=$2" | tee -a $GITHUB_OUTPUT | tee -a $GITHUB_ENV + } + + add dkml_host_abi "$dkml_host_abi" + add abi_pattern "$abi_pattern" + add opam_root "$opam_root" + add opam_root_cacheable "$opam_root_cacheable" + add exe_ext "${exe_ext:-}" + add bootstrap_opam_version "${bootstrap_opam_version:-}" + add ocaml_options "${ocaml_options:-}" + + - name: Teardown DKML build apparatus + shell: msys2 {0} + env: + _STUB_FOR_AUTOGEN: "ON" # + + # autogen from global_env_vars. + DEFAULT_DKML_COMPILER: '4.12.1-v1.0.2' + PIN_BASE: 'v0.14.3' + PIN_BIGSTRINGAF: '0.8.0' + PIN_CORE_KERNEL: 'v0.14.2' + PIN_CTYPES_FOREIGN: '0.19.2-windowssupport-r4' + PIN_CTYPES: '0.19.2-windowssupport-r4' + PIN_CURLY: '0.2.1-windows-env_r2' + PIN_DIGESTIF: '1.0.1' + PIN_DUNE: '2.9.3+shim.1.0.2~r0' + PIN_DUNE_CONFIGURATOR: '2.9.3' + PIN_DKML_APPS: '1.0.1' + PIN_OCAMLBUILD: '0.14.0' + PIN_OCAMLFIND: '1.9.1' + PIN_OCP_INDENT: '1.8.2-windowssupport' + PIN_PPX_EXPECT: 'v0.14.1' + PIN_PTIME: '0.8.6-msvcsupport' + PIN_TIME_NOW: 'v0.14.0' + PIN_WITH_DKML: '1.0.1' + run: | + sh .ci/sd4/run-teardown-dkml.sh GITHUB_WORKSPACE "$GITHUB_WORKSPACE" diff --git a/ci/setup-dkml/gh-windows/post/dune b/ci/setup-dkml/gh-windows/post/dune new file mode 100644 index 0000000..bc72faa --- /dev/null +++ b/ci/setup-dkml/gh-windows/post/dune @@ -0,0 +1,17 @@ +; DO NOT EDIT THIS FILE. It is auto-generated by generate-setup-dkml-scaffold +; Typical upgrade steps: +; opam upgrade dkml-workflows && opam exec -- generate-setup-dkml-scaffold && dune build '@gen-dkml' --auto-promote + +(rule + (alias gen-dkml) + (target action.gen.yml) + (action + (setenv + OCAMLRUNPARAM + b + (run gh-dkml-action-yml --phase post --output-windows %{target})))) + +(rule + (alias gen-dkml) + (action + (diff action.yml action.gen.yml))) diff --git a/ci/setup-dkml/gh-windows/pre/action.yml b/ci/setup-dkml/gh-windows/pre/action.yml new file mode 100644 index 0000000..bf1de91 --- /dev/null +++ b/ci/setup-dkml/gh-windows/pre/action.yml @@ -0,0 +1,1864 @@ +# setup-dkml +# Short form: sd4 + +# Any GitHub Job that includes this action must be in a strategy matrix. +# The matrix variables must include: +# - gh_os: windows-2019 +# abi_pattern: win32-windows_x86 +# dkml_host_abi: windows_x86 + +name: pre-dkml +author: Diskuv, Inc. +description: Setup before building OCaml native executables for Windows + +inputs: + FDOPEN_OPAMEXE_BOOTSTRAP: + description: "Use opam.exe from fdopen on Windows. Typically only used when bootstrapping Opam for the first time. May be needed to solve '\"create_process\" failed on sleep: Bad file descriptor' which may need https://github.com/ocaml/opam/commit/417b97d8cfada35682a0f4107eb2e4f9e24fba91" + required: false + default: 'false' + CACHE_PREFIX: + description: The prefix of the cache keys. + required: false + default: "v1" + OCAML_COMPILER: + description: "The version of the OCaml compiler, as tagged by the https://github.com/diskuv/dkml-base-compiler repository. If not specified the latest compiler, or the version associated with the 'dkml-compiler' input, is used. Currently only 4.12.1 is supported" + required: false + DKML_COMPILER: + description: "Git branch, tag or commit for dkml-compiler. However if OCAML_COMPILER is specified the 'dkml-base-compiler' version is taken from OCAML_COMPILER." + required: false + default: "" # "@repository@" = Opam ; "" = latest from default branch of git clone + CONF_DKML_CROSS_TOOLCHAIN: + description: "Git branch, tag or commit for conf-dkml-cross-toolchain" + required: false + default: "@repository@" # "@repository@" = Opam ; "" = latest from default branch of git clone + DISKUV_OPAM_REPOSITORY: + description: "Git branch, tag or commit for diskuv-opam-repository" + required: false + default: "" # DEFAULT_DISKUV_OPAM_REPOSITORY_TAG is used as default for empty strings + VERBOSE: + description: "Turn on verbose logging" + required: false + default: 'false' + SECONDARY_SWITCH: + description: "Turn on secondary 'two' switch" + required: false + default: 'false' + +runs: + using: "composite" + + steps: + - name: Full matrix variables + shell: bash # bash on Windows is Git Bash (an non-upgradable MSYS2 system) + # Every matrix variable lookup in this Action should use the output of this step. Even + # the matrix variables that the user must specify (ex. dkml_host_abi) should be + # referenced using [steps.full_matrix_vars.outputs.dkml_host_abi] rather than + # [matrix.dkml_host_abi] so that there is a single place to edit for variable changes. + id: full_matrix_vars + run: | + # Select correct Windows matrix variables + case "${{ matrix.dkml_host_abi }}" in + + windows_x86) + dkml_host_os='windows'; + opam_root_cacheable='D:/.opam'; + abi_pattern='win32-windows_x86'; + gh_os='windows-2019'; + gh_unix_shell='msys2 {0}'; + msys2_system='MINGW32'; + msys2_packages='mingw-w64-i686-pkg-config'; + exe_ext='.exe'; + bootstrap_opam_version='2.2.0-dkml20220801T155940Z'; + opam_abi='windows_x86'; + dkml_host_abi='windows_x86'; + opam_root='D:/.opam'; + vsstudio_hostarch='x64'; + vsstudio_arch='x86'; + ocaml_options='ocaml-option-32bit' ;; + + windows_x86_64) + dkml_host_os='windows'; + opam_root_cacheable='D:/.opam'; + abi_pattern='win32-windows_x86_64'; + gh_os='windows-2019'; + gh_unix_shell='msys2 {0}'; + msys2_system='CLANG64'; + msys2_packages='mingw-w64-clang-x86_64-pkg-config'; + exe_ext='.exe'; + bootstrap_opam_version='2.2.0-dkml20220801T155940Z'; + opam_abi='windows_x86_64'; + dkml_host_abi='windows_x86_64'; + opam_root='D:/.opam'; + vsstudio_hostarch='x64'; + vsstudio_arch='x64' ;; + + *) echo "FATAL: Unsupported dkml_host_abi=$dkml_host_abi in Windows action.yml"; exit 107 ;; + esac + + add() { + echo "$1=$2" | tee -a $GITHUB_OUTPUT | tee -a $GITHUB_ENV + } + + add dkml_host_abi "$dkml_host_abi" + add abi_pattern "$abi_pattern" + add opam_root "$opam_root" + add opam_root_cacheable "$opam_root_cacheable" + add exe_ext "${exe_ext:-}" + add bootstrap_opam_version "${bootstrap_opam_version:-}" + add ocaml_options "${ocaml_options:-}" + + add msys2_system "${msys2_system:-}" + add msys2_packages "${msys2_packages:-}" + add vsstudio_hostarch "${vsstudio_hostarch:-}" + add vsstudio_arch "${vsstudio_arch:-}" + add vsstudio_dir "${vsstudio_dir:-}" + add vsstudio_vcvarsver "${vsstudio_vcvarsver:-}" + add vsstudio_winsdkver "${vsstudio_winsdkver:-}" + add vsstudio_msvspreference "${vsstudio_msvspreference:-}" + add vsstudio_cmakegenerator "${vsstudio_cmakegenerator:-}" + + - name: Install MSYS2 (Windows) + uses: msys2/setup-msys2@v2 + with: + msystem: "${{ steps.full_matrix_vars.outputs.msys2_system }}" + update: true + install: >- + ${{ steps.full_matrix_vars.outputs.msys2_packages }} + wget + make + rsync + diffutils + patch + unzip + git + tar + xz + + - name: Uninstall MSYS2 conflicting executables (Windows) + # link.exe interferes with MSVC's link.exe + run: rm -vf /usr/bin/link.exe + shell: msys2 {0} + + - name: Write POSIX shell and AWK scripts + shell: bash + run: | + install -d .ci/sd4 + + cat > .ci/sd4/common-values.sh <<'end_of_script' + #!/bin/sh + + # ------------------------ Log Formatting ------------------------ + + TXT_SECTION="\e[94m" # bright blue + TXT_CLEAR="\e[0m" + + if [ "${GITLAB_CI:-}" = "true" ]; then + # https://docs.gitlab.com/ee/ci/jobs/#expand-and-collapse-job-log-sections + print_section_start() { + print_section_start_NAME=$1 + shift + printf "\e[0Ksection_start:%s:%s[collapsed=true]\r\e[0K" \ + "$(date +%s)" \ + "$print_section_start_NAME" + } + print_section_end() { + print_section_end_NAME=$1 + shift + printf "\e[0Ksection_end:%s:%s\r\e[0K\n" \ + "$(date +%s)" \ + "$print_section_end_NAME" + } + elif [ -n "${GITHUB_ENV:-}" ]; then + # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines + print_section_start() { + print_section_start_NAME=$1 + shift + printf "::group::" + } + print_section_end() { + print_section_end_NAME=$1 + shift + printf "::endgroup::\n" + } + else + print_section_start() { + print_section_start_NAME=$1 + shift + } + print_section_end() { + print_section_end_NAME=$1 + shift + } + fi + + section_begin() { + # https://docs.gitlab.com/ee/ci/yaml/script.html#add-color-codes-to-script-output + section_NAME=$1 + shift + section_HEADER=$1 + shift + print_section_start "$section_NAME" + printf "${TXT_SECTION}%s${TXT_CLEAR}\n" "$section_HEADER" + } + + section_end() { + section_NAME=$1 + shift + print_section_end "$section_NAME" + } + + # ------------------- Other Functions ----------------- + + transfer_dir() { + transfer_dir_SRC=$1 + shift + transfer_dir_DST=$1 + shift + # Remove the destination directory completely, but make sure the parent of the + # destination directory exists so `mv` will work + install -d "$transfer_dir_DST" + rm -rf "$transfer_dir_DST" + # Move + mv "$transfer_dir_SRC" "$transfer_dir_DST" + } + + # Set TEMP variable which is used, among other things, for OCaml's + # [Filename.temp_dir_name] on Win32, and by with-dkml.exe on Windows + export_temp_for_windows() { + if [ -x /usr/bin/cygpath ]; then + if [ -n "${RUNNER_TEMP:-}" ]; then + # GitHub Actions + TEMP=$(cygpath -am "$RUNNER_TEMP") + else + # GitLab CI/CD or desktop + install -d .ci/tmp + TEMP=$(cygpath -am ".ci/tmp") + fi + export TEMP + fi + } + + # Fixup opam_root on Windows to be mixed case. + # On input the following variables must be present: + # - opam_root + # - opam_root_cacheable + # On output the input variables will be modified _and_ the + # following variables will be available: + # - original_opam_root + # - original_opam_root_cacheable + # - unix_opam_root + # - unix_opam_root_cacheable + fixup_opam_root() { + # shellcheck disable=SC2034 + original_opam_root=$opam_root + # shellcheck disable=SC2034 + original_opam_root_cacheable=$opam_root_cacheable + if [ -x /usr/bin/cygpath ]; then + opam_root=$(/usr/bin/cygpath -m "$opam_root") + opam_root_cacheable=$(/usr/bin/cygpath -m "$opam_root_cacheable") + unix_opam_root=$(/usr/bin/cygpath -u "$opam_root") + unix_opam_root_cacheable=$(/usr/bin/cygpath -u "$opam_root_cacheable") + else + # shellcheck disable=SC2034 + unix_opam_root=$opam_root + # shellcheck disable=SC2034 + unix_opam_root_cacheable=$opam_root_cacheable + fi + } + end_of_script + + cat > .ci/sd4/run-checkout-code.sh <<'end_of_script' + #!/bin/sh + + # ================ + # checkout-code.sh + # ================ + # + # Checkouts all of the git source code. + # + # This should be done outside of + # dockcross (used by Linux) since a Docker-in-Docker container can have + # difficulties doing a git checkout (the Git credentials for any private + # repositories are likely not present). We don't care about any private + # repositories for DKML but any code that extends this (ex. DKSDK) may + # need to use private repositories. + + set -euf + + setup_WORKSPACE_VARNAME=$1 + shift + setup_WORKSPACE=$1 + shift + + # ------------------------ Functions ------------------------ + + # shellcheck source=./common-values.sh + . .ci/sd4/common-values.sh + + # Disable automatic garbage collection + git_disable_gc() { + git_disable_gc_NAME=$1 + shift + git -C ".ci/sd4/g/$git_disable_gc_NAME" config --local gc.auto 0 + } + + # Mimic the behavior of GitHub's actions/checkout@v3 + # - the plus symbol in 'git fetch ... origin +REF:refs/tags/v0.0' overrides any existing REF + git_checkout() { + git_checkout_NAME=$1 + shift + git_checkout_URL=$1 + shift + git_checkout_REF=$1 + shift + + if [ -e ".ci/sd4/g/$git_checkout_NAME" ]; then + git_disable_gc "$git_checkout_NAME" + git -C ".ci/sd4/g/$git_checkout_NAME" remote set-url origin "$git_checkout_URL" + git -C ".ci/sd4/g/$git_checkout_NAME" fetch --no-tags --progress --no-recurse-submodules --depth=1 origin "+${git_checkout_REF}:refs/tags/v0.0" + else + install -d ".ci/sd4/g/$git_checkout_NAME" + git -C ".ci/sd4/g/$git_checkout_NAME" -c init.defaultBranch=main init + git_disable_gc "$git_checkout_NAME" + git -C ".ci/sd4/g/$git_checkout_NAME" remote add origin "$git_checkout_URL" + git -C ".ci/sd4/g/$git_checkout_NAME" fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin "+${git_checkout_REF}:refs/tags/v0.0" + fi + git -C ".ci/sd4/g/$git_checkout_NAME" -c advice.detachedHead=false checkout --progress --force refs/tags/v0.0 + git -C ".ci/sd4/g/$git_checkout_NAME" log -1 --format='%H' + } + + # --------------------------------------------------------------------- + + section_begin checkout-info "Summary: code checkout" + + # shellcheck disable=SC2154 + echo " + ================ + checkout-code.sh + ================ + . + --------- + Arguments + --------- + WORKSPACE_VARNAME=$setup_WORKSPACE_VARNAME + WORKSPACE=$setup_WORKSPACE + . + ------ + Inputs + ------ + VERBOSE=${VERBOSE:-} + . + ------ + Matrix + ------ + dkml_host_abi=$dkml_host_abi + . + " + + section_end checkout-info + + install -d .ci/sd4/g + + # dkml-runtime-distribution + + case "$dkml_host_abi" in + windows_*) + section_begin checkout-dkml-runtime-distribution 'Checkout dkml-runtime-distribution' + git_checkout dkml-runtime-distribution https://github.com/diskuv/dkml-runtime-distribution.git "1a3ec82dd851751a95e6a4797387a8163c51520e" + section_end checkout-dkml-runtime-distribution + ;; + esac + + end_of_script + + cat > .ci/sd4/run-setup-dkml.sh <<'end_of_script' + #!/bin/sh + set -euf + + # Constants + SHA512_DEVNULL='cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e' + # Edited by https://gitlab.com/diskuv/diskuv-ocaml/contributors/release.sh + DEFAULT_DISKUV_OPAM_REPOSITORY_TAG=cc9518fa5630bfbe24d4c5e0e2cc29af513037ce + # Constants + # Should be edited by release.sh, but ... + # Can't be 1.0.0 or later until https://github.com/ocaml/opam-repository/pull/21704 ocaml-option-32bit + # can come back in. + DKML_VERSION=0.4.0 + + setup_WORKSPACE_VARNAME=$1 + shift + setup_WORKSPACE=$1 + shift + + # ------------------ Variables and functions ------------------------ + + # shellcheck source=./common-values.sh + . .ci/sd4/common-values.sh + + if [ "${VERBOSE:-}" = "true" ]; then + do_tar_rf() { + tar rvf "$@" + } + else + do_tar_rf() { + tar rf "$@" + } + fi + + # Make the standard input work as an OCaml string. + # This currently only escapes backslashes and double quotes. + escape_arg_as_ocaml_string() { + escape_arg_as_ocaml_string_ARG=$1 + shift + printf "%s" "$escape_arg_as_ocaml_string_ARG" | sed 's#\\#\\\\#g; s#"#\\"#g;' + } + + # Fixup opam_root on Windows to be mixed case. Set original_* and unix_* as well. + fixup_opam_root + + # Set TEMP variable for Windows + export_temp_for_windows + + # Load VS studio environment + if [ -e .ci/sd4/vsenv.sh ]; then + # shellcheck disable=SC1091 + . .ci/sd4/vsenv.sh + fi + + # ------------------------------------------------------------------- + + section_begin setup-info "Summary: setup-dkml" + + # shellcheck disable=SC2154 + echo " + ============= + setup-dkml.sh + ============= + . + --------- + Arguments + --------- + WORKSPACE_VARNAME=$setup_WORKSPACE_VARNAME + WORKSPACE=$setup_WORKSPACE + . + ------ + Inputs + ------ + FDOPEN_OPAMEXE_BOOTSTRAP=${FDOPEN_OPAMEXE_BOOTSTRAP:-} + DISKUV_OPAM_REPOSITORY=${DISKUV_OPAM_REPOSITORY:-} + DKML_COMPILER=${DKML_COMPILER:-} + OCAML_COMPILER=${OCAML_COMPILER:-} + CONF_DKML_CROSS_TOOLCHAIN=${CONF_DKML_CROSS_TOOLCHAIN:-} + SECONDARY_SWITCH=${SECONDARY_SWITCH:-} + MANYLINUX=${MANYLINUX:-} + VERBOSE=${VERBOSE:-} + . + ------------------- + Generated Constants + ------------------- + DKML_VERSION=$DKML_VERSION + DEFAULT_DISKUV_OPAM_REPOSITORY_TAG=$DEFAULT_DISKUV_OPAM_REPOSITORY_TAG + DEFAULT_DKML_COMPILER=$DEFAULT_DKML_COMPILER + . + ------ + Matrix + ------ + dkml_host_abi=$dkml_host_abi + bootstrap_opam_version=$bootstrap_opam_version + abi_pattern=$abi_pattern + opam_root=${opam_root} + opam_root_cacheable=${opam_root_cacheable} + original_opam_root=${original_opam_root} + original_opam_root_cacheable=${original_opam_root_cacheable} + unix_opam_root=${unix_opam_root} + unix_opam_root_cacheable=${unix_opam_root_cacheable} + dockcross_image=${dockcross_image:-} + dockcross_image_custom_prefix=${dockcross_image_custom_prefix:-} + dockcross_run_extra_args=${dockcross_run_extra_args:-} + docker_runner=${docker_runner:-} + in_docker=${in_docker:-} + ocaml_options=${ocaml_options:-} + . + ---- + Pins + ---- + PIN_BASE=${PIN_BASE} + PIN_BIGSTRINGAF=${PIN_BIGSTRINGAF} + PIN_CORE_KERNEL=${PIN_CORE_KERNEL} + PIN_CTYPES=${PIN_CTYPES} + PIN_CTYPES_FOREIGN=${PIN_CTYPES_FOREIGN} + PIN_CURLY=${PIN_CURLY} + PIN_DIGESTIF=${PIN_DIGESTIF} + PIN_DKML_APPS=${PIN_DKML_APPS} + PIN_DUNE=${PIN_DUNE} + PIN_DUNE_CONFIGURATOR=${PIN_DUNE_CONFIGURATOR} + PIN_OCAMLBUILD=${PIN_OCAMLBUILD} + PIN_OCAMLFIND=${PIN_OCAMLFIND} + PIN_OCP_INDENT=${PIN_OCP_INDENT} + PIN_PPX_EXPECT=${PIN_PPX_EXPECT} + PIN_PTIME=${PIN_PTIME} + PIN_TIME_NOW=${PIN_TIME_NOW} + PIN_WITH_DKML=${PIN_WITH_DKML} + . + " + case "$dkml_host_abi" in + windows_*) + # shellcheck disable=SC2153 + echo " + ------------- + Visual Studio + ------------- + VS_DIR=$VS_DIR + VS_VCVARSVER=$VS_VCVARSVER + VS_WINSDKVER=$VS_WINSDKVER + VS_MSVSPREFERENCE=$VS_MSVSPREFERENCE + VS_CMAKEGENERATOR=$VS_CMAKEGENERATOR + . + " + ;; + esac + section_end setup-info + + do_bootstrap() { + # Bootstrap from historical release + runit_BOOTSTRAPPED=0 + + # Bootstrap opam from fdopen (Windows) + if [ "$runit_BOOTSTRAPPED" = 0 ] && [ "${FDOPEN_OPAMEXE_BOOTSTRAP:-}" = "true" ]; then + if [ -e .ci/sd4/opam64/bin/opam.exe ] && [ -e .ci/sd4/opam64/bin/opam-installer.exe ]; then + runit_BOOTSTRAPPED=1 + else + case "$dkml_host_abi" in + windows_*) + echo 'Bootstrap opam from fdopen (Windows) ...' + install -d .ci/sd4/bs/bin + wget -O "$setup_WORKSPACE"/.ci/sd4/opam64.tar.xz https://github.com/fdopen/opam-repository-mingw/releases/download/0.0.0.2/opam64.tar.xz + + # this stalls: tar xvCfJ "$setup_WORKSPACE"/.ci/sd4 "$setup_WORKSPACE"/.ci/sd4/opam64.tar.xz + xz -v -d "$setup_WORKSPACE"/.ci/sd4/opam64.tar.xz + tar xvCf .ci/sd4 .ci/sd4/opam64.tar + + rm -rf "$setup_WORKSPACE"/.ci/sd4/bs/bin/Opam.Runtime.amd64 + mv -v "$setup_WORKSPACE"/.ci/sd4/opam64/bin/Opam.Runtime.amd64/ "$setup_WORKSPACE"/.ci/sd4/bs/bin/ + mv -v "$setup_WORKSPACE"/.ci/sd4/opam64/bin/opam.exe "$setup_WORKSPACE"/.ci/sd4/bs/bin/ + mv -v "$setup_WORKSPACE"/.ci/sd4/opam64/bin/opam-installer.exe "$setup_WORKSPACE"/.ci/sd4/bs/bin/ + + # diagnostics + ldd "$setup_WORKSPACE"/.ci/sd4/bs/bin/opam.exe + ldd "$setup_WORKSPACE"/.ci/sd4/bs/bin/opam-installer.exe + + runit_BOOTSTRAPPED=1 + ;; + esac + fi + fi + + # Bootstrap from historical release + if [ "$runit_BOOTSTRAPPED" = 0 ] && [ "$bootstrap_opam_version" != "os" ]; then + install -d .ci/sd4/bs + cd .ci/sd4/bs + + if [ ! -e version ] || [ "$(cat version)" != "$bootstrap_opam_version" ]; then + echo 'Bootstrap opam from historical release (non-Windows; Windows non-fdopen) ...' + if command -v curl; then + curl -L -o opam.tar.gz "https://github.com/diskuv/dkml-component-opam/releases/download/v${bootstrap_opam_version}/dkml-component-staging-opam.tar.gz" + else + wget -O opam.tar.gz "https://github.com/diskuv/dkml-component-opam/releases/download/v${bootstrap_opam_version}/dkml-component-staging-opam.tar.gz" + fi + tar tvfz opam.tar.gz + tar xfz opam.tar.gz "./staging-files/${dkml_host_abi}/" + rm -rf bin/ + mv "staging-files/${dkml_host_abi}/bin" . + rm -rf "${abi_pattern}" + printf "%s" "${bootstrap_opam_version}" >version + fi + + rm -f opam.tar.gz + cd ../../.. + + runit_BOOTSTRAPPED=1 + fi + + # Bootstrap from package manager or GitHub ocaml/opam release + case "$runit_BOOTSTRAPPED,$bootstrap_opam_version,$dkml_host_abi" in + 0,os,darwin_*) + if ! command -v opam; then + echo 'Bootstrap opam from package manager (macOS) ...' + brew install gpatch + brew install opam + fi + runit_BOOTSTRAPPED=1 + ;; + 0,os,linux_x86) + if [ ! -x .ci/sd4/bs/bin/opam ]; then + echo 'Bootstrap opam from GitHub ocaml/opam release (Linux x86) ...' + install -d .ci/sd4/bs/bin + wget -O .ci/sd4/bs/bin/opam.tmp https://github.com/ocaml/opam/releases/download/2.1.2/opam-2.1.2-i686-linux + sha512_check=$(openssl sha512 2>&1 /dev/null | cut -f 2 -d ' ') + check="85a480d60e09a7d37fa0d0434ed97a3187434772ceb4e7e8faa5b06bc18423d004af3ad5849c7d35e72dca155103257fd6b1178872df8291583929eb8f884b6a" + test "$sha512" = "$check" + chmod +x .ci/sd4/bs/bin/opam.tmp + mv .ci/sd4/bs/bin/opam.tmp .ci/sd4/bs/bin/opam + else + echo "openssl 512 option unsupported." + exit 61 + fi + fi + runit_BOOTSTRAPPED=1 + ;; + 0,os,linux_x86_64) + if [ ! -x .ci/sd4/bs/bin/opam ]; then + echo 'Bootstrap opam from GitHub ocaml/opam release (Linux x86_64) ...' + install -d .ci/sd4/bs/bin + wget -O .ci/sd4/bs/bin/opam.tmp https://github.com/ocaml/opam/releases/download/2.1.2/opam-2.1.2-x86_64-linux + sha512_check=$(openssl sha512 2>&1 /dev/null | cut -f 2 -d ' ') + check="c0657ecbd4dc212587a4da70c5ff0402df95d148867be0e1eb1be8863a2851015f191437c3c99b7c2b153fcaa56cac99169c76ec94c5787750d7a59cd1fbb68b" + test "$sha512" = "$check" + chmod +x .ci/sd4/bs/bin/opam.tmp + mv .ci/sd4/bs/bin/opam.tmp .ci/sd4/bs/bin/opam + else + echo "openssl 512 option unsupported." + exit 61 + fi + fi + runit_BOOTSTRAPPED=1 + ;; + esac + } + section_begin bootstrap-opam 'Bootstrap opam' + do_bootstrap + section_end bootstrap-opam + + # Start environment distribution tarball + # We use .tar rather than .tar.gz/.tar.bz2 because we can repeatedly add to an uncompressed .tar. But we need to + # start with an empty tarball since some tar programs will only add ('tar rf xyz.tar') to an existing .tar. + install -d .ci/sd4/dist + tar cf .ci/sd4/dist/opam-with-env.tar -T /dev/null + + do_get_dockcross() { + if [ -n "${dockcross_image:-}" ]; then + # The dockcross script is super-slow + section_begin get-dockcross 'Get dockcross binary (ManyLinux)' + install -d .ci/sd4 + # shellcheck disable=SC2086 + docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_custom_prefix:-}${dockcross_image:-}" >.ci/sd4/dockcross.gen + + # PROBLEM 1 + # --------- + # Super-annoying stderr output from dockcross at line: + # tty -s && [ -z "$MSYS" ] && TTY_ARGS=-ti + # When there is no tty, get: + # tty: ignoring all arguments + # not a tty + # So replace 'tty -s &&' with 'false &&' + sed 's/tty -s &&/false \&\&/' .ci/sd4/dockcross.gen >.ci/sd4/dockcross-real + rm -f .ci/sd4/dockcross.gen + chmod +x .ci/sd4/dockcross-real + + # PROBLEM 2 + # --------- + # By default dockcross for ManyLinux will chown -R all python packages; super-slow (~10 seconds)! + # Confer: https://github.com/dockcross/dockcross/blob/master/manylinux-common/pre_exec.sh + # That kills speed for any repetitive dockcross invocation. + # + # BUT it is unnecessary to chown -R when the current user is root, because inside the Docker container + # the files are already root! + # + # The chown -R (within pre_exec.sh) is not run when the user ids are not passed in. + # Confer: https://github.com/dockcross/dockcross/blob/96d87416f639af0204bdd42553e4b99315ca8476/imagefiles/entrypoint.sh#L21-L53 + # + # So explicitly call the entrypoint if root! + case "$dkml_host_abi" in + # https://github.com/dockcross/dockcross/blob/master/linux-x86/linux32-entrypoint.sh + linux_x86) dockcross_entrypoint=/dockcross/linux32-entrypoint.sh ;; + *) dockcross_entrypoint=/dockcross/entrypoint.sh ;; + esac + cat > .ci/sd4/dockcross <&2 "Missing functionality (\${f}) (in cygwin)." ; exit 1 ; } ; + done ; + HOST_PWD="\$( cygpath -w "\$( readlink -f "\$( pwd ;)" ; )" ; )" ; + else + HOST_PWD=\$PWD + [ -L \$HOST_PWD ] && HOST_PWD=\$(readlink \$HOST_PWD) + fi + + # ---------- End of dockcross script snippet ------- + + # Handle: dockcross --args "-v X:Y --platform P" + ARGS= + if [ "\$#" -ge 1 ] && [ "\$1" = "--args" ]; then + shift + ARGS=\$1 + shift + fi + + # Directly invoke entrypoint + exec docker run --entrypoint /bin/bash \ + --rm \ + \${ARGS:-} \ + -v "\$HOST_PWD":/work \ + ${dockcross_image_custom_prefix:-}${dockcross_image:-} ${dockcross_entrypoint} "\$@" + else + HERE=\$(dirname "\$0") + HERE=\$(cd "\$HERE" && pwd) + exec "\$HERE/dockcross-real" "\$@" + fi + EOF + chmod +x .ci/sd4/dockcross + + # Bundle for consumers of setup-dkml.yml + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/dockcross .ci/sd4/dockcross-real + + section_end get-dockcross + fi + } + do_get_dockcross + + if [ -n "${dockcross_image:-}" ]; then + # rsync needs to be available, even after Docker container disappears + if [ ! -e .ci/sd4/bs/bin/rsync ]; then + section_begin get-opam-prereqs-in-dockcross 'Get Opam prerequisites (ManyLinux)' + install -d .ci/sd4/bs/bin + # shellcheck disable=SC2016 + .ci/sd4/dockcross --args "${dockcross_run_extra_args:-}" sh -c 'sudo yum install -y rsync && install $(command -v rsync) .ci/sd4/bs/bin' + section_end get-opam-prereqs-in-dockcross + fi + fi + + # Opam prerequisites for using opam (not for installing opam) + + { + if [ -n "${docker_runner:-}" ]; then + # rsync needs to be available, even after Docker container disappears + if [ ! -e .ci/sd4/bs/bin/rsync.deps ]; then + section_begin get-opam-prereqs-in-docker 'Get Opam prerequisites (Linux Docker)' + install -d .ci/sd4/bs/bin + ${docker_runner} sh -c ' + apt-get update && + apt-get install -y rsync && + ldd /usr/bin/rsync && + ls -l /lib/i386-linux-gnu/libpopt.so.0 /lib/i386-linux-gnu/libacl.so.1 /lib/i386-linux-gnu/libattr.so.1 && + tar cCfhz / /work/.ci/sd4/bs/bin/deps.tar.gz /usr/bin/rsync /lib/i386-linux-gnu/libpopt.so.0 + ' + touch .ci/sd4/bs/bin/rsync.deps + section_end get-opam-prereqs-in-docker + fi + fi + + # Bundle Opam prerequisites (ManyLinux or Linux Docker) + if [ -n "${docker_runner:-}" ] || [ -n "${dockcross_image:-}" ]; then + # Bundle for consumers of setup-dkml.yml + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/bs/bin/rsync + fi + } + + # Get Opam Cache + do_get_opam_cache() { + if [ "$unix_opam_root_cacheable" = "$unix_opam_root" ]; then return; fi + if [ ! -e "$unix_opam_root_cacheable" ]; then return; fi + section_begin get-opam-cache "Transferring Opam cache to $original_opam_root_cacheable" + echo Starting transfer # need some output or GitLab CI will not display the section duration + transfer_dir "$unix_opam_root_cacheable" "$unix_opam_root" + echo Finished transfer + section_end get-opam-cache + } + do_get_opam_cache + + # Setup Opam + + do_write_opam_scripts() { + case "${FDOPEN_OPAMEXE_BOOTSTRAP:-},$dkml_host_abi" in + true,windows_*) + # With fdopen's opam.exe, 'os-distribution = "cygwinports"'. But native Windows opam.exe has 'os-distribution = "win32"'. + # But on Windows we always want MSYS2 or native Windows libraries, not Cygwin. If cygwinports then + # code like https://github.com/ocaml/opam-repository/blob/08cbb8258bd4bf30cd6f307c958911a29d537b54/packages/conf-pkg-config/conf-pkg-config.2/opam#L36 + # will fail. So always set 'os-distribution = "win32"' on Windows. + PATCH_OS_DISTRIBUTION_WIN32=true + # With fdopen's opam.exe, no 'exe = ".exe"' is set because Cygwin does not need file extensions. + # Native Windows requires a .exe extension. + PATCH_EXE_WIN32=true + ;; + *) + PATCH_OS_DISTRIBUTION_WIN32=false + PATCH_EXE_WIN32=false + ;; + esac + + # --------------------- + # Empty opam repository + # --------------------- + + install -d .ci/sd4/eor + cat >.ci/sd4/eor/repo <.ci/sd4/troubleshoot-opam.sh <&2 + find "\$OPAMROOT"/log -mindepth 1 -maxdepth 1 -name "*.out" ! -name "log-*.out" ! -name "ocaml-variants-*.out" | while read -r dump_on_error_LOG; do + dump_on_error_BLOG=\$(basename "\$dump_on_error_LOG") + printf "\n\n========= [TROUBLESHOOTING] %s ===========\n\n" "\$dump_on_error_BLOG" >&2 + awk -v BLOG="\$dump_on_error_BLOG" '{print "[" BLOG "]", \$0}' "\$dump_on_error_LOG" >&2 + done + printf "\nScroll up to see the [TROUBLESHOOTING] logs that begin at the [START OF TROUBLESHOOTING] line\n" >&2 + EOF + + chmod +x .ci/sd4/troubleshoot-opam.sh + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/troubleshoot-opam.sh + + # --------------- + # Create Opam support scripts (not needed for all platforms) + # The PATH to find opam must work internally in setup-dkml.yml (sd4/bs/bin) and + # by consumers of setup-dkml.yml (sd4/opamexe) + # --------------- + + USER_ID=$(id -u) + GROUP_ID=$(id -g) + USER_NAME=$(id -un) + GROUP_NAME=$(id -gn) + + case "${opam_root}" in + /* | ?:*) # /a/b/c or C:\Windows + validate_supports_docker() { + echo "Docker only supported with relative paths for the opam root, not: ${opam_root}" >&2 + exit 3 + } + ;; + *) # relative path + validate_supports_docker() { + true + } + cat >.ci/sd4/opam-in-docker <&2 + set +e + opam "\$@" + exitcode=\$? + if [ \$troubleshooting = 1 ]; then + [ \$exitcode = 0 ] || "/work/.ci/sd4/troubleshoot-opam.sh" \$OPAMROOT + fi + exit \$exitcode + EOF + chmod +x .ci/sd4/opam-in-docker + ;; + esac + + cat >.ci/sd4/deescalate <.ci/sd4/opam-with-env <&2 + cat .ci/sd4/opam-in-docker >&2 + echo '________________________' >&2 + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/opam-with-env .ci/sd4/opam-in-docker .ci/sd4/edr + + elif [ -n "${docker_runner:-}" ]; then + + cat >.ci/sd4/opam-with-env <&2 + cat .ci/sd4/opam-in-docker >&2 + echo '________________________' >&2 + echo '__ deescalate __' >&2 + cat .ci/sd4/deescalate >&2 + echo '________________' >&2 + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/opam-with-env .ci/sd4/opam-in-docker .ci/sd4/deescalate + + else + + cat >.ci/sd4/opam-with-env <&2 + set +e + opam "\$@" + exitcode=\$? + if [ \$troubleshooting = 1 ]; then + [ \$exitcode = 0 ] || "\${PROJECT_DIR}/.ci/sd4/troubleshoot-opam.sh" \$OPAMROOT + fi + exit \$exitcode + EOF + chmod +x .ci/sd4/opam-with-env + + # Bundle for consumers of setup-dkml.yml + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/opam-with-env + + fi + echo '__ opam-with-env __' >&2 + cat .ci/sd4/opam-with-env >&2 + echo '___________________' >&2 + + # ------- + # opamrun + # ------- + + install -d .ci/sd4/opamrun + cat >.ci/sd4/opamrun/opamrun <>"$GITHUB_PATH" + # Special case: GITHUB_PATH does not influence msys2.CMD of msys2/setup-msys2@v2, so place in real MSYS2 PATH + if [ -n "${MSYSTEM:-}" ]; then + install -d /usr/local/bin + install .ci/sd4/opamrun/opamrun /usr/local/bin/opamrun + fi + fi + + # Place opamrun in the immediate PATH + PATH="$setup_WORKSPACE/.ci/sd4/opamrun:$PATH" + + # Complicated Opam sequence is because: + # 1. Opam's default curl does not work on Windows, + # and `opam init` does not provide a way to change it (TODO: need + # a PR!). + # 2. We have to separate the Opam download cache from the other Opam + # caches + if [ ! -s "$opam_root/.ci.root-init" ]; then # non-empty init file so can be cached irrespective of existence + section_begin opam-init 'Initialize opam root' + + # Clear any partial previous attempt + rm -rf "$opam_root" + + case "$dkml_host_abi,${in_docker:-}" in + windows_*,*) + eor=$(cygpath -am "$setup_WORKSPACE"/.ci/sd4/eor) + opamrun init --disable-sandboxing --no-setup --kind local --bare "$eor" + case "$(opamrun --version)" in + 2.0.*) echo 'download-command: wget' >>"$opam_root/config" ;; + *) opamrun option --yes --global download-command=wget ;; + esac + ;; + *,true) + opamrun init --disable-sandboxing --no-setup --kind local --bare "/work/.ci/sd4/eor" + ;; + *) + opamrun init --disable-sandboxing --no-setup --kind local --bare "$setup_WORKSPACE/.ci/sd4/eor" + ;; + esac + echo yes > "$opam_root/.ci.root-init" + + section_end opam-init + fi + + section_begin opam-vars "Summary: opam global variables" + opamrun --no-troubleshooting var --global || true + section_end opam-vars + + # Build OCaml + + do_switch_create() { + do_switch_create_NAME=$1 + shift + + section_begin "switch-create-$do_switch_create_NAME" "Create opam switch '$do_switch_create_NAME'" + # Create, or recreate, the Opam switch. The Opam switch should not be + # cached except for the compiler (confer docs for setup-ocaml GitHub + # Action) which is the 'dkml' switch (or the 'two' switch). + # Check if the switch name is present in the Opam root (which may come from cache) + NOMINALLY_PRESENT=false + if opamrun switch list --short | grep "^${do_switch_create_NAME}\$"; then NOMINALLY_PRESENT=true; fi + + # Check if the switch is actually present in case of cache incoherence + # or corrupt Opam state that could result in: + # Error: No config file found for switch dkml. Switch broken? + if [ $NOMINALLY_PRESENT = true ] && [ ! -e "$opam_root/$do_switch_create_NAME/.opam-switch/switch-config" ]; then + # Remove the switch name from Opam root, and any partial switch state. + # Ignore inevitable warnings/failure about missing switch. + opamrun --no-troubleshooting switch remove "$do_switch_create_NAME" --yes || true + rm -rf "${opam_root:?}/$do_switch_create_NAME" + NOMINALLY_PRESENT=false + fi + + if [ $NOMINALLY_PRESENT = false ]; then + opamrun switch create "$do_switch_create_NAME" --empty --yes + fi + section_end "switch-create-$do_switch_create_NAME" + } + do_switch_create dkml + if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_switch_create two + else + section_begin "switch-create-two" "Create empty opam switch 'two'" + # Always create a secondary switch ... just empty. Avoid problems with cache content missing + # and idempotency. + opamrun --no-troubleshooting switch remove two --yes || true + rm -rf "$opam_root/two" + opamrun switch create two --empty --yes + section_end "switch-create-two" + fi + + do_switch_active() { + section_begin "switch-active" "Set dkml as active switch" + opamrun switch set dkml --yes + section_end "switch-active" + } + do_switch_active + + do_opam_repositories_add() { + section_begin "opam-repo-add" "Add 'diskuv' opam repository" + if ! opamrun --no-troubleshooting repository list -s | grep '^diskuv'; then + opamrun repository add diskuv "git+https://github.com/diskuv/diskuv-opam-repository.git#${DISKUV_OPAM_REPOSITORY:-$DEFAULT_DISKUV_OPAM_REPOSITORY_TAG}" --yes --dont-select + fi + section_end "opam-repo-add" + } + do_opam_repositories_add + + do_opam_repositories_config() { + do_opam_repositories_config_NAME=$1 + shift + + section_begin "opam-repo-$do_opam_repositories_config_NAME" "Attach repositories to $do_opam_repositories_config_NAME" + + if [ ! -s "$opam_root/.ci.$do_opam_repositories_config_NAME.repo-init" ]; then # non-empty init file so can be cached irrespective of existence + opamrun --no-troubleshooting repository remove default --switch "$do_opam_repositories_config_NAME" --yes || true + opamrun --no-troubleshooting repository remove diskuv --switch "$do_opam_repositories_config_NAME" --yes || true + opamrun repository add default --switch "$do_opam_repositories_config_NAME" --yes + opamrun repository add diskuv --switch "$do_opam_repositories_config_NAME" --yes + echo yes > "$opam_root/.ci.$do_opam_repositories_config_NAME.repo-init" + fi + + section_end "opam-repo-$do_opam_repositories_config_NAME" + } + do_opam_repositories_config dkml + if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_opam_repositories_config two + fi + + do_opam_repositories_update() { + section_begin "opam-repo-update" "Update opam repositories" + # The default repository may be the initial 'eor' (empty) repository + opamrun repository set-url default https://opam.ocaml.org --yes + # Always set the `diskuv` repository url since it can change + opamrun repository set-url diskuv "git+https://github.com/diskuv/diskuv-opam-repository.git#${DISKUV_OPAM_REPOSITORY:-$DEFAULT_DISKUV_OPAM_REPOSITORY_TAG}" --yes --dont-select + # Update both `default` and `diskuv` Opam repositories + opamrun update default diskuv + section_end "opam-repo-update" + } + do_opam_repositories_update + + do_pins() { + do_pins_NAME=$1 + shift + + # dkml-base-compiler + + if [ "${DKML_COMPILER:-}" != '@repository@' ] && [ -z "${DKML_COMPILER:-}" ] && [ -z "${OCAML_COMPILER:-}" ]; then + section_begin checkout-dkml-base-compiler "Pin dkml-base-compiler to default ${DEFAULT_DKML_COMPILER} (neither dkml-base-compiler nor OCAML_COMPILER specified) for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action dkml-base-compiler "https://github.com/diskuv/dkml-compiler.git#${DEFAULT_DKML_COMPILER}" + section_end checkout-dkml-base-compiler + elif [ "${DKML_COMPILER:-}" != '@repository@' ] && [ -n "${DKML_COMPILER:-}" ] && [ -z "${OCAML_COMPILER:-}" ]; then + section_begin checkout-dkml-base-compiler "Pin dkml-base-compiler to $DKML_COMPILER (dkml-base-compiler specified; no OCAML_COMPILER specified) for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action dkml-base-compiler "https://github.com/diskuv/dkml-compiler.git#${DKML_COMPILER}" + section_end checkout-dkml-base-compiler + elif [ -n "${OCAML_COMPILER:-}" ]; then + # Validate OCAML_COMPILER (OCAML_COMPILER specified) + case "${OCAML_COMPILER:-}" in + 4.12.1) true ;; + *) + echo "OCAML_COMPILER version ${OCAML_COMPILER:-} is not supported" >&2 + exit 109 + ;; + esac + + section_begin checkout-dkml-base-compiler "Pin dkml-base-compiler (OCAML_COMPILER specified) for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action dkml-base-compiler "https://github.com/diskuv/dkml-compiler.git#${OCAML_COMPILER}-v${DKML_VERSION}" + section_end checkout-dkml-base-compiler + fi + + # conf-dkml-cross-toolchain + + if [ "${CONF_DKML_CROSS_TOOLCHAIN:-}" != '@repository@' ]; then + section_begin checkout-conf-dkml-cross-toolchain "Pin conf-dkml-cross-toolchain for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action conf-dkml-cross-toolchain "https://github.com/diskuv/conf-dkml-cross-toolchain.git#$CONF_DKML_CROSS_TOOLCHAIN" + section_end checkout-conf-dkml-cross-toolchain + fi + + # patches necessary for Windows in diskuv-opam-repository + # + # - ocamlfind and ocamlbuild + # + # - dune-configurator (and hence Dune) + # Dune 2.9.1 and 3.0.2 will fail to build jst-config.v0.14.1 because for jst-config/discover/discover.ml Dune does: + # cl -nologo -O2 -Gy- -MD -I Z:/.opam_root-cached-8/installer-ocaml/lib/ocaml -o C:\Users\beckf\AppData\Local\Temp\build_f18aec_dune\ocaml-configurator4d3858\c-test-31\test.obj -c C:\Users\beckf\AppData\Local\Temp\build_f18aec_dune\ocaml-configurator4d3858\c-test-31\test.c advapi32.lib ws2_32.lib version.lib + # instead of + # cl -nologo -O2 -Gy- -MD -I Z:/.opam_root-cached-8/installer-ocaml/lib/ocaml /FoC:\Users\beckf\AppData\Local\Temp\build_f18aec_dune\ocaml-configurator4d3858\c-test-31\test.obj -c C:\Users\beckf\AppData\Local\Temp\build_f18aec_dune\ocaml-configurator4d3858\c-test-31\test.c advapi32.lib ws2_32.lib version.lib + # with the (irrelevant) test.c file: + # #include + # #include + # + # #ifdef ARCH_BIG_ENDIAN + # const char *s0 = "BEGIN-0-true-END"; + # #else + # const char *s0 = "BEGIN-0-false-END"; + # #endif + # + # #ifdef ARCH_SIXTYFOUR + # const char *s1 = "BEGIN-1-true-END"; + # #else + # const char *s1 = "BEGIN-1-false-END"; + # #endif + # + # The actual problem is dune-configurator ... we only have patches in Diskuv + # repository up until 2.9.3. Need to upstream fix the problem. + # + # - ppx_expect; only patch is for v0.14.1. Need to upstream fix the problem. + # - base; patches for v0.14.1/2/3. Need to upstream fix the problem. + section_begin "opam-pins-$do_pins_NAME" "Opam pins for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version base "${PIN_BASE}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version bigstringaf "${PIN_BIGSTRINGAF}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version core_kernel "${PIN_CORE_KERNEL}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ctypes "${PIN_CTYPES}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ctypes-foreign "${PIN_CTYPES_FOREIGN}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version curly "${PIN_CURLY}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version digestif "${PIN_DIGESTIF}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version dkml-apps "${PIN_DKML_APPS}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version dune "${PIN_DUNE}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version dune-configurator "${PIN_DUNE_CONFIGURATOR}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ocamlbuild "${PIN_OCAMLBUILD}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ocamlfind "${PIN_OCAMLFIND}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ocp-indent "${PIN_OCP_INDENT}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ppx_expect "${PIN_PPX_EXPECT}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ptime "${PIN_PTIME}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version time_now "${PIN_TIME_NOW}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version with-dkml "${PIN_WITH_DKML}" + section_end "opam-pins-$do_pins_NAME" + } + + do_pins dkml + if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_pins two + fi + + do_use_vsstudio() { + do_use_vsstudio_NAME=$1 + shift + case "$dkml_host_abi" in + windows_*) + section_begin "use-vsstudio-$do_use_vsstudio_NAME" "Use Visual Studio in dkml-* Opam packages (Windows) for $do_use_vsstudio_NAME switch" + + # shellcheck disable=SC2153 + E_VS_DIR=$(escape_arg_as_ocaml_string "$VS_DIR") + # shellcheck disable=SC2153 + E_VS_VCVARSVER=$(escape_arg_as_ocaml_string "$VS_VCVARSVER") + # shellcheck disable=SC2153 + E_VS_WINSDKVER=$(escape_arg_as_ocaml_string "$VS_WINSDKVER") + # shellcheck disable=SC2153 + E_VS_MSVSPREFERENCE=$(escape_arg_as_ocaml_string "$VS_MSVSPREFERENCE") + # shellcheck disable=SC2153 + E_VS_CMAKEGENERATOR=$(escape_arg_as_ocaml_string "$VS_CMAKEGENERATOR") + + case "$(opamrun --version)" in + 2.0.*) + if [ "${in_docker}" = "true" ]; then + echo Opam 2.0 support in dockcross to use a portable opam var prefix not yet implemented + exit 67 + fi + OP=$(opamrun var prefix --switch "$do_use_vsstudio_NAME") + OPSC=$OP/.opam-switch/switch-config + if grep setenv: "$OPSC"; then + echo "INFO: Updating switch-config. Old was:" + awk '{print ">> " $0}' "$OPSC" + + awk '$1=="setenv:"{x=1} x==0{print} x==1 && $0=="]"{x=0}' "$OPSC" >"$OPSC".trimmed + mv "$OPSC".trimmed "$OPSC" + fi + echo 'setenv: [' >>"$OPSC" + echo ' [DKML_COMPILE_SPEC = "1"]' >>"$OPSC" + echo ' [DKML_COMPILE_TYPE = "VS"]' >>"$OPSC" + echo " [DKML_COMPILE_VS_DIR = \"$E_VS_DIR\"]" >>"$OPSC" + echo " [DKML_COMPILE_VS_VCVARSVER = \"$E_VS_VCVARSVER\"]" >>"$OPSC" + echo " [DKML_COMPILE_VS_WINSDKVER = \"$E_VS_WINSDKVER\"]" >>"$OPSC" + echo " [DKML_COMPILE_VS_MSVSPREFERENCE = \"$E_VS_MSVSPREFERENCE\"]" >>"$OPSC" + echo " [DKML_COMPILE_VS_CMAKEGENERATOR = \"$E_VS_CMAKEGENERATOR\"]" >>"$OPSC" + echo " [DKML_HOST_ABI = \"${dkml_host_abi}\"]" >>"$OPSC" + echo ']' >>"$OPSC" + cat "$OPSC" >&2 # print + ;; + *) + opamrun option --switch "$do_use_vsstudio_NAME" setenv= # reset + opamrun option --switch "$do_use_vsstudio_NAME" setenv+='DKML_COMPILE_SPEC = "1"' + opamrun option --switch "$do_use_vsstudio_NAME" setenv+='DKML_COMPILE_TYPE = "VS"' + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_DIR = \"$E_VS_DIR\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_VCVARSVER = \"$E_VS_VCVARSVER\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_WINSDKVER = \"$E_VS_WINSDKVER\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_MSVSPREFERENCE = \"$E_VS_MSVSPREFERENCE\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_CMAKEGENERATOR = \"$E_VS_CMAKEGENERATOR\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_HOST_ABI = \"${dkml_host_abi}\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv # print + ;; + esac + + # shellcheck disable=SC2016 + opamrun exec --switch "$do_use_vsstudio_NAME" -- sh -c 'echo $VCToolsRedistDir' + + section_end "use-vsstudio-$do_use_vsstudio_NAME" + ;; + esac + } + if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_use_vsstudio two + fi + do_use_vsstudio dkml + + # Because dune.X.Y.Z+shim (and any user DKML packages) requires DKML installed (after all, it is just + # a with-dkml.exe shim), we need either dkmlvars-v2.sexp or DKML environment + # variables. Confer: Dkml_runtimelib.Dkml_context.get_dkmlversion + # + # grep matches either: + # [... [DiskuvOCamlVersion = "1.0.1"] ...] + # DiskuvOCamlVersion = "1.0.1" + do_setenv() { + do_setenv_SWITCH=$1 + shift + opamrun option --switch "$do_setenv_SWITCH" setenv > ".ci/sd4/setenv.$do_setenv_SWITCH.txt" + if ! grep -q '\(^|\[\)DiskuvOCamlVarsVersion ' ".ci/sd4/setenv.$do_setenv_SWITCH.txt"; then + opamrun option --switch "$do_setenv_SWITCH" setenv+='DiskuvOCamlVarsVersion = "2"' + fi + if ! grep -q '\(^|\[\)DiskuvOCamlVersion ' ".ci/sd4/setenv.$do_setenv_SWITCH.txt"; then + opamrun option --switch "$do_setenv_SWITCH" setenv+="DiskuvOCamlVersion = \"$DKML_VERSION\"" + fi + case "${dkml_host_abi}" in + windows_*) + if ! grep -q '\(^|\[\)DiskuvOCamlMSYS2Dir ' ".ci/sd4/setenv.$do_setenv_SWITCH.txt"; then + if [ -x /usr/bin/cygpath ]; then + MSYS2_DIR_NATIVE=$(/usr/bin/cygpath -aw /) + else + # If we are already inside MSYS2 then MSYSTEM_PREFIX should be set. But cygpath should be there as well!! + echo "FATAL: Could not locate MSYS2: there was no cygpath" >&2 + exit 3 + fi + MSYS2_DIR_NATIVE_ESCAPED=$(printf "%s" "$MSYS2_DIR_NATIVE" | sed 's/\\/\\\\/g') + opamrun option --switch "$do_setenv_SWITCH" setenv+="DiskuvOCamlMSYS2Dir = \"$MSYS2_DIR_NATIVE_ESCAPED\"" + fi + esac + } + if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_setenv two + fi + do_setenv dkml + + do_install_compiler() { + do_install_compiler_NAME=$1 + shift + section_begin "install-compiler-$do_install_compiler_NAME" "Install OCaml compiler for $do_install_compiler_NAME switch" + opamrun pin list --switch "$do_install_compiler_NAME" + # shellcheck disable=SC2086 + opamrun upgrade --switch "$do_install_compiler_NAME" --yes dkml-base-compiler conf-dkml-cross-toolchain ${ocaml_options:-} + section_end "install-compiler-$do_install_compiler_NAME" + } + do_install_compiler dkml + if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_install_compiler two + fi + + do_summary() { + do_summary_NAME=$1 + shift + section_begin "summary-$do_summary_NAME" "Summary: $do_summary_NAME switch" + opamrun var --switch "$do_summary_NAME" + opamrun exec --switch "$do_summary_NAME" -- ocamlc -config + section_end "summary-$do_summary_NAME" + } + do_summary dkml + if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_summary two + fi + + end_of_script + + cat > .ci/sd4/run-teardown-dkml.sh <<'end_of_script' + #!/bin/sh + set -euf + + teardown_WORKSPACE_VARNAME=$1 + shift + teardown_WORKSPACE=$1 + shift + + # ------------------ Variables and functions ------------------------ + + # shellcheck source=./common-values.sh + . .ci/sd4/common-values.sh + + # Fixup opam_root on Windows to be mixed case. Set original_* and unix_* as well. + fixup_opam_root + + # Set TEMP variable for Windows + export_temp_for_windows + + # ------------------------------------------------------------------- + + section_begin teardown-info "Summary: teardown-dkml" + + # shellcheck disable=SC2154 + echo " + ================ + teardown-dkml.sh + ================ + . + --------- + Arguments + --------- + WORKSPACE_VARNAME=$teardown_WORKSPACE_VARNAME + WORKSPACE=$teardown_WORKSPACE + . + ------ + Inputs + ------ + VERBOSE=${VERBOSE:-} + . + ------ + Matrix + ------ + dkml_host_abi=$dkml_host_abi + opam_root=${opam_root} + opam_root_cacheable=${opam_root_cacheable} + original_opam_root=${original_opam_root} + original_opam_root_cacheable=${original_opam_root_cacheable} + unix_opam_root=${unix_opam_root} + unix_opam_root_cacheable=${unix_opam_root_cacheable} + . + " + section_end teardown-info + + # Done with Opam cache! + do_save_opam_cache() { + if [ "$unix_opam_root_cacheable" = "$unix_opam_root" ]; then return; fi + section_begin save-opam-cache "Transfer Opam cache to $original_opam_root" + echo Starting transfer # need some output or GitLab CI will not display the section duration + transfer_dir "$unix_opam_root" "$unix_opam_root_cacheable" + echo Finished transfer + section_end save-opam-cache + } + do_save_opam_cache + + do_fill_skipped_cache_entries() { + section_begin fill-skipped-cache-entries "Populate skipped cache entries" + + # Needed to stop GitLab CI/CD cache warnings 'no matching files', etc. + if [ ! -e .ci/sd4/vsenv.sh ]; then + install -d .ci/sd4 + rm -f .ci/sd4/vsenv.sh + touch .ci/sd4/vsenv.sh + chmod +x .ci/sd4/vsenv.sh + echo "Created empty vsenv.sh" + else + echo "Found vsenv.sh" + fi + + if [ ! -e msys64 ]; then + rm -rf msys64 + install -d msys64 + touch msys64/.keep + echo "Created empty msys64" + else + echo "Found msys64/" + fi + + if [ -s "$unix_opam_root_cacheable/.ci.dkml.repo-init" ]; then + echo "Found non-empty $unix_opam_root_cacheable/.ci.dkml.repo-init" + else + touch "$unix_opam_root_cacheable/.ci.dkml.repo-init" + echo "Created empty $unix_opam_root_cacheable/.ci.dkml.repo-init" + fi + + if [ -s "$unix_opam_root_cacheable/.ci.two.repo-init" ]; then + echo "Found non-empty $unix_opam_root_cacheable/.ci.two.repo-init" + else + touch "$unix_opam_root_cacheable/.ci.two.repo-init" + echo "Created empty $unix_opam_root_cacheable/.ci.two.repo-init" + fi + + section_end fill-skipped-cache-entries + } + do_fill_skipped_cache_entries + + do_at_least_one_artifact() { + install -d dist + find dist -mindepth 1 -maxdepth 1 >.ci/dist.files + if [ ! -s .ci/dist.files ]; then + section_begin one-artifact "Create empty artifact file" + + # Avoid confusing "ERROR: No files to upload" in GitLab CI + touch dist/.keep + echo "Created dist/.keep" + + section_end one-artifact + fi + } + do_at_least_one_artifact + + end_of_script + + install -d .ci/sd4 + + cat > .ci/sd4/msvcenv.awk <<'end_of_script' + # MSVC environment variables: + # 1. https://docs.microsoft.com/en-us/cpp/build/reference/cl-environment-variables?view=msvc-170 + # 2. https://docs.microsoft.com/en-us/cpp/build/reference/linking?view=msvc-170#link-environment-variables (except TMP) + # 3. VCToolsRedistDir: https://docs.microsoft.com/en-us/cpp/windows/redistributing-visual-cpp-files?view=msvc-170#locate-the-redistributable-files + BEGIN{FS="="} + $1=="CL"||$1=="_CL_"||$1=="INCLUDE"||$1=="LIBPATH" {print "export " $0} + $1=="LINK"||$1=="_LINK_"||$1=="LIB"||$1=="PATH" {print "export " $0} + $1=="VCToolsRedistDir" {print "export " $0} + + end_of_script + + cat > .ci/sd4/msvcpath.awk <<'end_of_script' + { + # trim leading and trailing space + sub(/^ */, ""); + sub(/ *$/, ""); + + print "export PATH='" $0 "'"; + } + end_of_script + + - name: Write PowerShell and Batch scripts + shell: pwsh + run: | + $Content = @' + # Diagnose Visual Studio environment variables (Windows) + # This wastes time and has lots of rows! Only run if "VERBOSE" GitHub input key. + if ( "${env:VERBOSE}" -eq "true" ) { + if (Test-Path -Path "C:\Program Files (x86)\Windows Kits\10\include") { + Get-ChildItem "C:\Program Files (x86)\Windows Kits\10\include" + } + if (Test-Path -Path "C:\Program Files (x86)\Windows Kits\10\Extension SDKs\WindowsDesktop") { + Get-ChildItem "C:\Program Files (x86)\Windows Kits\10\Extension SDKs\WindowsDesktop" + } + + $env:PSModulePath += "$([System.IO.Path]::PathSeparator).ci\sd4\g\dkml-runtime-distribution\src\windows" + Import-Module Machine + + $allinstances = Get-VSSetupInstance + $allinstances | ConvertTo-Json -Depth 5 + } + + # Make export expression [SN]NAME=[SV]VALUE[EV] + # where [SN] is start name and [SV] and [EV] are start and end value + if (("${env:GITLAB_CI}" -eq "true") -or ("${env:PC_CI}" -eq "true")) { + # Executed immediately in POSIX shell, so must be a real POSIX shell variable declaration + $ExportSN = "export " + $ExportSV = "'" + $ExportEV = "'" + $ExportExt = ".sh" + } else { + # Goes into $env:GITHUB_ENV, so must be plain NAME=VALUE + $ExportSN = "" + $ExportSV = "" + $ExportEV = "" + $ExportExt = ".github" + } + + # Locate Visual Studio (Windows) + if ("${env:vsstudio_dir}" -eq "" -and (!(Test-Path -Path .ci/sd4/vsenv${ExportExt}))) { + $env:PSModulePath += "$([System.IO.Path]::PathSeparator).ci\sd4\g\dkml-runtime-distribution\src\windows" + Import-Module Machine + + $CompatibleVisualStudios = Get-CompatibleVisualStudios -ErrorIfNotFound + $CompatibleVisualStudios + $ChosenVisualStudio = ($CompatibleVisualStudios | Select-Object -First 1) + $VisualStudioProps = Get-VisualStudioProperties -VisualStudioInstallation $ChosenVisualStudio + $VisualStudioProps + + Write-Output "${ExportSN}VS_DIR=${ExportSV}$($VisualStudioProps.InstallPath)${ExportEV}" > .ci/sd4/vsenv${ExportExt} + Write-Output "${ExportSN}VS_VCVARSVER=${ExportSV}$($VisualStudioProps.VcVarsVer)${ExportEV}" >> .ci/sd4/vsenv${ExportExt} + Write-Output "${ExportSN}VS_WINSDKVER=${ExportSV}$($VisualStudioProps.WinSdkVer)${ExportEV}" >> .ci/sd4/vsenv${ExportExt} + Write-Output "${ExportSN}VS_MSVSPREFERENCE=${ExportSV}$($VisualStudioProps.MsvsPreference)${ExportEV}" >> .ci/sd4/vsenv${ExportExt} + Write-Output "${ExportSN}VS_CMAKEGENERATOR=${ExportSV}$($VisualStudioProps.CMakeGenerator)${ExportEV}" >> .ci/sd4/vsenv${ExportExt} + } + + # Link to hardcoded Visual Studio (Windows) + if ("${env:vsstudio_dir}" -ne "") { + Write-Output "${ExportSN}VS_DIR=${ExportSV}${env:vsstudio_dir}${ExportEV}" > .ci/sd4/vsenv${ExportExt} + Write-Output "${ExportSN}VS_VCVARSVER=${ExportSV}${env:vsstudio_vcvarsver}${ExportEV}" >> .ci/sd4/vsenv${ExportExt} + Write-Output "${ExportSN}VS_WINSDKVER=${ExportSV}${env:vsstudio_winsdkver}${ExportEV}" >> .ci/sd4/vsenv${ExportExt} + Write-Output "${ExportSN}VS_MSVSPREFERENCE=${ExportSV}${env:vsstudio_msvspreference}${ExportEV}" >> .ci/sd4/vsenv${ExportExt} + Write-Output "${ExportSN}VS_CMAKEGENERATOR=${ExportSV}${env:vsstudio_cmakegenerator}${ExportEV}" >> .ci/sd4/vsenv${ExportExt} + } + + '@ + Set-Content -Path ".ci\sd4\config-vsstudio.ps1" -Encoding Unicode -Value $Content + + $Content = @' + @ECHO OFF + + REM The OCaml dkml-base-compiler will compile fine but any other + REM packages (ocamlbuild, etc.) which + REM need a native compiler will fail without the MSVC compiler in the + REM PATH. There isn't a `with-dkml.exe` alternative available at + REM this stage of the GitHub workflow. + call "%VS_DIR%\Common7\Tools\VsDevCmd.bat" -no_logo -host_arch=%vsstudio_hostarch% -arch=%vsstudio_arch% -vcvars_ver=%VS_VCVARSVER% -winsdk=%VS_WINSDKVER% + if %ERRORLEVEL% neq 0 ( + echo. + echo.The "%VS_DIR%\Common7\Tools\VsDevCmd.bat" command failed + echo.with exit code %ERRORLEVEL%. + echo. + exit /b %ERRORLEVEL% + ) + + REM VsDevCmd.bat turns off echo; be explicit if we want it on or off + @echo OFF + + REM MSVC environment variables in Unix format. + echo %PATH% > .ci\sd4\msvcpath + + + REM * We can't use `msys2 -c` directly to query for all MSVC environment variables + REM because it stomps over the PATH. So we are inside a Batch script to do the query. + msys2 -c "set | grep -v '^PATH=' | awk -f .ci/sd4/msvcenv.awk > .ci/sd4/msvcenv" + + REM (For some reason no shell commands can happen after the last msys2 -c in GitHub Actions + REM so split into multiple steps! + '@ + Set-Content -Path ".ci\sd4\get-msvcpath-into-msys2.cmd" -Encoding Default -Value $Content + + - name: Prepare cache keys + # An undocumented bug with GitHub Actions is that a space in the key + # will "succeed" but it never gets restored. So we will hash a + # user-friendly file instead. + shell: bash + run: | + set -x + echo '317446-${{ inputs.FDOPEN_OPAMEXE_BOOTSTRAP }}-${{ steps.full_matrix_vars.outputs.dkml_host_abi }}-${{ steps.full_matrix_vars.outputs.opam_abi }}-${{ steps.full_matrix_vars.outputs.bootstrap_opam_version }}' > .ci/sd4/cachekey.opam.binaries + echo '${{ steps.full_matrix_vars.outputs.abi_pattern }}-${{ steps.full_matrix_vars.outputs.vsstudio_arch }}-${{ steps.full_matrix_vars.outputs.vsstudio_hostarch }}-${{ steps.full_matrix_vars.outputs.vsstudio_dir }}-${{ steps.full_matrix_vars.outputs.vsstudio_vcvarsver }}-${{ steps.full_matrix_vars.outputs.vsstudio_winsdkver }}-${{ steps.full_matrix_vars.outputs.vsstudio_msvspreference }}-${{ steps.full_matrix_vars.outputs.vsstudio_cmakegenerator }}' > .ci/sd4/cachekey.vsstudio + echo '${{ inputs.OCAML_COMPILER }}-${{ inputs.DISKUV_OPAM_REPOSITORY }}-${{ inputs.DKML_COMPILER }}-${{ inputs.CONF_DKML_CROSS_TOOLCHAIN }}' > .ci/sd4/cachekey.ci.inputs + + # Bootstrapping Opam + + - name: Cache Opam bootstrap by OS + uses: actions/cache@v3 + id: cache-sd4-bs + with: + path: .ci/sd4/bs + key: + "1_${{ inputs.CACHE_PREFIX }}_${{ runner.os }}-sd4-bs-${{ hashFiles('.ci/sd4/cachekey.opam.binaries') }}" + + - name: Cache Opam root by (host,target) + uses: actions/cache@v3 + id: cache-sd4-opamroot + with: + path: | + ${{ steps.full_matrix_vars.outputs.opam_root_cacheable }}/config + ${{ steps.full_matrix_vars.outputs.opam_root_cacheable }}/dkml + ${{ steps.full_matrix_vars.outputs.opam_root_cacheable }}/two + ${{ steps.full_matrix_vars.outputs.opam_root_cacheable }}/repo + ${{ steps.full_matrix_vars.outputs.opam_root_cacheable }}/download-cache + ${{ steps.full_matrix_vars.outputs.opam_root_cacheable }}/.ci.root-init + ${{ steps.full_matrix_vars.outputs.opam_root_cacheable }}/.ci.dkml.repo-init + ${{ steps.full_matrix_vars.outputs.opam_root_cacheable }}/.ci.two.repo-init + key: + "1_${{ inputs.CACHE_PREFIX }}_${{ env.DKML_VERSION }}_${{ steps.full_matrix_vars.outputs.abi_pattern }}-sd4-opamroot-${{ hashFiles('.ci/sd4/cachekey.opam.binaries') }}-${{ hashFiles('cachekey.ci.inputs') }}" + + # The action/checkout steps used for pinning will re-use existing Git objects + # because of caching + - name: Cache Git checkouts of Opam pins by OS + uses: actions/cache@v3 + id: cache-sd4-git + with: + path: .ci/sd4/g + key: + "1_${{ inputs.CACHE_PREFIX }}_${{ runner.os }}-sd4-git-${{ hashFiles('.ci/sd4/cachekey.ci.inputs') }}" + + # Checkout code + + - name: Checkout DKML code + shell: bash + env: + # Push down inputs variables for GitHub/GitLab portable scripts + VERBOSE: "${{ inputs.VERBOSE }}" + run: 'sh .ci/sd4/run-checkout-code.sh GITHUB_WORKSPACE "$GITHUB_WORKSPACE"' + + # Setup C compiler + + - name: Cache Visual Studio environment variables by (host,target) + uses: actions/cache@v3 + id: cache-sd4-vsenv + with: + path: .ci/sd4/vsenv.github + key: "1_${{ inputs.CACHE_PREFIX }}_sd4-vsenv-${{ hashFiles('cachekey.vsstudio') }}" + + - name: Configure Visual Studio (Windows) + shell: pwsh + run: | + .ci\sd4\config-vsstudio.ps1 + + - name: Export Visual Studio location and type (Windows) + shell: pwsh + run: | + Get-Content ${env:GITHUB_WORKSPACE}/.ci/sd4/vsenv.github + Get-Content ${env:GITHUB_WORKSPACE}/.ci/sd4/vsenv.github >> $env:GITHUB_ENV + + - name: Capture Visual Studio compiler environment (1/2) (Windows) + shell: cmd + run: | + call .ci\sd4\get-msvcpath-into-msys2.cmd + + - name: Capture Visual Studio compiler environment (2/2) (Windows) + shell: msys2 {0} + run: | + #!/bin/sh + set -euf + cat .ci/sd4/msvcpath | tr -d '\r' | cygpath --path -f - | awk -f .ci/sd4/msvcpath.awk >> .ci/sd4/msvcenv + + tail -n100 .ci/sd4/msvcpath .ci/sd4/msvcenv >&2 + + - name: Compile 'dkml' switch + shell: msys2 {0} + env: + # Push down inputs variables for GitHub/GitLab portable scripts + # When non-empty, instead of building the standard components from the + # central Opam repository, use the github/gitlab development repositories + # directly. + FDOPEN_OPAMEXE_BOOTSTRAP: "${{ inputs.FDOPEN_OPAMEXE_BOOTSTRAP }}" + CONF_DKML_CROSS_TOOLCHAIN: "${{ inputs.CONF_DKML_CROSS_TOOLCHAIN }}" + DISKUV_OPAM_REPOSITORY: "${{ inputs.DISKUV_OPAM_REPOSITORY }}" + DKML_COMPILER: "${{ inputs.DKML_COMPILER }}" + OCAML_COMPILER: "${{ inputs.OCAML_COMPILER }}" + SECONDARY_SWITCH: "${{ inputs.SECONDARY_SWITCH }}" + VERBOSE: "${{ inputs.VERBOSE }}" + + # autogen from global_env_vars. + DEFAULT_DKML_COMPILER: '4.12.1-v1.0.2' + PIN_BASE: 'v0.14.3' + PIN_BIGSTRINGAF: '0.8.0' + PIN_CORE_KERNEL: 'v0.14.2' + PIN_CTYPES_FOREIGN: '0.19.2-windowssupport-r4' + PIN_CTYPES: '0.19.2-windowssupport-r4' + PIN_CURLY: '0.2.1-windows-env_r2' + PIN_DIGESTIF: '1.0.1' + PIN_DUNE: '2.9.3+shim.1.0.2~r0' + PIN_DUNE_CONFIGURATOR: '2.9.3' + PIN_DKML_APPS: '1.0.1' + PIN_OCAMLBUILD: '0.14.0' + PIN_OCAMLFIND: '1.9.1' + PIN_OCP_INDENT: '1.8.2-windowssupport' + PIN_PPX_EXPECT: 'v0.14.1' + PIN_PTIME: '0.8.6-msvcsupport' + PIN_TIME_NOW: 'v0.14.0' + PIN_WITH_DKML: '1.0.1' + run: | + sh .ci/sd4/run-setup-dkml.sh GITHUB_WORKSPACE "$GITHUB_WORKSPACE" + + # During cache read (the "Cache Opam root by (host,target)" step) we get: + # Cache Size: ~212 MB (222698916 B) + # C:\Windows\System32\tar.exe -z -xf D:/a/_temp/9b656ed6-e727-4499-b4f8-b34d5979d42b/cache.tgz -P -C D:/a/dkml-workflows-regular-example/dkml-workflows-regular-example + # ../../.opam/download-cache/md5/65/65e6dc9b305ccbed1267275fe180f538: Can't create '\\\\?\\D:\\a\\dkml-workflows-regular-example\\dkml-workflows-regular-example\\..\\..\\.opam\\download-cache\\md5\\65\\65e6dc9b305ccbed1267275fe180f538' + # ../../.opam/download-cache/md5/63/63b2ecad76cf56102074f3203fc8c0be: Can't create '\\\\?\\D:\\a\\dkml-workflows-regular-example\\dkml-workflows-regular-example\\..\\..\\.opam\\download-cache\\md5\\63\\63b2ecad76cf56102074f3203fc8c0be' + # tar.exe: Error exit delayed from previous errors. + # Warning: Failed to restore: Tar failed with error: The process 'C:\Windows\System32\tar.exe' failed with exit code 1 + # Cache not found for input keys: v1_win32-windows_x86-sd4-opamroot-f674af417c46862703911bc9aee57c19d6c436776c657533610dc233fae8e8af-5850674459249f5d9ea150785baa5177201737ce2c7d59834aa7457c06c929e7 + # So get rid of Opam's md5 download cache. + rm -rf "${opam_root}/download-cache/md5" + + # Expose opamrun to remaining GitHub Action steps + opamrunabs="$GITHUB_WORKSPACE/.ci/sd4/opamrun" + if [ -x /usr/bin/cygpath ]; then opamrunabs=$(/usr/bin/cygpath -aw "$opamrunabs"); fi + echo "$opamrunabs" >> $GITHUB_PATH + + # Special case: GITHUB_PATH does not influence msys2.CMD of msys2/setup-msys2@v2, so place in real MSYS2 PATH + if [ -n "${MSYSTEM:-}" ]; then + install -d /usr/local/bin + install -v .ci/sd4/opamrun/opamrun /usr/local/bin/opamrun + fi diff --git a/ci/setup-dkml/gh-windows/pre/dune b/ci/setup-dkml/gh-windows/pre/dune new file mode 100644 index 0000000..1235657 --- /dev/null +++ b/ci/setup-dkml/gh-windows/pre/dune @@ -0,0 +1,17 @@ +; DO NOT EDIT THIS FILE. It is auto-generated by generate-setup-dkml-scaffold +; Typical upgrade steps: +; opam upgrade dkml-workflows && opam exec -- generate-setup-dkml-scaffold && dune build '@gen-dkml' --auto-promote + +(rule + (alias gen-dkml) + (target action.gen.yml) + (action + (setenv + OCAMLRUNPARAM + b + (run gh-dkml-action-yml --phase pre --output-windows %{target})))) + +(rule + (alias gen-dkml) + (action + (diff action.yml action.gen.yml))) diff --git a/ci/setup-dkml/pc/setup-dkml-linux_x86.sh b/ci/setup-dkml/pc/setup-dkml-linux_x86.sh new file mode 100644 index 0000000..3c7e9da --- /dev/null +++ b/ci/setup-dkml/pc/setup-dkml-linux_x86.sh @@ -0,0 +1,1512 @@ +#!/bin/sh +set -euf + +# Reset environment so no conflicts with a parent Opam or OCaml system +unset OPAMROOT +unset OPAM_SWITCH_PREFIX +unset OPAMSWITCH +unset CAML_LD_LIBRARY_PATH +unset OCAMLLIB +unset OCAML_TOPLEVEL_PATH + +export PC_PROJECT_DIR="$PWD" +export FDOPEN_OPAMEXE_BOOTSTRAP=false +export CACHE_PREFIX=v1 +export OCAML_COMPILER= +export DKML_COMPILER= +export CONF_DKML_CROSS_TOOLCHAIN=@repository@ +export DISKUV_OPAM_REPOSITORY= +export SECONDARY_SWITCH=false +# autogen from global_env_vars. +export DEFAULT_DKML_COMPILER='4.12.1-v1.0.2' +export PIN_BASE='v0.14.3' +export PIN_BIGSTRINGAF='0.8.0' +export PIN_CORE_KERNEL='v0.14.2' +export PIN_CTYPES_FOREIGN='0.19.2-windowssupport-r4' +export PIN_CTYPES='0.19.2-windowssupport-r4' +export PIN_CURLY='0.2.1-windows-env_r2' +export PIN_DIGESTIF='1.0.1' +export PIN_DUNE='2.9.3+shim.1.0.2~r0' +export PIN_DUNE_CONFIGURATOR='2.9.3' +export PIN_DKML_APPS='1.0.1' +export PIN_OCAMLBUILD='0.14.0' +export PIN_OCAMLFIND='1.9.1' +export PIN_OCP_INDENT='1.8.2-windowssupport' +export PIN_PPX_EXPECT='v0.14.1' +export PIN_PTIME='0.8.6-msvcsupport' +export PIN_TIME_NOW='v0.14.0' +export PIN_WITH_DKML='1.0.1' + +usage() { + echo 'Setup Diskuv OCaml (DKML) compiler on a desktop PC.' >&2 + echo 'usage: setup-dkml-linux_x86.sh [options]' >&2 + echo 'Options:' >&2 + + # Context variables + echo " --PC_PROJECT_DIR=. Defaults to the current directory (${PC_PROJECT_DIR})" >&2 + + # Input variables + echo " --FDOPEN_OPAMEXE_BOOTSTRAP=true|false. Defaults to: ${FDOPEN_OPAMEXE_BOOTSTRAP}" >&2 + echo " --CACHE_PREFIX=. Defaults to: ${CACHE_PREFIX}" >&2 + echo " --OCAML_COMPILER=. --DKML_COMPILER takes priority. If --DKML_COMPILER is not set and --OCAML_COMPILER is set, then the specified OCaml version tag of dkml-compiler (ex. 4.12.1) is used. Defaults to: ${OCAML_COMPILER}" >&2 + echo " --DKML_COMPILER=. Unspecified or blank is the latest from the default branch (main) of dkml-compiler. Defaults to: ${DKML_COMPILER}" >&2 + echo " --SECONDARY_SWITCH=true|false. If true then the secondary switch named 'two' is created, in addition to the always-present 'dkml' switch. Defaults to: ${SECONDARY_SWITCH}" >&2 + echo " --CONF_DKML_CROSS_TOOLCHAIN=. Unspecified or blank is the latest from the default branch (main) of conf-dkml-cross-toolchain. @repository@ is the latest from Opam. Defaults to: ${CONF_DKML_CROSS_TOOLCHAIN}" >&2 + echo " --DISKUV_OPAM_REPOSITORY=. Defaults to the value of --DEFAULT_DISKUV_OPAM_REPOSITORY_TAG (see below)" >&2 + + # autogen from global_env_vars. + echo " --DEFAULT_DKML_COMPILER=. Defaults to: ${DEFAULT_DKML_COMPILER}" >&2 + echo " --PIN_BASE=. Defaults to: ${PIN_BASE}" >&2 + echo " --PIN_BIGSTRINGAF=. Defaults to: ${PIN_BIGSTRINGAF}" >&2 + echo " --PIN_CORE_KERNEL=. Defaults to: ${PIN_CORE_KERNEL}" >&2 + echo " --PIN_CTYPES_FOREIGN=. Defaults to: ${PIN_CTYPES_FOREIGN}" >&2 + echo " --PIN_CTYPES=. Defaults to: ${PIN_CTYPES}" >&2 + echo " --PIN_CURLY=. Defaults to: ${PIN_CURLY}" >&2 + echo " --PIN_DIGESTIF=. Defaults to: ${PIN_DIGESTIF}" >&2 + echo " --PIN_DUNE=. Defaults to: ${PIN_DUNE}" >&2 + echo " --PIN_DUNE_CONFIGURATOR=. Defaults to: ${PIN_DUNE_CONFIGURATOR}" >&2 + echo " --PIN_DKML_APPS=. Defaults to: ${PIN_DKML_APPS}" >&2 + echo " --PIN_OCAMLBUILD=. Defaults to: ${PIN_OCAMLBUILD}" >&2 + echo " --PIN_OCAMLFIND=. Defaults to: ${PIN_OCAMLFIND}" >&2 + echo " --PIN_OCP_INDENT=. Defaults to: ${PIN_OCP_INDENT}" >&2 + echo " --PIN_PPX_EXPECT=. Defaults to: ${PIN_PPX_EXPECT}" >&2 + echo " --PIN_PTIME=. Defaults to: ${PIN_PTIME}" >&2 + echo " --PIN_TIME_NOW=. Defaults to: ${PIN_TIME_NOW}" >&2 + echo " --PIN_WITH_DKML=. Defaults to: ${PIN_WITH_DKML}" >&2 + exit 2 +} +fail() { + echo "Error: $*" >&2 + exit 3 +} +unset file + +OPTIND=1 +while getopts :h-: option; do + case $option in + h) usage ;; + -) case $OPTARG in + PC_PROJECT_DIR) fail "Option \"$OPTARG\" missing argument" ;; + PC_PROJECT_DIR=*) PC_PROJECT_DIR=${OPTARG#*=} ;; + CACHE_PREFIX) fail "Option \"$OPTARG\" missing argument" ;; + CACHE_PREFIX=*) CACHE_PREFIX=${OPTARG#*=} ;; + FDOPEN_OPAMEXE_BOOTSTRAP) fail "Option \"$OPTARG\" missing argument" ;; + FDOPEN_OPAMEXE_BOOTSTRAP=*) FDOPEN_OPAMEXE_BOOTSTRAP=${OPTARG#*=} ;; + OCAML_COMPILER) fail "Option \"$OPTARG\" missing argument" ;; + OCAML_COMPILER=*) OCAML_COMPILER=${OPTARG#*=} ;; + DKML_COMPILER) fail "Option \"$OPTARG\" missing argument" ;; + DKML_COMPILER=*) DKML_COMPILER=${OPTARG#*=} ;; + SECONDARY_SWITCH) fail "Option \"$OPTARG\" missing argument" ;; + SECONDARY_SWITCH=*) SECONDARY_SWITCH=${OPTARG#*=} ;; + CONF_DKML_CROSS_TOOLCHAIN) fail "Option \"$OPTARG\" missing argument" ;; + CONF_DKML_CROSS_TOOLCHAIN=*) CONF_DKML_CROSS_TOOLCHAIN=${OPTARG#*=} ;; + DISKUV_OPAM_REPOSITORY) fail "Option \"$OPTARG\" missing argument" ;; + DISKUV_OPAM_REPOSITORY=*) DISKUV_OPAM_REPOSITORY=${OPTARG#*=} ;; + # autogen from global_env_vars. + DEFAULT_DKML_COMPILER) fail "Option \"$OPTARG\" missing argument" ;; + DEFAULT_DKML_COMPILER=*) DEFAULT_DKML_COMPILER=${OPTARG#*=} ;; + PIN_BASE) fail "Option \"$OPTARG\" missing argument" ;; + PIN_BASE=*) PIN_BASE=${OPTARG#*=} ;; + PIN_BIGSTRINGAF) fail "Option \"$OPTARG\" missing argument" ;; + PIN_BIGSTRINGAF=*) PIN_BIGSTRINGAF=${OPTARG#*=} ;; + PIN_CORE_KERNEL) fail "Option \"$OPTARG\" missing argument" ;; + PIN_CORE_KERNEL=*) PIN_CORE_KERNEL=${OPTARG#*=} ;; + PIN_CTYPES_FOREIGN) fail "Option \"$OPTARG\" missing argument" ;; + PIN_CTYPES_FOREIGN=*) PIN_CTYPES_FOREIGN=${OPTARG#*=} ;; + PIN_CTYPES) fail "Option \"$OPTARG\" missing argument" ;; + PIN_CTYPES=*) PIN_CTYPES=${OPTARG#*=} ;; + PIN_CURLY) fail "Option \"$OPTARG\" missing argument" ;; + PIN_CURLY=*) PIN_CURLY=${OPTARG#*=} ;; + PIN_DIGESTIF) fail "Option \"$OPTARG\" missing argument" ;; + PIN_DIGESTIF=*) PIN_DIGESTIF=${OPTARG#*=} ;; + PIN_DUNE) fail "Option \"$OPTARG\" missing argument" ;; + PIN_DUNE=*) PIN_DUNE=${OPTARG#*=} ;; + PIN_DUNE_CONFIGURATOR) fail "Option \"$OPTARG\" missing argument" ;; + PIN_DUNE_CONFIGURATOR=*) PIN_DUNE_CONFIGURATOR=${OPTARG#*=} ;; + PIN_DKML_APPS) fail "Option \"$OPTARG\" missing argument" ;; + PIN_DKML_APPS=*) PIN_DKML_APPS=${OPTARG#*=} ;; + PIN_OCAMLBUILD) fail "Option \"$OPTARG\" missing argument" ;; + PIN_OCAMLBUILD=*) PIN_OCAMLBUILD=${OPTARG#*=} ;; + PIN_OCAMLFIND) fail "Option \"$OPTARG\" missing argument" ;; + PIN_OCAMLFIND=*) PIN_OCAMLFIND=${OPTARG#*=} ;; + PIN_OCP_INDENT) fail "Option \"$OPTARG\" missing argument" ;; + PIN_OCP_INDENT=*) PIN_OCP_INDENT=${OPTARG#*=} ;; + PIN_PPX_EXPECT) fail "Option \"$OPTARG\" missing argument" ;; + PIN_PPX_EXPECT=*) PIN_PPX_EXPECT=${OPTARG#*=} ;; + PIN_PTIME) fail "Option \"$OPTARG\" missing argument" ;; + PIN_PTIME=*) PIN_PTIME=${OPTARG#*=} ;; + PIN_TIME_NOW) fail "Option \"$OPTARG\" missing argument" ;; + PIN_TIME_NOW=*) PIN_TIME_NOW=${OPTARG#*=} ;; + PIN_WITH_DKML) fail "Option \"$OPTARG\" missing argument" ;; + PIN_WITH_DKML=*) PIN_WITH_DKML=${OPTARG#*=} ;; + help) usage ;; + help=*) fail "Option \"${OPTARG%%=*}\" has unexpected argument" ;; + *) fail "Unknown long option \"${OPTARG%%=*}\"" ;; + esac ;; + '?') fail "Unknown short option \"$OPTARG\"" ;; + :) fail "Short option \"$OPTARG\" missing argument" ;; + *) fail "Bad state in getopts (OPTARG=\"$OPTARG\")" ;; + esac +done +shift $((OPTIND - 1)) + +# Set matrix variables +# autogen from pc_vars. only linux_x86 +export dkml_host_os="linux" +export opam_root_cacheable=".ci/o" +export abi_pattern="manylinux2014-linux_x86" +export comment="(CentOS 7, etc.)" +export bootstrap_opam_version="2.2.0-dkml20220801T155940Z" +export dkml_host_abi="linux_x86" +export opam_root=".ci/o" +export in_docker="true" +export dockcross_image="dockcross/manylinux2014-x86" +export dockcross_run_extra_args="--platform linux/386" + + +########################### before_script ############################### + +echo "Writing scripts ..." +install -d .ci/sd4 + +cat > .ci/sd4/common-values.sh <<'end_of_script' +#!/bin/sh + +# ------------------------ Log Formatting ------------------------ + +TXT_SECTION="\e[94m" # bright blue +TXT_CLEAR="\e[0m" + +if [ "${GITLAB_CI:-}" = "true" ]; then + # https://docs.gitlab.com/ee/ci/jobs/#expand-and-collapse-job-log-sections + print_section_start() { + print_section_start_NAME=$1 + shift + printf "\e[0Ksection_start:%s:%s[collapsed=true]\r\e[0K" \ + "$(date +%s)" \ + "$print_section_start_NAME" + } + print_section_end() { + print_section_end_NAME=$1 + shift + printf "\e[0Ksection_end:%s:%s\r\e[0K\n" \ + "$(date +%s)" \ + "$print_section_end_NAME" + } +elif [ -n "${GITHUB_ENV:-}" ]; then + # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines + print_section_start() { + print_section_start_NAME=$1 + shift + printf "::group::" + } + print_section_end() { + print_section_end_NAME=$1 + shift + printf "::endgroup::\n" + } +else + print_section_start() { + print_section_start_NAME=$1 + shift + } + print_section_end() { + print_section_end_NAME=$1 + shift + } +fi + +section_begin() { + # https://docs.gitlab.com/ee/ci/yaml/script.html#add-color-codes-to-script-output + section_NAME=$1 + shift + section_HEADER=$1 + shift + print_section_start "$section_NAME" + printf "${TXT_SECTION}%s${TXT_CLEAR}\n" "$section_HEADER" +} + +section_end() { + section_NAME=$1 + shift + print_section_end "$section_NAME" +} + +# ------------------- Other Functions ----------------- + +transfer_dir() { + transfer_dir_SRC=$1 + shift + transfer_dir_DST=$1 + shift + # Remove the destination directory completely, but make sure the parent of the + # destination directory exists so `mv` will work + install -d "$transfer_dir_DST" + rm -rf "$transfer_dir_DST" + # Move + mv "$transfer_dir_SRC" "$transfer_dir_DST" +} + +# Set TEMP variable which is used, among other things, for OCaml's +# [Filename.temp_dir_name] on Win32, and by with-dkml.exe on Windows +export_temp_for_windows() { + if [ -x /usr/bin/cygpath ]; then + if [ -n "${RUNNER_TEMP:-}" ]; then + # GitHub Actions + TEMP=$(cygpath -am "$RUNNER_TEMP") + else + # GitLab CI/CD or desktop + install -d .ci/tmp + TEMP=$(cygpath -am ".ci/tmp") + fi + export TEMP + fi +} + +# Fixup opam_root on Windows to be mixed case. +# On input the following variables must be present: +# - opam_root +# - opam_root_cacheable +# On output the input variables will be modified _and_ the +# following variables will be available: +# - original_opam_root +# - original_opam_root_cacheable +# - unix_opam_root +# - unix_opam_root_cacheable +fixup_opam_root() { + # shellcheck disable=SC2034 + original_opam_root=$opam_root + # shellcheck disable=SC2034 + original_opam_root_cacheable=$opam_root_cacheable + if [ -x /usr/bin/cygpath ]; then + opam_root=$(/usr/bin/cygpath -m "$opam_root") + opam_root_cacheable=$(/usr/bin/cygpath -m "$opam_root_cacheable") + unix_opam_root=$(/usr/bin/cygpath -u "$opam_root") + unix_opam_root_cacheable=$(/usr/bin/cygpath -u "$opam_root_cacheable") + else + # shellcheck disable=SC2034 + unix_opam_root=$opam_root + # shellcheck disable=SC2034 + unix_opam_root_cacheable=$opam_root_cacheable + fi +} +end_of_script + +cat > .ci/sd4/run-checkout-code.sh <<'end_of_script' +#!/bin/sh + +# ================ +# checkout-code.sh +# ================ +# +# Checkouts all of the git source code. +# +# This should be done outside of +# dockcross (used by Linux) since a Docker-in-Docker container can have +# difficulties doing a git checkout (the Git credentials for any private +# repositories are likely not present). We don't care about any private +# repositories for DKML but any code that extends this (ex. DKSDK) may +# need to use private repositories. + +set -euf + +setup_WORKSPACE_VARNAME=$1 +shift +setup_WORKSPACE=$1 +shift + +# ------------------------ Functions ------------------------ + +# shellcheck source=./common-values.sh +. .ci/sd4/common-values.sh + +# Disable automatic garbage collection +git_disable_gc() { + git_disable_gc_NAME=$1 + shift + git -C ".ci/sd4/g/$git_disable_gc_NAME" config --local gc.auto 0 +} + +# Mimic the behavior of GitHub's actions/checkout@v3 +# - the plus symbol in 'git fetch ... origin +REF:refs/tags/v0.0' overrides any existing REF +git_checkout() { + git_checkout_NAME=$1 + shift + git_checkout_URL=$1 + shift + git_checkout_REF=$1 + shift + + if [ -e ".ci/sd4/g/$git_checkout_NAME" ]; then + git_disable_gc "$git_checkout_NAME" + git -C ".ci/sd4/g/$git_checkout_NAME" remote set-url origin "$git_checkout_URL" + git -C ".ci/sd4/g/$git_checkout_NAME" fetch --no-tags --progress --no-recurse-submodules --depth=1 origin "+${git_checkout_REF}:refs/tags/v0.0" + else + install -d ".ci/sd4/g/$git_checkout_NAME" + git -C ".ci/sd4/g/$git_checkout_NAME" -c init.defaultBranch=main init + git_disable_gc "$git_checkout_NAME" + git -C ".ci/sd4/g/$git_checkout_NAME" remote add origin "$git_checkout_URL" + git -C ".ci/sd4/g/$git_checkout_NAME" fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin "+${git_checkout_REF}:refs/tags/v0.0" + fi + git -C ".ci/sd4/g/$git_checkout_NAME" -c advice.detachedHead=false checkout --progress --force refs/tags/v0.0 + git -C ".ci/sd4/g/$git_checkout_NAME" log -1 --format='%H' +} + +# --------------------------------------------------------------------- + +section_begin checkout-info "Summary: code checkout" + +# shellcheck disable=SC2154 +echo " +================ +checkout-code.sh +================ +. +--------- +Arguments +--------- +WORKSPACE_VARNAME=$setup_WORKSPACE_VARNAME +WORKSPACE=$setup_WORKSPACE +. +------ +Inputs +------ +VERBOSE=${VERBOSE:-} +. +------ +Matrix +------ +dkml_host_abi=$dkml_host_abi +. +" + +section_end checkout-info + +install -d .ci/sd4/g + +# dkml-runtime-distribution + +case "$dkml_host_abi" in +windows_*) + section_begin checkout-dkml-runtime-distribution 'Checkout dkml-runtime-distribution' + git_checkout dkml-runtime-distribution https://github.com/diskuv/dkml-runtime-distribution.git "1a3ec82dd851751a95e6a4797387a8163c51520e" + section_end checkout-dkml-runtime-distribution + ;; +esac + +end_of_script + +cat > .ci/sd4/run-setup-dkml.sh <<'end_of_script' +#!/bin/sh +set -euf + +# Constants +SHA512_DEVNULL='cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e' +# Edited by https://gitlab.com/diskuv/diskuv-ocaml/contributors/release.sh +DEFAULT_DISKUV_OPAM_REPOSITORY_TAG=cc9518fa5630bfbe24d4c5e0e2cc29af513037ce +# Constants +# Should be edited by release.sh, but ... +# Can't be 1.0.0 or later until https://github.com/ocaml/opam-repository/pull/21704 ocaml-option-32bit +# can come back in. +DKML_VERSION=0.4.0 + +setup_WORKSPACE_VARNAME=$1 +shift +setup_WORKSPACE=$1 +shift + +# ------------------ Variables and functions ------------------------ + +# shellcheck source=./common-values.sh +. .ci/sd4/common-values.sh + +if [ "${VERBOSE:-}" = "true" ]; then + do_tar_rf() { + tar rvf "$@" + } +else + do_tar_rf() { + tar rf "$@" + } +fi + +# Make the standard input work as an OCaml string. +# This currently only escapes backslashes and double quotes. +escape_arg_as_ocaml_string() { + escape_arg_as_ocaml_string_ARG=$1 + shift + printf "%s" "$escape_arg_as_ocaml_string_ARG" | sed 's#\\#\\\\#g; s#"#\\"#g;' +} + +# Fixup opam_root on Windows to be mixed case. Set original_* and unix_* as well. +fixup_opam_root + +# Set TEMP variable for Windows +export_temp_for_windows + +# Load VS studio environment +if [ -e .ci/sd4/vsenv.sh ]; then + # shellcheck disable=SC1091 + . .ci/sd4/vsenv.sh +fi + +# ------------------------------------------------------------------- + +section_begin setup-info "Summary: setup-dkml" + +# shellcheck disable=SC2154 +echo " +============= +setup-dkml.sh +============= +. +--------- +Arguments +--------- +WORKSPACE_VARNAME=$setup_WORKSPACE_VARNAME +WORKSPACE=$setup_WORKSPACE +. +------ +Inputs +------ +FDOPEN_OPAMEXE_BOOTSTRAP=${FDOPEN_OPAMEXE_BOOTSTRAP:-} +DISKUV_OPAM_REPOSITORY=${DISKUV_OPAM_REPOSITORY:-} +DKML_COMPILER=${DKML_COMPILER:-} +OCAML_COMPILER=${OCAML_COMPILER:-} +CONF_DKML_CROSS_TOOLCHAIN=${CONF_DKML_CROSS_TOOLCHAIN:-} +SECONDARY_SWITCH=${SECONDARY_SWITCH:-} +MANYLINUX=${MANYLINUX:-} +VERBOSE=${VERBOSE:-} +. +------------------- +Generated Constants +------------------- +DKML_VERSION=$DKML_VERSION +DEFAULT_DISKUV_OPAM_REPOSITORY_TAG=$DEFAULT_DISKUV_OPAM_REPOSITORY_TAG +DEFAULT_DKML_COMPILER=$DEFAULT_DKML_COMPILER +. +------ +Matrix +------ +dkml_host_abi=$dkml_host_abi +bootstrap_opam_version=$bootstrap_opam_version +abi_pattern=$abi_pattern +opam_root=${opam_root} +opam_root_cacheable=${opam_root_cacheable} +original_opam_root=${original_opam_root} +original_opam_root_cacheable=${original_opam_root_cacheable} +unix_opam_root=${unix_opam_root} +unix_opam_root_cacheable=${unix_opam_root_cacheable} +dockcross_image=${dockcross_image:-} +dockcross_image_custom_prefix=${dockcross_image_custom_prefix:-} +dockcross_run_extra_args=${dockcross_run_extra_args:-} +docker_runner=${docker_runner:-} +in_docker=${in_docker:-} +ocaml_options=${ocaml_options:-} +. +---- +Pins +---- +PIN_BASE=${PIN_BASE} +PIN_BIGSTRINGAF=${PIN_BIGSTRINGAF} +PIN_CORE_KERNEL=${PIN_CORE_KERNEL} +PIN_CTYPES=${PIN_CTYPES} +PIN_CTYPES_FOREIGN=${PIN_CTYPES_FOREIGN} +PIN_CURLY=${PIN_CURLY} +PIN_DIGESTIF=${PIN_DIGESTIF} +PIN_DKML_APPS=${PIN_DKML_APPS} +PIN_DUNE=${PIN_DUNE} +PIN_DUNE_CONFIGURATOR=${PIN_DUNE_CONFIGURATOR} +PIN_OCAMLBUILD=${PIN_OCAMLBUILD} +PIN_OCAMLFIND=${PIN_OCAMLFIND} +PIN_OCP_INDENT=${PIN_OCP_INDENT} +PIN_PPX_EXPECT=${PIN_PPX_EXPECT} +PIN_PTIME=${PIN_PTIME} +PIN_TIME_NOW=${PIN_TIME_NOW} +PIN_WITH_DKML=${PIN_WITH_DKML} +. +" +case "$dkml_host_abi" in +windows_*) + # shellcheck disable=SC2153 + echo " +------------- +Visual Studio +------------- +VS_DIR=$VS_DIR +VS_VCVARSVER=$VS_VCVARSVER +VS_WINSDKVER=$VS_WINSDKVER +VS_MSVSPREFERENCE=$VS_MSVSPREFERENCE +VS_CMAKEGENERATOR=$VS_CMAKEGENERATOR +. +" + ;; +esac +section_end setup-info + +do_bootstrap() { + # Bootstrap from historical release + runit_BOOTSTRAPPED=0 + + # Bootstrap opam from fdopen (Windows) + if [ "$runit_BOOTSTRAPPED" = 0 ] && [ "${FDOPEN_OPAMEXE_BOOTSTRAP:-}" = "true" ]; then + if [ -e .ci/sd4/opam64/bin/opam.exe ] && [ -e .ci/sd4/opam64/bin/opam-installer.exe ]; then + runit_BOOTSTRAPPED=1 + else + case "$dkml_host_abi" in + windows_*) + echo 'Bootstrap opam from fdopen (Windows) ...' + install -d .ci/sd4/bs/bin + wget -O "$setup_WORKSPACE"/.ci/sd4/opam64.tar.xz https://github.com/fdopen/opam-repository-mingw/releases/download/0.0.0.2/opam64.tar.xz + + # this stalls: tar xvCfJ "$setup_WORKSPACE"/.ci/sd4 "$setup_WORKSPACE"/.ci/sd4/opam64.tar.xz + xz -v -d "$setup_WORKSPACE"/.ci/sd4/opam64.tar.xz + tar xvCf .ci/sd4 .ci/sd4/opam64.tar + + rm -rf "$setup_WORKSPACE"/.ci/sd4/bs/bin/Opam.Runtime.amd64 + mv -v "$setup_WORKSPACE"/.ci/sd4/opam64/bin/Opam.Runtime.amd64/ "$setup_WORKSPACE"/.ci/sd4/bs/bin/ + mv -v "$setup_WORKSPACE"/.ci/sd4/opam64/bin/opam.exe "$setup_WORKSPACE"/.ci/sd4/bs/bin/ + mv -v "$setup_WORKSPACE"/.ci/sd4/opam64/bin/opam-installer.exe "$setup_WORKSPACE"/.ci/sd4/bs/bin/ + + # diagnostics + ldd "$setup_WORKSPACE"/.ci/sd4/bs/bin/opam.exe + ldd "$setup_WORKSPACE"/.ci/sd4/bs/bin/opam-installer.exe + + runit_BOOTSTRAPPED=1 + ;; + esac + fi + fi + + # Bootstrap from historical release + if [ "$runit_BOOTSTRAPPED" = 0 ] && [ "$bootstrap_opam_version" != "os" ]; then + install -d .ci/sd4/bs + cd .ci/sd4/bs + + if [ ! -e version ] || [ "$(cat version)" != "$bootstrap_opam_version" ]; then + echo 'Bootstrap opam from historical release (non-Windows; Windows non-fdopen) ...' + if command -v curl; then + curl -L -o opam.tar.gz "https://github.com/diskuv/dkml-component-opam/releases/download/v${bootstrap_opam_version}/dkml-component-staging-opam.tar.gz" + else + wget -O opam.tar.gz "https://github.com/diskuv/dkml-component-opam/releases/download/v${bootstrap_opam_version}/dkml-component-staging-opam.tar.gz" + fi + tar tvfz opam.tar.gz + tar xfz opam.tar.gz "./staging-files/${dkml_host_abi}/" + rm -rf bin/ + mv "staging-files/${dkml_host_abi}/bin" . + rm -rf "${abi_pattern}" + printf "%s" "${bootstrap_opam_version}" >version + fi + + rm -f opam.tar.gz + cd ../../.. + + runit_BOOTSTRAPPED=1 + fi + + # Bootstrap from package manager or GitHub ocaml/opam release + case "$runit_BOOTSTRAPPED,$bootstrap_opam_version,$dkml_host_abi" in + 0,os,darwin_*) + if ! command -v opam; then + echo 'Bootstrap opam from package manager (macOS) ...' + brew install gpatch + brew install opam + fi + runit_BOOTSTRAPPED=1 + ;; + 0,os,linux_x86) + if [ ! -x .ci/sd4/bs/bin/opam ]; then + echo 'Bootstrap opam from GitHub ocaml/opam release (Linux x86) ...' + install -d .ci/sd4/bs/bin + wget -O .ci/sd4/bs/bin/opam.tmp https://github.com/ocaml/opam/releases/download/2.1.2/opam-2.1.2-i686-linux + sha512_check=$(openssl sha512 2>&1 /dev/null | cut -f 2 -d ' ') + check="85a480d60e09a7d37fa0d0434ed97a3187434772ceb4e7e8faa5b06bc18423d004af3ad5849c7d35e72dca155103257fd6b1178872df8291583929eb8f884b6a" + test "$sha512" = "$check" + chmod +x .ci/sd4/bs/bin/opam.tmp + mv .ci/sd4/bs/bin/opam.tmp .ci/sd4/bs/bin/opam + else + echo "openssl 512 option unsupported." + exit 61 + fi + fi + runit_BOOTSTRAPPED=1 + ;; + 0,os,linux_x86_64) + if [ ! -x .ci/sd4/bs/bin/opam ]; then + echo 'Bootstrap opam from GitHub ocaml/opam release (Linux x86_64) ...' + install -d .ci/sd4/bs/bin + wget -O .ci/sd4/bs/bin/opam.tmp https://github.com/ocaml/opam/releases/download/2.1.2/opam-2.1.2-x86_64-linux + sha512_check=$(openssl sha512 2>&1 /dev/null | cut -f 2 -d ' ') + check="c0657ecbd4dc212587a4da70c5ff0402df95d148867be0e1eb1be8863a2851015f191437c3c99b7c2b153fcaa56cac99169c76ec94c5787750d7a59cd1fbb68b" + test "$sha512" = "$check" + chmod +x .ci/sd4/bs/bin/opam.tmp + mv .ci/sd4/bs/bin/opam.tmp .ci/sd4/bs/bin/opam + else + echo "openssl 512 option unsupported." + exit 61 + fi + fi + runit_BOOTSTRAPPED=1 + ;; + esac +} +section_begin bootstrap-opam 'Bootstrap opam' +do_bootstrap +section_end bootstrap-opam + +# Start environment distribution tarball +# We use .tar rather than .tar.gz/.tar.bz2 because we can repeatedly add to an uncompressed .tar. But we need to +# start with an empty tarball since some tar programs will only add ('tar rf xyz.tar') to an existing .tar. +install -d .ci/sd4/dist +tar cf .ci/sd4/dist/opam-with-env.tar -T /dev/null + +do_get_dockcross() { + if [ -n "${dockcross_image:-}" ]; then + # The dockcross script is super-slow + section_begin get-dockcross 'Get dockcross binary (ManyLinux)' + install -d .ci/sd4 + # shellcheck disable=SC2086 + docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_custom_prefix:-}${dockcross_image:-}" >.ci/sd4/dockcross.gen + + # PROBLEM 1 + # --------- + # Super-annoying stderr output from dockcross at line: + # tty -s && [ -z "$MSYS" ] && TTY_ARGS=-ti + # When there is no tty, get: + # tty: ignoring all arguments + # not a tty + # So replace 'tty -s &&' with 'false &&' + sed 's/tty -s &&/false \&\&/' .ci/sd4/dockcross.gen >.ci/sd4/dockcross-real + rm -f .ci/sd4/dockcross.gen + chmod +x .ci/sd4/dockcross-real + + # PROBLEM 2 + # --------- + # By default dockcross for ManyLinux will chown -R all python packages; super-slow (~10 seconds)! + # Confer: https://github.com/dockcross/dockcross/blob/master/manylinux-common/pre_exec.sh + # That kills speed for any repetitive dockcross invocation. + # + # BUT it is unnecessary to chown -R when the current user is root, because inside the Docker container + # the files are already root! + # + # The chown -R (within pre_exec.sh) is not run when the user ids are not passed in. + # Confer: https://github.com/dockcross/dockcross/blob/96d87416f639af0204bdd42553e4b99315ca8476/imagefiles/entrypoint.sh#L21-L53 + # + # So explicitly call the entrypoint if root! + case "$dkml_host_abi" in + # https://github.com/dockcross/dockcross/blob/master/linux-x86/linux32-entrypoint.sh + linux_x86) dockcross_entrypoint=/dockcross/linux32-entrypoint.sh ;; + *) dockcross_entrypoint=/dockcross/entrypoint.sh ;; + esac + cat > .ci/sd4/dockcross <&2 "Missing functionality (\${f}) (in cygwin)." ; exit 1 ; } ; + done ; + HOST_PWD="\$( cygpath -w "\$( readlink -f "\$( pwd ;)" ; )" ; )" ; + else + HOST_PWD=\$PWD + [ -L \$HOST_PWD ] && HOST_PWD=\$(readlink \$HOST_PWD) + fi + + # ---------- End of dockcross script snippet ------- + + # Handle: dockcross --args "-v X:Y --platform P" + ARGS= + if [ "\$#" -ge 1 ] && [ "\$1" = "--args" ]; then + shift + ARGS=\$1 + shift + fi + + # Directly invoke entrypoint + exec docker run --entrypoint /bin/bash \ + --rm \ + \${ARGS:-} \ + -v "\$HOST_PWD":/work \ + ${dockcross_image_custom_prefix:-}${dockcross_image:-} ${dockcross_entrypoint} "\$@" +else + HERE=\$(dirname "\$0") + HERE=\$(cd "\$HERE" && pwd) + exec "\$HERE/dockcross-real" "\$@" +fi +EOF + chmod +x .ci/sd4/dockcross + + # Bundle for consumers of setup-dkml.yml + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/dockcross .ci/sd4/dockcross-real + + section_end get-dockcross + fi +} +do_get_dockcross + +if [ -n "${dockcross_image:-}" ]; then + # rsync needs to be available, even after Docker container disappears + if [ ! -e .ci/sd4/bs/bin/rsync ]; then + section_begin get-opam-prereqs-in-dockcross 'Get Opam prerequisites (ManyLinux)' + install -d .ci/sd4/bs/bin + # shellcheck disable=SC2016 + .ci/sd4/dockcross --args "${dockcross_run_extra_args:-}" sh -c 'sudo yum install -y rsync && install $(command -v rsync) .ci/sd4/bs/bin' + section_end get-opam-prereqs-in-dockcross + fi +fi + +# Opam prerequisites for using opam (not for installing opam) + +{ + if [ -n "${docker_runner:-}" ]; then + # rsync needs to be available, even after Docker container disappears + if [ ! -e .ci/sd4/bs/bin/rsync.deps ]; then + section_begin get-opam-prereqs-in-docker 'Get Opam prerequisites (Linux Docker)' + install -d .ci/sd4/bs/bin + ${docker_runner} sh -c ' + apt-get update && + apt-get install -y rsync && + ldd /usr/bin/rsync && + ls -l /lib/i386-linux-gnu/libpopt.so.0 /lib/i386-linux-gnu/libacl.so.1 /lib/i386-linux-gnu/libattr.so.1 && + tar cCfhz / /work/.ci/sd4/bs/bin/deps.tar.gz /usr/bin/rsync /lib/i386-linux-gnu/libpopt.so.0 + ' + touch .ci/sd4/bs/bin/rsync.deps + section_end get-opam-prereqs-in-docker + fi + fi + + # Bundle Opam prerequisites (ManyLinux or Linux Docker) + if [ -n "${docker_runner:-}" ] || [ -n "${dockcross_image:-}" ]; then + # Bundle for consumers of setup-dkml.yml + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/bs/bin/rsync + fi +} + +# Get Opam Cache +do_get_opam_cache() { + if [ "$unix_opam_root_cacheable" = "$unix_opam_root" ]; then return; fi + if [ ! -e "$unix_opam_root_cacheable" ]; then return; fi + section_begin get-opam-cache "Transferring Opam cache to $original_opam_root_cacheable" + echo Starting transfer # need some output or GitLab CI will not display the section duration + transfer_dir "$unix_opam_root_cacheable" "$unix_opam_root" + echo Finished transfer + section_end get-opam-cache +} +do_get_opam_cache + +# Setup Opam + +do_write_opam_scripts() { + case "${FDOPEN_OPAMEXE_BOOTSTRAP:-},$dkml_host_abi" in + true,windows_*) + # With fdopen's opam.exe, 'os-distribution = "cygwinports"'. But native Windows opam.exe has 'os-distribution = "win32"'. + # But on Windows we always want MSYS2 or native Windows libraries, not Cygwin. If cygwinports then + # code like https://github.com/ocaml/opam-repository/blob/08cbb8258bd4bf30cd6f307c958911a29d537b54/packages/conf-pkg-config/conf-pkg-config.2/opam#L36 + # will fail. So always set 'os-distribution = "win32"' on Windows. + PATCH_OS_DISTRIBUTION_WIN32=true + # With fdopen's opam.exe, no 'exe = ".exe"' is set because Cygwin does not need file extensions. + # Native Windows requires a .exe extension. + PATCH_EXE_WIN32=true + ;; + *) + PATCH_OS_DISTRIBUTION_WIN32=false + PATCH_EXE_WIN32=false + ;; + esac + + # --------------------- + # Empty opam repository + # --------------------- + + install -d .ci/sd4/eor + cat >.ci/sd4/eor/repo <.ci/sd4/troubleshoot-opam.sh <&2 +find "\$OPAMROOT"/log -mindepth 1 -maxdepth 1 -name "*.out" ! -name "log-*.out" ! -name "ocaml-variants-*.out" | while read -r dump_on_error_LOG; do + dump_on_error_BLOG=\$(basename "\$dump_on_error_LOG") + printf "\n\n========= [TROUBLESHOOTING] %s ===========\n\n" "\$dump_on_error_BLOG" >&2 + awk -v BLOG="\$dump_on_error_BLOG" '{print "[" BLOG "]", \$0}' "\$dump_on_error_LOG" >&2 +done +printf "\nScroll up to see the [TROUBLESHOOTING] logs that begin at the [START OF TROUBLESHOOTING] line\n" >&2 +EOF + + chmod +x .ci/sd4/troubleshoot-opam.sh + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/troubleshoot-opam.sh + + # --------------- + # Create Opam support scripts (not needed for all platforms) + # The PATH to find opam must work internally in setup-dkml.yml (sd4/bs/bin) and + # by consumers of setup-dkml.yml (sd4/opamexe) + # --------------- + + USER_ID=$(id -u) + GROUP_ID=$(id -g) + USER_NAME=$(id -un) + GROUP_NAME=$(id -gn) + + case "${opam_root}" in + /* | ?:*) # /a/b/c or C:\Windows + validate_supports_docker() { + echo "Docker only supported with relative paths for the opam root, not: ${opam_root}" >&2 + exit 3 + } + ;; + *) # relative path + validate_supports_docker() { + true + } + cat >.ci/sd4/opam-in-docker <&2 +set +e +opam "\$@" +exitcode=\$? +if [ \$troubleshooting = 1 ]; then + [ \$exitcode = 0 ] || "/work/.ci/sd4/troubleshoot-opam.sh" \$OPAMROOT +fi +exit \$exitcode +EOF + chmod +x .ci/sd4/opam-in-docker + ;; + esac + + cat >.ci/sd4/deescalate <.ci/sd4/opam-with-env <&2 + cat .ci/sd4/opam-in-docker >&2 + echo '________________________' >&2 + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/opam-with-env .ci/sd4/opam-in-docker .ci/sd4/edr + + elif [ -n "${docker_runner:-}" ]; then + + cat >.ci/sd4/opam-with-env <&2 + cat .ci/sd4/opam-in-docker >&2 + echo '________________________' >&2 + echo '__ deescalate __' >&2 + cat .ci/sd4/deescalate >&2 + echo '________________' >&2 + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/opam-with-env .ci/sd4/opam-in-docker .ci/sd4/deescalate + + else + + cat >.ci/sd4/opam-with-env <&2 +set +e +opam "\$@" +exitcode=\$? +if [ \$troubleshooting = 1 ]; then + [ \$exitcode = 0 ] || "\${PROJECT_DIR}/.ci/sd4/troubleshoot-opam.sh" \$OPAMROOT +fi +exit \$exitcode +EOF + chmod +x .ci/sd4/opam-with-env + + # Bundle for consumers of setup-dkml.yml + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/opam-with-env + + fi + echo '__ opam-with-env __' >&2 + cat .ci/sd4/opam-with-env >&2 + echo '___________________' >&2 + + # ------- + # opamrun + # ------- + + install -d .ci/sd4/opamrun + cat >.ci/sd4/opamrun/opamrun <>"$GITHUB_PATH" + # Special case: GITHUB_PATH does not influence msys2.CMD of msys2/setup-msys2@v2, so place in real MSYS2 PATH + if [ -n "${MSYSTEM:-}" ]; then + install -d /usr/local/bin + install .ci/sd4/opamrun/opamrun /usr/local/bin/opamrun + fi +fi + +# Place opamrun in the immediate PATH +PATH="$setup_WORKSPACE/.ci/sd4/opamrun:$PATH" + +# Complicated Opam sequence is because: +# 1. Opam's default curl does not work on Windows, +# and `opam init` does not provide a way to change it (TODO: need +# a PR!). +# 2. We have to separate the Opam download cache from the other Opam +# caches +if [ ! -s "$opam_root/.ci.root-init" ]; then # non-empty init file so can be cached irrespective of existence + section_begin opam-init 'Initialize opam root' + + # Clear any partial previous attempt + rm -rf "$opam_root" + + case "$dkml_host_abi,${in_docker:-}" in + windows_*,*) + eor=$(cygpath -am "$setup_WORKSPACE"/.ci/sd4/eor) + opamrun init --disable-sandboxing --no-setup --kind local --bare "$eor" + case "$(opamrun --version)" in + 2.0.*) echo 'download-command: wget' >>"$opam_root/config" ;; + *) opamrun option --yes --global download-command=wget ;; + esac + ;; + *,true) + opamrun init --disable-sandboxing --no-setup --kind local --bare "/work/.ci/sd4/eor" + ;; + *) + opamrun init --disable-sandboxing --no-setup --kind local --bare "$setup_WORKSPACE/.ci/sd4/eor" + ;; + esac + echo yes > "$opam_root/.ci.root-init" + + section_end opam-init +fi + +section_begin opam-vars "Summary: opam global variables" +opamrun --no-troubleshooting var --global || true +section_end opam-vars + +# Build OCaml + +do_switch_create() { + do_switch_create_NAME=$1 + shift + + section_begin "switch-create-$do_switch_create_NAME" "Create opam switch '$do_switch_create_NAME'" + # Create, or recreate, the Opam switch. The Opam switch should not be + # cached except for the compiler (confer docs for setup-ocaml GitHub + # Action) which is the 'dkml' switch (or the 'two' switch). + # Check if the switch name is present in the Opam root (which may come from cache) + NOMINALLY_PRESENT=false + if opamrun switch list --short | grep "^${do_switch_create_NAME}\$"; then NOMINALLY_PRESENT=true; fi + + # Check if the switch is actually present in case of cache incoherence + # or corrupt Opam state that could result in: + # Error: No config file found for switch dkml. Switch broken? + if [ $NOMINALLY_PRESENT = true ] && [ ! -e "$opam_root/$do_switch_create_NAME/.opam-switch/switch-config" ]; then + # Remove the switch name from Opam root, and any partial switch state. + # Ignore inevitable warnings/failure about missing switch. + opamrun --no-troubleshooting switch remove "$do_switch_create_NAME" --yes || true + rm -rf "${opam_root:?}/$do_switch_create_NAME" + NOMINALLY_PRESENT=false + fi + + if [ $NOMINALLY_PRESENT = false ]; then + opamrun switch create "$do_switch_create_NAME" --empty --yes + fi + section_end "switch-create-$do_switch_create_NAME" +} +do_switch_create dkml +if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_switch_create two +else + section_begin "switch-create-two" "Create empty opam switch 'two'" + # Always create a secondary switch ... just empty. Avoid problems with cache content missing + # and idempotency. + opamrun --no-troubleshooting switch remove two --yes || true + rm -rf "$opam_root/two" + opamrun switch create two --empty --yes + section_end "switch-create-two" +fi + +do_switch_active() { + section_begin "switch-active" "Set dkml as active switch" + opamrun switch set dkml --yes + section_end "switch-active" +} +do_switch_active + +do_opam_repositories_add() { + section_begin "opam-repo-add" "Add 'diskuv' opam repository" + if ! opamrun --no-troubleshooting repository list -s | grep '^diskuv'; then + opamrun repository add diskuv "git+https://github.com/diskuv/diskuv-opam-repository.git#${DISKUV_OPAM_REPOSITORY:-$DEFAULT_DISKUV_OPAM_REPOSITORY_TAG}" --yes --dont-select + fi + section_end "opam-repo-add" +} +do_opam_repositories_add + +do_opam_repositories_config() { + do_opam_repositories_config_NAME=$1 + shift + + section_begin "opam-repo-$do_opam_repositories_config_NAME" "Attach repositories to $do_opam_repositories_config_NAME" + + if [ ! -s "$opam_root/.ci.$do_opam_repositories_config_NAME.repo-init" ]; then # non-empty init file so can be cached irrespective of existence + opamrun --no-troubleshooting repository remove default --switch "$do_opam_repositories_config_NAME" --yes || true + opamrun --no-troubleshooting repository remove diskuv --switch "$do_opam_repositories_config_NAME" --yes || true + opamrun repository add default --switch "$do_opam_repositories_config_NAME" --yes + opamrun repository add diskuv --switch "$do_opam_repositories_config_NAME" --yes + echo yes > "$opam_root/.ci.$do_opam_repositories_config_NAME.repo-init" + fi + + section_end "opam-repo-$do_opam_repositories_config_NAME" +} +do_opam_repositories_config dkml +if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_opam_repositories_config two +fi + +do_opam_repositories_update() { + section_begin "opam-repo-update" "Update opam repositories" + # The default repository may be the initial 'eor' (empty) repository + opamrun repository set-url default https://opam.ocaml.org --yes + # Always set the `diskuv` repository url since it can change + opamrun repository set-url diskuv "git+https://github.com/diskuv/diskuv-opam-repository.git#${DISKUV_OPAM_REPOSITORY:-$DEFAULT_DISKUV_OPAM_REPOSITORY_TAG}" --yes --dont-select + # Update both `default` and `diskuv` Opam repositories + opamrun update default diskuv + section_end "opam-repo-update" +} +do_opam_repositories_update + +do_pins() { + do_pins_NAME=$1 + shift + + # dkml-base-compiler + + if [ "${DKML_COMPILER:-}" != '@repository@' ] && [ -z "${DKML_COMPILER:-}" ] && [ -z "${OCAML_COMPILER:-}" ]; then + section_begin checkout-dkml-base-compiler "Pin dkml-base-compiler to default ${DEFAULT_DKML_COMPILER} (neither dkml-base-compiler nor OCAML_COMPILER specified) for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action dkml-base-compiler "https://github.com/diskuv/dkml-compiler.git#${DEFAULT_DKML_COMPILER}" + section_end checkout-dkml-base-compiler + elif [ "${DKML_COMPILER:-}" != '@repository@' ] && [ -n "${DKML_COMPILER:-}" ] && [ -z "${OCAML_COMPILER:-}" ]; then + section_begin checkout-dkml-base-compiler "Pin dkml-base-compiler to $DKML_COMPILER (dkml-base-compiler specified; no OCAML_COMPILER specified) for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action dkml-base-compiler "https://github.com/diskuv/dkml-compiler.git#${DKML_COMPILER}" + section_end checkout-dkml-base-compiler + elif [ -n "${OCAML_COMPILER:-}" ]; then + # Validate OCAML_COMPILER (OCAML_COMPILER specified) + case "${OCAML_COMPILER:-}" in + 4.12.1) true ;; + *) + echo "OCAML_COMPILER version ${OCAML_COMPILER:-} is not supported" >&2 + exit 109 + ;; + esac + + section_begin checkout-dkml-base-compiler "Pin dkml-base-compiler (OCAML_COMPILER specified) for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action dkml-base-compiler "https://github.com/diskuv/dkml-compiler.git#${OCAML_COMPILER}-v${DKML_VERSION}" + section_end checkout-dkml-base-compiler + fi + + # conf-dkml-cross-toolchain + + if [ "${CONF_DKML_CROSS_TOOLCHAIN:-}" != '@repository@' ]; then + section_begin checkout-conf-dkml-cross-toolchain "Pin conf-dkml-cross-toolchain for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action conf-dkml-cross-toolchain "https://github.com/diskuv/conf-dkml-cross-toolchain.git#$CONF_DKML_CROSS_TOOLCHAIN" + section_end checkout-conf-dkml-cross-toolchain + fi + + # patches necessary for Windows in diskuv-opam-repository + # + # - ocamlfind and ocamlbuild + # + # - dune-configurator (and hence Dune) + # Dune 2.9.1 and 3.0.2 will fail to build jst-config.v0.14.1 because for jst-config/discover/discover.ml Dune does: + # cl -nologo -O2 -Gy- -MD -I Z:/.opam_root-cached-8/installer-ocaml/lib/ocaml -o C:\Users\beckf\AppData\Local\Temp\build_f18aec_dune\ocaml-configurator4d3858\c-test-31\test.obj -c C:\Users\beckf\AppData\Local\Temp\build_f18aec_dune\ocaml-configurator4d3858\c-test-31\test.c advapi32.lib ws2_32.lib version.lib + # instead of + # cl -nologo -O2 -Gy- -MD -I Z:/.opam_root-cached-8/installer-ocaml/lib/ocaml /FoC:\Users\beckf\AppData\Local\Temp\build_f18aec_dune\ocaml-configurator4d3858\c-test-31\test.obj -c C:\Users\beckf\AppData\Local\Temp\build_f18aec_dune\ocaml-configurator4d3858\c-test-31\test.c advapi32.lib ws2_32.lib version.lib + # with the (irrelevant) test.c file: + # #include + # #include + # + # #ifdef ARCH_BIG_ENDIAN + # const char *s0 = "BEGIN-0-true-END"; + # #else + # const char *s0 = "BEGIN-0-false-END"; + # #endif + # + # #ifdef ARCH_SIXTYFOUR + # const char *s1 = "BEGIN-1-true-END"; + # #else + # const char *s1 = "BEGIN-1-false-END"; + # #endif + # + # The actual problem is dune-configurator ... we only have patches in Diskuv + # repository up until 2.9.3. Need to upstream fix the problem. + # + # - ppx_expect; only patch is for v0.14.1. Need to upstream fix the problem. + # - base; patches for v0.14.1/2/3. Need to upstream fix the problem. + section_begin "opam-pins-$do_pins_NAME" "Opam pins for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version base "${PIN_BASE}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version bigstringaf "${PIN_BIGSTRINGAF}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version core_kernel "${PIN_CORE_KERNEL}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ctypes "${PIN_CTYPES}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ctypes-foreign "${PIN_CTYPES_FOREIGN}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version curly "${PIN_CURLY}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version digestif "${PIN_DIGESTIF}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version dkml-apps "${PIN_DKML_APPS}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version dune "${PIN_DUNE}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version dune-configurator "${PIN_DUNE_CONFIGURATOR}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ocamlbuild "${PIN_OCAMLBUILD}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ocamlfind "${PIN_OCAMLFIND}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ocp-indent "${PIN_OCP_INDENT}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ppx_expect "${PIN_PPX_EXPECT}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ptime "${PIN_PTIME}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version time_now "${PIN_TIME_NOW}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version with-dkml "${PIN_WITH_DKML}" + section_end "opam-pins-$do_pins_NAME" +} + +do_pins dkml +if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_pins two +fi + +do_use_vsstudio() { + do_use_vsstudio_NAME=$1 + shift + case "$dkml_host_abi" in + windows_*) + section_begin "use-vsstudio-$do_use_vsstudio_NAME" "Use Visual Studio in dkml-* Opam packages (Windows) for $do_use_vsstudio_NAME switch" + + # shellcheck disable=SC2153 + E_VS_DIR=$(escape_arg_as_ocaml_string "$VS_DIR") + # shellcheck disable=SC2153 + E_VS_VCVARSVER=$(escape_arg_as_ocaml_string "$VS_VCVARSVER") + # shellcheck disable=SC2153 + E_VS_WINSDKVER=$(escape_arg_as_ocaml_string "$VS_WINSDKVER") + # shellcheck disable=SC2153 + E_VS_MSVSPREFERENCE=$(escape_arg_as_ocaml_string "$VS_MSVSPREFERENCE") + # shellcheck disable=SC2153 + E_VS_CMAKEGENERATOR=$(escape_arg_as_ocaml_string "$VS_CMAKEGENERATOR") + + case "$(opamrun --version)" in + 2.0.*) + if [ "${in_docker}" = "true" ]; then + echo Opam 2.0 support in dockcross to use a portable opam var prefix not yet implemented + exit 67 + fi + OP=$(opamrun var prefix --switch "$do_use_vsstudio_NAME") + OPSC=$OP/.opam-switch/switch-config + if grep setenv: "$OPSC"; then + echo "INFO: Updating switch-config. Old was:" + awk '{print ">> " $0}' "$OPSC" + + awk '$1=="setenv:"{x=1} x==0{print} x==1 && $0=="]"{x=0}' "$OPSC" >"$OPSC".trimmed + mv "$OPSC".trimmed "$OPSC" + fi + echo 'setenv: [' >>"$OPSC" + echo ' [DKML_COMPILE_SPEC = "1"]' >>"$OPSC" + echo ' [DKML_COMPILE_TYPE = "VS"]' >>"$OPSC" + echo " [DKML_COMPILE_VS_DIR = \"$E_VS_DIR\"]" >>"$OPSC" + echo " [DKML_COMPILE_VS_VCVARSVER = \"$E_VS_VCVARSVER\"]" >>"$OPSC" + echo " [DKML_COMPILE_VS_WINSDKVER = \"$E_VS_WINSDKVER\"]" >>"$OPSC" + echo " [DKML_COMPILE_VS_MSVSPREFERENCE = \"$E_VS_MSVSPREFERENCE\"]" >>"$OPSC" + echo " [DKML_COMPILE_VS_CMAKEGENERATOR = \"$E_VS_CMAKEGENERATOR\"]" >>"$OPSC" + echo " [DKML_HOST_ABI = \"${dkml_host_abi}\"]" >>"$OPSC" + echo ']' >>"$OPSC" + cat "$OPSC" >&2 # print + ;; + *) + opamrun option --switch "$do_use_vsstudio_NAME" setenv= # reset + opamrun option --switch "$do_use_vsstudio_NAME" setenv+='DKML_COMPILE_SPEC = "1"' + opamrun option --switch "$do_use_vsstudio_NAME" setenv+='DKML_COMPILE_TYPE = "VS"' + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_DIR = \"$E_VS_DIR\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_VCVARSVER = \"$E_VS_VCVARSVER\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_WINSDKVER = \"$E_VS_WINSDKVER\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_MSVSPREFERENCE = \"$E_VS_MSVSPREFERENCE\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_CMAKEGENERATOR = \"$E_VS_CMAKEGENERATOR\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_HOST_ABI = \"${dkml_host_abi}\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv # print + ;; + esac + + # shellcheck disable=SC2016 + opamrun exec --switch "$do_use_vsstudio_NAME" -- sh -c 'echo $VCToolsRedistDir' + + section_end "use-vsstudio-$do_use_vsstudio_NAME" + ;; + esac +} +if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_use_vsstudio two +fi +do_use_vsstudio dkml + +# Because dune.X.Y.Z+shim (and any user DKML packages) requires DKML installed (after all, it is just +# a with-dkml.exe shim), we need either dkmlvars-v2.sexp or DKML environment +# variables. Confer: Dkml_runtimelib.Dkml_context.get_dkmlversion +# +# grep matches either: +# [... [DiskuvOCamlVersion = "1.0.1"] ...] +# DiskuvOCamlVersion = "1.0.1" +do_setenv() { + do_setenv_SWITCH=$1 + shift + opamrun option --switch "$do_setenv_SWITCH" setenv > ".ci/sd4/setenv.$do_setenv_SWITCH.txt" + if ! grep -q '\(^|\[\)DiskuvOCamlVarsVersion ' ".ci/sd4/setenv.$do_setenv_SWITCH.txt"; then + opamrun option --switch "$do_setenv_SWITCH" setenv+='DiskuvOCamlVarsVersion = "2"' + fi + if ! grep -q '\(^|\[\)DiskuvOCamlVersion ' ".ci/sd4/setenv.$do_setenv_SWITCH.txt"; then + opamrun option --switch "$do_setenv_SWITCH" setenv+="DiskuvOCamlVersion = \"$DKML_VERSION\"" + fi + case "${dkml_host_abi}" in + windows_*) + if ! grep -q '\(^|\[\)DiskuvOCamlMSYS2Dir ' ".ci/sd4/setenv.$do_setenv_SWITCH.txt"; then + if [ -x /usr/bin/cygpath ]; then + MSYS2_DIR_NATIVE=$(/usr/bin/cygpath -aw /) + else + # If we are already inside MSYS2 then MSYSTEM_PREFIX should be set. But cygpath should be there as well!! + echo "FATAL: Could not locate MSYS2: there was no cygpath" >&2 + exit 3 + fi + MSYS2_DIR_NATIVE_ESCAPED=$(printf "%s" "$MSYS2_DIR_NATIVE" | sed 's/\\/\\\\/g') + opamrun option --switch "$do_setenv_SWITCH" setenv+="DiskuvOCamlMSYS2Dir = \"$MSYS2_DIR_NATIVE_ESCAPED\"" + fi + esac +} +if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_setenv two +fi +do_setenv dkml + +do_install_compiler() { + do_install_compiler_NAME=$1 + shift + section_begin "install-compiler-$do_install_compiler_NAME" "Install OCaml compiler for $do_install_compiler_NAME switch" + opamrun pin list --switch "$do_install_compiler_NAME" + # shellcheck disable=SC2086 + opamrun upgrade --switch "$do_install_compiler_NAME" --yes dkml-base-compiler conf-dkml-cross-toolchain ${ocaml_options:-} + section_end "install-compiler-$do_install_compiler_NAME" +} +do_install_compiler dkml +if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_install_compiler two +fi + +do_summary() { + do_summary_NAME=$1 + shift + section_begin "summary-$do_summary_NAME" "Summary: $do_summary_NAME switch" + opamrun var --switch "$do_summary_NAME" + opamrun exec --switch "$do_summary_NAME" -- ocamlc -config + section_end "summary-$do_summary_NAME" +} +do_summary dkml +if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_summary two +fi + +end_of_script + +sh .ci/sd4/run-checkout-code.sh PC_PROJECT_DIR "${PC_PROJECT_DIR}" +sh .ci/sd4/run-setup-dkml.sh PC_PROJECT_DIR "${PC_PROJECT_DIR}" + +# shellcheck disable=SC2154 +echo " +Finished setup. + +To continue your testing, run: + export dkml_host_abi='${dkml_host_abi}' + export abi_pattern='${abi_pattern}' + export opam_root='${opam_root}' + export exe_ext='${exe_ext:-}' + +Now you can use 'opamrun' to do opam commands like: + + PATH=\"$PWD/.ci/sd4/opamrun:\$PATH\" opamrun install XYZ.opam + PATH=\"$PWD/.ci/sd4/opamrun:\$PATH\" opamrun -it exec -- bash + sh ci/build-test.sh +" \ No newline at end of file diff --git a/ci/setup-dkml/pc/setup-dkml-linux_x86_64.sh b/ci/setup-dkml/pc/setup-dkml-linux_x86_64.sh new file mode 100644 index 0000000..53262a1 --- /dev/null +++ b/ci/setup-dkml/pc/setup-dkml-linux_x86_64.sh @@ -0,0 +1,1512 @@ +#!/bin/sh +set -euf + +# Reset environment so no conflicts with a parent Opam or OCaml system +unset OPAMROOT +unset OPAM_SWITCH_PREFIX +unset OPAMSWITCH +unset CAML_LD_LIBRARY_PATH +unset OCAMLLIB +unset OCAML_TOPLEVEL_PATH + +export PC_PROJECT_DIR="$PWD" +export FDOPEN_OPAMEXE_BOOTSTRAP=false +export CACHE_PREFIX=v1 +export OCAML_COMPILER= +export DKML_COMPILER= +export CONF_DKML_CROSS_TOOLCHAIN=@repository@ +export DISKUV_OPAM_REPOSITORY= +export SECONDARY_SWITCH=false +# autogen from global_env_vars. +export DEFAULT_DKML_COMPILER='4.12.1-v1.0.2' +export PIN_BASE='v0.14.3' +export PIN_BIGSTRINGAF='0.8.0' +export PIN_CORE_KERNEL='v0.14.2' +export PIN_CTYPES_FOREIGN='0.19.2-windowssupport-r4' +export PIN_CTYPES='0.19.2-windowssupport-r4' +export PIN_CURLY='0.2.1-windows-env_r2' +export PIN_DIGESTIF='1.0.1' +export PIN_DUNE='2.9.3+shim.1.0.2~r0' +export PIN_DUNE_CONFIGURATOR='2.9.3' +export PIN_DKML_APPS='1.0.1' +export PIN_OCAMLBUILD='0.14.0' +export PIN_OCAMLFIND='1.9.1' +export PIN_OCP_INDENT='1.8.2-windowssupport' +export PIN_PPX_EXPECT='v0.14.1' +export PIN_PTIME='0.8.6-msvcsupport' +export PIN_TIME_NOW='v0.14.0' +export PIN_WITH_DKML='1.0.1' + +usage() { + echo 'Setup Diskuv OCaml (DKML) compiler on a desktop PC.' >&2 + echo 'usage: setup-dkml-linux_x86_64.sh [options]' >&2 + echo 'Options:' >&2 + + # Context variables + echo " --PC_PROJECT_DIR=. Defaults to the current directory (${PC_PROJECT_DIR})" >&2 + + # Input variables + echo " --FDOPEN_OPAMEXE_BOOTSTRAP=true|false. Defaults to: ${FDOPEN_OPAMEXE_BOOTSTRAP}" >&2 + echo " --CACHE_PREFIX=. Defaults to: ${CACHE_PREFIX}" >&2 + echo " --OCAML_COMPILER=. --DKML_COMPILER takes priority. If --DKML_COMPILER is not set and --OCAML_COMPILER is set, then the specified OCaml version tag of dkml-compiler (ex. 4.12.1) is used. Defaults to: ${OCAML_COMPILER}" >&2 + echo " --DKML_COMPILER=. Unspecified or blank is the latest from the default branch (main) of dkml-compiler. Defaults to: ${DKML_COMPILER}" >&2 + echo " --SECONDARY_SWITCH=true|false. If true then the secondary switch named 'two' is created, in addition to the always-present 'dkml' switch. Defaults to: ${SECONDARY_SWITCH}" >&2 + echo " --CONF_DKML_CROSS_TOOLCHAIN=. Unspecified or blank is the latest from the default branch (main) of conf-dkml-cross-toolchain. @repository@ is the latest from Opam. Defaults to: ${CONF_DKML_CROSS_TOOLCHAIN}" >&2 + echo " --DISKUV_OPAM_REPOSITORY=. Defaults to the value of --DEFAULT_DISKUV_OPAM_REPOSITORY_TAG (see below)" >&2 + + # autogen from global_env_vars. + echo " --DEFAULT_DKML_COMPILER=. Defaults to: ${DEFAULT_DKML_COMPILER}" >&2 + echo " --PIN_BASE=. Defaults to: ${PIN_BASE}" >&2 + echo " --PIN_BIGSTRINGAF=. Defaults to: ${PIN_BIGSTRINGAF}" >&2 + echo " --PIN_CORE_KERNEL=. Defaults to: ${PIN_CORE_KERNEL}" >&2 + echo " --PIN_CTYPES_FOREIGN=. Defaults to: ${PIN_CTYPES_FOREIGN}" >&2 + echo " --PIN_CTYPES=. Defaults to: ${PIN_CTYPES}" >&2 + echo " --PIN_CURLY=. Defaults to: ${PIN_CURLY}" >&2 + echo " --PIN_DIGESTIF=. Defaults to: ${PIN_DIGESTIF}" >&2 + echo " --PIN_DUNE=. Defaults to: ${PIN_DUNE}" >&2 + echo " --PIN_DUNE_CONFIGURATOR=. Defaults to: ${PIN_DUNE_CONFIGURATOR}" >&2 + echo " --PIN_DKML_APPS=. Defaults to: ${PIN_DKML_APPS}" >&2 + echo " --PIN_OCAMLBUILD=. Defaults to: ${PIN_OCAMLBUILD}" >&2 + echo " --PIN_OCAMLFIND=. Defaults to: ${PIN_OCAMLFIND}" >&2 + echo " --PIN_OCP_INDENT=. Defaults to: ${PIN_OCP_INDENT}" >&2 + echo " --PIN_PPX_EXPECT=. Defaults to: ${PIN_PPX_EXPECT}" >&2 + echo " --PIN_PTIME=. Defaults to: ${PIN_PTIME}" >&2 + echo " --PIN_TIME_NOW=. Defaults to: ${PIN_TIME_NOW}" >&2 + echo " --PIN_WITH_DKML=. Defaults to: ${PIN_WITH_DKML}" >&2 + exit 2 +} +fail() { + echo "Error: $*" >&2 + exit 3 +} +unset file + +OPTIND=1 +while getopts :h-: option; do + case $option in + h) usage ;; + -) case $OPTARG in + PC_PROJECT_DIR) fail "Option \"$OPTARG\" missing argument" ;; + PC_PROJECT_DIR=*) PC_PROJECT_DIR=${OPTARG#*=} ;; + CACHE_PREFIX) fail "Option \"$OPTARG\" missing argument" ;; + CACHE_PREFIX=*) CACHE_PREFIX=${OPTARG#*=} ;; + FDOPEN_OPAMEXE_BOOTSTRAP) fail "Option \"$OPTARG\" missing argument" ;; + FDOPEN_OPAMEXE_BOOTSTRAP=*) FDOPEN_OPAMEXE_BOOTSTRAP=${OPTARG#*=} ;; + OCAML_COMPILER) fail "Option \"$OPTARG\" missing argument" ;; + OCAML_COMPILER=*) OCAML_COMPILER=${OPTARG#*=} ;; + DKML_COMPILER) fail "Option \"$OPTARG\" missing argument" ;; + DKML_COMPILER=*) DKML_COMPILER=${OPTARG#*=} ;; + SECONDARY_SWITCH) fail "Option \"$OPTARG\" missing argument" ;; + SECONDARY_SWITCH=*) SECONDARY_SWITCH=${OPTARG#*=} ;; + CONF_DKML_CROSS_TOOLCHAIN) fail "Option \"$OPTARG\" missing argument" ;; + CONF_DKML_CROSS_TOOLCHAIN=*) CONF_DKML_CROSS_TOOLCHAIN=${OPTARG#*=} ;; + DISKUV_OPAM_REPOSITORY) fail "Option \"$OPTARG\" missing argument" ;; + DISKUV_OPAM_REPOSITORY=*) DISKUV_OPAM_REPOSITORY=${OPTARG#*=} ;; + # autogen from global_env_vars. + DEFAULT_DKML_COMPILER) fail "Option \"$OPTARG\" missing argument" ;; + DEFAULT_DKML_COMPILER=*) DEFAULT_DKML_COMPILER=${OPTARG#*=} ;; + PIN_BASE) fail "Option \"$OPTARG\" missing argument" ;; + PIN_BASE=*) PIN_BASE=${OPTARG#*=} ;; + PIN_BIGSTRINGAF) fail "Option \"$OPTARG\" missing argument" ;; + PIN_BIGSTRINGAF=*) PIN_BIGSTRINGAF=${OPTARG#*=} ;; + PIN_CORE_KERNEL) fail "Option \"$OPTARG\" missing argument" ;; + PIN_CORE_KERNEL=*) PIN_CORE_KERNEL=${OPTARG#*=} ;; + PIN_CTYPES_FOREIGN) fail "Option \"$OPTARG\" missing argument" ;; + PIN_CTYPES_FOREIGN=*) PIN_CTYPES_FOREIGN=${OPTARG#*=} ;; + PIN_CTYPES) fail "Option \"$OPTARG\" missing argument" ;; + PIN_CTYPES=*) PIN_CTYPES=${OPTARG#*=} ;; + PIN_CURLY) fail "Option \"$OPTARG\" missing argument" ;; + PIN_CURLY=*) PIN_CURLY=${OPTARG#*=} ;; + PIN_DIGESTIF) fail "Option \"$OPTARG\" missing argument" ;; + PIN_DIGESTIF=*) PIN_DIGESTIF=${OPTARG#*=} ;; + PIN_DUNE) fail "Option \"$OPTARG\" missing argument" ;; + PIN_DUNE=*) PIN_DUNE=${OPTARG#*=} ;; + PIN_DUNE_CONFIGURATOR) fail "Option \"$OPTARG\" missing argument" ;; + PIN_DUNE_CONFIGURATOR=*) PIN_DUNE_CONFIGURATOR=${OPTARG#*=} ;; + PIN_DKML_APPS) fail "Option \"$OPTARG\" missing argument" ;; + PIN_DKML_APPS=*) PIN_DKML_APPS=${OPTARG#*=} ;; + PIN_OCAMLBUILD) fail "Option \"$OPTARG\" missing argument" ;; + PIN_OCAMLBUILD=*) PIN_OCAMLBUILD=${OPTARG#*=} ;; + PIN_OCAMLFIND) fail "Option \"$OPTARG\" missing argument" ;; + PIN_OCAMLFIND=*) PIN_OCAMLFIND=${OPTARG#*=} ;; + PIN_OCP_INDENT) fail "Option \"$OPTARG\" missing argument" ;; + PIN_OCP_INDENT=*) PIN_OCP_INDENT=${OPTARG#*=} ;; + PIN_PPX_EXPECT) fail "Option \"$OPTARG\" missing argument" ;; + PIN_PPX_EXPECT=*) PIN_PPX_EXPECT=${OPTARG#*=} ;; + PIN_PTIME) fail "Option \"$OPTARG\" missing argument" ;; + PIN_PTIME=*) PIN_PTIME=${OPTARG#*=} ;; + PIN_TIME_NOW) fail "Option \"$OPTARG\" missing argument" ;; + PIN_TIME_NOW=*) PIN_TIME_NOW=${OPTARG#*=} ;; + PIN_WITH_DKML) fail "Option \"$OPTARG\" missing argument" ;; + PIN_WITH_DKML=*) PIN_WITH_DKML=${OPTARG#*=} ;; + help) usage ;; + help=*) fail "Option \"${OPTARG%%=*}\" has unexpected argument" ;; + *) fail "Unknown long option \"${OPTARG%%=*}\"" ;; + esac ;; + '?') fail "Unknown short option \"$OPTARG\"" ;; + :) fail "Short option \"$OPTARG\" missing argument" ;; + *) fail "Bad state in getopts (OPTARG=\"$OPTARG\")" ;; + esac +done +shift $((OPTIND - 1)) + +# Set matrix variables +# autogen from pc_vars. only linux_x86_64 +export dkml_host_os="linux" +export opam_root_cacheable=".ci/o" +export abi_pattern="manylinux2014-linux_x86_64" +export comment="(CentOS 7, etc.)" +export bootstrap_opam_version="2.2.0-dkml20220801T155940Z" +export dkml_host_abi="linux_x86_64" +export opam_root=".ci/o" +export dockcross_image="dockcross/manylinux2014-x64" +export dockcross_run_extra_args="--platform linux/amd64" +export in_docker="true" + + +########################### before_script ############################### + +echo "Writing scripts ..." +install -d .ci/sd4 + +cat > .ci/sd4/common-values.sh <<'end_of_script' +#!/bin/sh + +# ------------------------ Log Formatting ------------------------ + +TXT_SECTION="\e[94m" # bright blue +TXT_CLEAR="\e[0m" + +if [ "${GITLAB_CI:-}" = "true" ]; then + # https://docs.gitlab.com/ee/ci/jobs/#expand-and-collapse-job-log-sections + print_section_start() { + print_section_start_NAME=$1 + shift + printf "\e[0Ksection_start:%s:%s[collapsed=true]\r\e[0K" \ + "$(date +%s)" \ + "$print_section_start_NAME" + } + print_section_end() { + print_section_end_NAME=$1 + shift + printf "\e[0Ksection_end:%s:%s\r\e[0K\n" \ + "$(date +%s)" \ + "$print_section_end_NAME" + } +elif [ -n "${GITHUB_ENV:-}" ]; then + # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines + print_section_start() { + print_section_start_NAME=$1 + shift + printf "::group::" + } + print_section_end() { + print_section_end_NAME=$1 + shift + printf "::endgroup::\n" + } +else + print_section_start() { + print_section_start_NAME=$1 + shift + } + print_section_end() { + print_section_end_NAME=$1 + shift + } +fi + +section_begin() { + # https://docs.gitlab.com/ee/ci/yaml/script.html#add-color-codes-to-script-output + section_NAME=$1 + shift + section_HEADER=$1 + shift + print_section_start "$section_NAME" + printf "${TXT_SECTION}%s${TXT_CLEAR}\n" "$section_HEADER" +} + +section_end() { + section_NAME=$1 + shift + print_section_end "$section_NAME" +} + +# ------------------- Other Functions ----------------- + +transfer_dir() { + transfer_dir_SRC=$1 + shift + transfer_dir_DST=$1 + shift + # Remove the destination directory completely, but make sure the parent of the + # destination directory exists so `mv` will work + install -d "$transfer_dir_DST" + rm -rf "$transfer_dir_DST" + # Move + mv "$transfer_dir_SRC" "$transfer_dir_DST" +} + +# Set TEMP variable which is used, among other things, for OCaml's +# [Filename.temp_dir_name] on Win32, and by with-dkml.exe on Windows +export_temp_for_windows() { + if [ -x /usr/bin/cygpath ]; then + if [ -n "${RUNNER_TEMP:-}" ]; then + # GitHub Actions + TEMP=$(cygpath -am "$RUNNER_TEMP") + else + # GitLab CI/CD or desktop + install -d .ci/tmp + TEMP=$(cygpath -am ".ci/tmp") + fi + export TEMP + fi +} + +# Fixup opam_root on Windows to be mixed case. +# On input the following variables must be present: +# - opam_root +# - opam_root_cacheable +# On output the input variables will be modified _and_ the +# following variables will be available: +# - original_opam_root +# - original_opam_root_cacheable +# - unix_opam_root +# - unix_opam_root_cacheable +fixup_opam_root() { + # shellcheck disable=SC2034 + original_opam_root=$opam_root + # shellcheck disable=SC2034 + original_opam_root_cacheable=$opam_root_cacheable + if [ -x /usr/bin/cygpath ]; then + opam_root=$(/usr/bin/cygpath -m "$opam_root") + opam_root_cacheable=$(/usr/bin/cygpath -m "$opam_root_cacheable") + unix_opam_root=$(/usr/bin/cygpath -u "$opam_root") + unix_opam_root_cacheable=$(/usr/bin/cygpath -u "$opam_root_cacheable") + else + # shellcheck disable=SC2034 + unix_opam_root=$opam_root + # shellcheck disable=SC2034 + unix_opam_root_cacheable=$opam_root_cacheable + fi +} +end_of_script + +cat > .ci/sd4/run-checkout-code.sh <<'end_of_script' +#!/bin/sh + +# ================ +# checkout-code.sh +# ================ +# +# Checkouts all of the git source code. +# +# This should be done outside of +# dockcross (used by Linux) since a Docker-in-Docker container can have +# difficulties doing a git checkout (the Git credentials for any private +# repositories are likely not present). We don't care about any private +# repositories for DKML but any code that extends this (ex. DKSDK) may +# need to use private repositories. + +set -euf + +setup_WORKSPACE_VARNAME=$1 +shift +setup_WORKSPACE=$1 +shift + +# ------------------------ Functions ------------------------ + +# shellcheck source=./common-values.sh +. .ci/sd4/common-values.sh + +# Disable automatic garbage collection +git_disable_gc() { + git_disable_gc_NAME=$1 + shift + git -C ".ci/sd4/g/$git_disable_gc_NAME" config --local gc.auto 0 +} + +# Mimic the behavior of GitHub's actions/checkout@v3 +# - the plus symbol in 'git fetch ... origin +REF:refs/tags/v0.0' overrides any existing REF +git_checkout() { + git_checkout_NAME=$1 + shift + git_checkout_URL=$1 + shift + git_checkout_REF=$1 + shift + + if [ -e ".ci/sd4/g/$git_checkout_NAME" ]; then + git_disable_gc "$git_checkout_NAME" + git -C ".ci/sd4/g/$git_checkout_NAME" remote set-url origin "$git_checkout_URL" + git -C ".ci/sd4/g/$git_checkout_NAME" fetch --no-tags --progress --no-recurse-submodules --depth=1 origin "+${git_checkout_REF}:refs/tags/v0.0" + else + install -d ".ci/sd4/g/$git_checkout_NAME" + git -C ".ci/sd4/g/$git_checkout_NAME" -c init.defaultBranch=main init + git_disable_gc "$git_checkout_NAME" + git -C ".ci/sd4/g/$git_checkout_NAME" remote add origin "$git_checkout_URL" + git -C ".ci/sd4/g/$git_checkout_NAME" fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 origin "+${git_checkout_REF}:refs/tags/v0.0" + fi + git -C ".ci/sd4/g/$git_checkout_NAME" -c advice.detachedHead=false checkout --progress --force refs/tags/v0.0 + git -C ".ci/sd4/g/$git_checkout_NAME" log -1 --format='%H' +} + +# --------------------------------------------------------------------- + +section_begin checkout-info "Summary: code checkout" + +# shellcheck disable=SC2154 +echo " +================ +checkout-code.sh +================ +. +--------- +Arguments +--------- +WORKSPACE_VARNAME=$setup_WORKSPACE_VARNAME +WORKSPACE=$setup_WORKSPACE +. +------ +Inputs +------ +VERBOSE=${VERBOSE:-} +. +------ +Matrix +------ +dkml_host_abi=$dkml_host_abi +. +" + +section_end checkout-info + +install -d .ci/sd4/g + +# dkml-runtime-distribution + +case "$dkml_host_abi" in +windows_*) + section_begin checkout-dkml-runtime-distribution 'Checkout dkml-runtime-distribution' + git_checkout dkml-runtime-distribution https://github.com/diskuv/dkml-runtime-distribution.git "1a3ec82dd851751a95e6a4797387a8163c51520e" + section_end checkout-dkml-runtime-distribution + ;; +esac + +end_of_script + +cat > .ci/sd4/run-setup-dkml.sh <<'end_of_script' +#!/bin/sh +set -euf + +# Constants +SHA512_DEVNULL='cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e' +# Edited by https://gitlab.com/diskuv/diskuv-ocaml/contributors/release.sh +DEFAULT_DISKUV_OPAM_REPOSITORY_TAG=cc9518fa5630bfbe24d4c5e0e2cc29af513037ce +# Constants +# Should be edited by release.sh, but ... +# Can't be 1.0.0 or later until https://github.com/ocaml/opam-repository/pull/21704 ocaml-option-32bit +# can come back in. +DKML_VERSION=0.4.0 + +setup_WORKSPACE_VARNAME=$1 +shift +setup_WORKSPACE=$1 +shift + +# ------------------ Variables and functions ------------------------ + +# shellcheck source=./common-values.sh +. .ci/sd4/common-values.sh + +if [ "${VERBOSE:-}" = "true" ]; then + do_tar_rf() { + tar rvf "$@" + } +else + do_tar_rf() { + tar rf "$@" + } +fi + +# Make the standard input work as an OCaml string. +# This currently only escapes backslashes and double quotes. +escape_arg_as_ocaml_string() { + escape_arg_as_ocaml_string_ARG=$1 + shift + printf "%s" "$escape_arg_as_ocaml_string_ARG" | sed 's#\\#\\\\#g; s#"#\\"#g;' +} + +# Fixup opam_root on Windows to be mixed case. Set original_* and unix_* as well. +fixup_opam_root + +# Set TEMP variable for Windows +export_temp_for_windows + +# Load VS studio environment +if [ -e .ci/sd4/vsenv.sh ]; then + # shellcheck disable=SC1091 + . .ci/sd4/vsenv.sh +fi + +# ------------------------------------------------------------------- + +section_begin setup-info "Summary: setup-dkml" + +# shellcheck disable=SC2154 +echo " +============= +setup-dkml.sh +============= +. +--------- +Arguments +--------- +WORKSPACE_VARNAME=$setup_WORKSPACE_VARNAME +WORKSPACE=$setup_WORKSPACE +. +------ +Inputs +------ +FDOPEN_OPAMEXE_BOOTSTRAP=${FDOPEN_OPAMEXE_BOOTSTRAP:-} +DISKUV_OPAM_REPOSITORY=${DISKUV_OPAM_REPOSITORY:-} +DKML_COMPILER=${DKML_COMPILER:-} +OCAML_COMPILER=${OCAML_COMPILER:-} +CONF_DKML_CROSS_TOOLCHAIN=${CONF_DKML_CROSS_TOOLCHAIN:-} +SECONDARY_SWITCH=${SECONDARY_SWITCH:-} +MANYLINUX=${MANYLINUX:-} +VERBOSE=${VERBOSE:-} +. +------------------- +Generated Constants +------------------- +DKML_VERSION=$DKML_VERSION +DEFAULT_DISKUV_OPAM_REPOSITORY_TAG=$DEFAULT_DISKUV_OPAM_REPOSITORY_TAG +DEFAULT_DKML_COMPILER=$DEFAULT_DKML_COMPILER +. +------ +Matrix +------ +dkml_host_abi=$dkml_host_abi +bootstrap_opam_version=$bootstrap_opam_version +abi_pattern=$abi_pattern +opam_root=${opam_root} +opam_root_cacheable=${opam_root_cacheable} +original_opam_root=${original_opam_root} +original_opam_root_cacheable=${original_opam_root_cacheable} +unix_opam_root=${unix_opam_root} +unix_opam_root_cacheable=${unix_opam_root_cacheable} +dockcross_image=${dockcross_image:-} +dockcross_image_custom_prefix=${dockcross_image_custom_prefix:-} +dockcross_run_extra_args=${dockcross_run_extra_args:-} +docker_runner=${docker_runner:-} +in_docker=${in_docker:-} +ocaml_options=${ocaml_options:-} +. +---- +Pins +---- +PIN_BASE=${PIN_BASE} +PIN_BIGSTRINGAF=${PIN_BIGSTRINGAF} +PIN_CORE_KERNEL=${PIN_CORE_KERNEL} +PIN_CTYPES=${PIN_CTYPES} +PIN_CTYPES_FOREIGN=${PIN_CTYPES_FOREIGN} +PIN_CURLY=${PIN_CURLY} +PIN_DIGESTIF=${PIN_DIGESTIF} +PIN_DKML_APPS=${PIN_DKML_APPS} +PIN_DUNE=${PIN_DUNE} +PIN_DUNE_CONFIGURATOR=${PIN_DUNE_CONFIGURATOR} +PIN_OCAMLBUILD=${PIN_OCAMLBUILD} +PIN_OCAMLFIND=${PIN_OCAMLFIND} +PIN_OCP_INDENT=${PIN_OCP_INDENT} +PIN_PPX_EXPECT=${PIN_PPX_EXPECT} +PIN_PTIME=${PIN_PTIME} +PIN_TIME_NOW=${PIN_TIME_NOW} +PIN_WITH_DKML=${PIN_WITH_DKML} +. +" +case "$dkml_host_abi" in +windows_*) + # shellcheck disable=SC2153 + echo " +------------- +Visual Studio +------------- +VS_DIR=$VS_DIR +VS_VCVARSVER=$VS_VCVARSVER +VS_WINSDKVER=$VS_WINSDKVER +VS_MSVSPREFERENCE=$VS_MSVSPREFERENCE +VS_CMAKEGENERATOR=$VS_CMAKEGENERATOR +. +" + ;; +esac +section_end setup-info + +do_bootstrap() { + # Bootstrap from historical release + runit_BOOTSTRAPPED=0 + + # Bootstrap opam from fdopen (Windows) + if [ "$runit_BOOTSTRAPPED" = 0 ] && [ "${FDOPEN_OPAMEXE_BOOTSTRAP:-}" = "true" ]; then + if [ -e .ci/sd4/opam64/bin/opam.exe ] && [ -e .ci/sd4/opam64/bin/opam-installer.exe ]; then + runit_BOOTSTRAPPED=1 + else + case "$dkml_host_abi" in + windows_*) + echo 'Bootstrap opam from fdopen (Windows) ...' + install -d .ci/sd4/bs/bin + wget -O "$setup_WORKSPACE"/.ci/sd4/opam64.tar.xz https://github.com/fdopen/opam-repository-mingw/releases/download/0.0.0.2/opam64.tar.xz + + # this stalls: tar xvCfJ "$setup_WORKSPACE"/.ci/sd4 "$setup_WORKSPACE"/.ci/sd4/opam64.tar.xz + xz -v -d "$setup_WORKSPACE"/.ci/sd4/opam64.tar.xz + tar xvCf .ci/sd4 .ci/sd4/opam64.tar + + rm -rf "$setup_WORKSPACE"/.ci/sd4/bs/bin/Opam.Runtime.amd64 + mv -v "$setup_WORKSPACE"/.ci/sd4/opam64/bin/Opam.Runtime.amd64/ "$setup_WORKSPACE"/.ci/sd4/bs/bin/ + mv -v "$setup_WORKSPACE"/.ci/sd4/opam64/bin/opam.exe "$setup_WORKSPACE"/.ci/sd4/bs/bin/ + mv -v "$setup_WORKSPACE"/.ci/sd4/opam64/bin/opam-installer.exe "$setup_WORKSPACE"/.ci/sd4/bs/bin/ + + # diagnostics + ldd "$setup_WORKSPACE"/.ci/sd4/bs/bin/opam.exe + ldd "$setup_WORKSPACE"/.ci/sd4/bs/bin/opam-installer.exe + + runit_BOOTSTRAPPED=1 + ;; + esac + fi + fi + + # Bootstrap from historical release + if [ "$runit_BOOTSTRAPPED" = 0 ] && [ "$bootstrap_opam_version" != "os" ]; then + install -d .ci/sd4/bs + cd .ci/sd4/bs + + if [ ! -e version ] || [ "$(cat version)" != "$bootstrap_opam_version" ]; then + echo 'Bootstrap opam from historical release (non-Windows; Windows non-fdopen) ...' + if command -v curl; then + curl -L -o opam.tar.gz "https://github.com/diskuv/dkml-component-opam/releases/download/v${bootstrap_opam_version}/dkml-component-staging-opam.tar.gz" + else + wget -O opam.tar.gz "https://github.com/diskuv/dkml-component-opam/releases/download/v${bootstrap_opam_version}/dkml-component-staging-opam.tar.gz" + fi + tar tvfz opam.tar.gz + tar xfz opam.tar.gz "./staging-files/${dkml_host_abi}/" + rm -rf bin/ + mv "staging-files/${dkml_host_abi}/bin" . + rm -rf "${abi_pattern}" + printf "%s" "${bootstrap_opam_version}" >version + fi + + rm -f opam.tar.gz + cd ../../.. + + runit_BOOTSTRAPPED=1 + fi + + # Bootstrap from package manager or GitHub ocaml/opam release + case "$runit_BOOTSTRAPPED,$bootstrap_opam_version,$dkml_host_abi" in + 0,os,darwin_*) + if ! command -v opam; then + echo 'Bootstrap opam from package manager (macOS) ...' + brew install gpatch + brew install opam + fi + runit_BOOTSTRAPPED=1 + ;; + 0,os,linux_x86) + if [ ! -x .ci/sd4/bs/bin/opam ]; then + echo 'Bootstrap opam from GitHub ocaml/opam release (Linux x86) ...' + install -d .ci/sd4/bs/bin + wget -O .ci/sd4/bs/bin/opam.tmp https://github.com/ocaml/opam/releases/download/2.1.2/opam-2.1.2-i686-linux + sha512_check=$(openssl sha512 2>&1 /dev/null | cut -f 2 -d ' ') + check="85a480d60e09a7d37fa0d0434ed97a3187434772ceb4e7e8faa5b06bc18423d004af3ad5849c7d35e72dca155103257fd6b1178872df8291583929eb8f884b6a" + test "$sha512" = "$check" + chmod +x .ci/sd4/bs/bin/opam.tmp + mv .ci/sd4/bs/bin/opam.tmp .ci/sd4/bs/bin/opam + else + echo "openssl 512 option unsupported." + exit 61 + fi + fi + runit_BOOTSTRAPPED=1 + ;; + 0,os,linux_x86_64) + if [ ! -x .ci/sd4/bs/bin/opam ]; then + echo 'Bootstrap opam from GitHub ocaml/opam release (Linux x86_64) ...' + install -d .ci/sd4/bs/bin + wget -O .ci/sd4/bs/bin/opam.tmp https://github.com/ocaml/opam/releases/download/2.1.2/opam-2.1.2-x86_64-linux + sha512_check=$(openssl sha512 2>&1 /dev/null | cut -f 2 -d ' ') + check="c0657ecbd4dc212587a4da70c5ff0402df95d148867be0e1eb1be8863a2851015f191437c3c99b7c2b153fcaa56cac99169c76ec94c5787750d7a59cd1fbb68b" + test "$sha512" = "$check" + chmod +x .ci/sd4/bs/bin/opam.tmp + mv .ci/sd4/bs/bin/opam.tmp .ci/sd4/bs/bin/opam + else + echo "openssl 512 option unsupported." + exit 61 + fi + fi + runit_BOOTSTRAPPED=1 + ;; + esac +} +section_begin bootstrap-opam 'Bootstrap opam' +do_bootstrap +section_end bootstrap-opam + +# Start environment distribution tarball +# We use .tar rather than .tar.gz/.tar.bz2 because we can repeatedly add to an uncompressed .tar. But we need to +# start with an empty tarball since some tar programs will only add ('tar rf xyz.tar') to an existing .tar. +install -d .ci/sd4/dist +tar cf .ci/sd4/dist/opam-with-env.tar -T /dev/null + +do_get_dockcross() { + if [ -n "${dockcross_image:-}" ]; then + # The dockcross script is super-slow + section_begin get-dockcross 'Get dockcross binary (ManyLinux)' + install -d .ci/sd4 + # shellcheck disable=SC2086 + docker run ${dockcross_run_extra_args:-} --rm "${dockcross_image_custom_prefix:-}${dockcross_image:-}" >.ci/sd4/dockcross.gen + + # PROBLEM 1 + # --------- + # Super-annoying stderr output from dockcross at line: + # tty -s && [ -z "$MSYS" ] && TTY_ARGS=-ti + # When there is no tty, get: + # tty: ignoring all arguments + # not a tty + # So replace 'tty -s &&' with 'false &&' + sed 's/tty -s &&/false \&\&/' .ci/sd4/dockcross.gen >.ci/sd4/dockcross-real + rm -f .ci/sd4/dockcross.gen + chmod +x .ci/sd4/dockcross-real + + # PROBLEM 2 + # --------- + # By default dockcross for ManyLinux will chown -R all python packages; super-slow (~10 seconds)! + # Confer: https://github.com/dockcross/dockcross/blob/master/manylinux-common/pre_exec.sh + # That kills speed for any repetitive dockcross invocation. + # + # BUT it is unnecessary to chown -R when the current user is root, because inside the Docker container + # the files are already root! + # + # The chown -R (within pre_exec.sh) is not run when the user ids are not passed in. + # Confer: https://github.com/dockcross/dockcross/blob/96d87416f639af0204bdd42553e4b99315ca8476/imagefiles/entrypoint.sh#L21-L53 + # + # So explicitly call the entrypoint if root! + case "$dkml_host_abi" in + # https://github.com/dockcross/dockcross/blob/master/linux-x86/linux32-entrypoint.sh + linux_x86) dockcross_entrypoint=/dockcross/linux32-entrypoint.sh ;; + *) dockcross_entrypoint=/dockcross/entrypoint.sh ;; + esac + cat > .ci/sd4/dockcross <&2 "Missing functionality (\${f}) (in cygwin)." ; exit 1 ; } ; + done ; + HOST_PWD="\$( cygpath -w "\$( readlink -f "\$( pwd ;)" ; )" ; )" ; + else + HOST_PWD=\$PWD + [ -L \$HOST_PWD ] && HOST_PWD=\$(readlink \$HOST_PWD) + fi + + # ---------- End of dockcross script snippet ------- + + # Handle: dockcross --args "-v X:Y --platform P" + ARGS= + if [ "\$#" -ge 1 ] && [ "\$1" = "--args" ]; then + shift + ARGS=\$1 + shift + fi + + # Directly invoke entrypoint + exec docker run --entrypoint /bin/bash \ + --rm \ + \${ARGS:-} \ + -v "\$HOST_PWD":/work \ + ${dockcross_image_custom_prefix:-}${dockcross_image:-} ${dockcross_entrypoint} "\$@" +else + HERE=\$(dirname "\$0") + HERE=\$(cd "\$HERE" && pwd) + exec "\$HERE/dockcross-real" "\$@" +fi +EOF + chmod +x .ci/sd4/dockcross + + # Bundle for consumers of setup-dkml.yml + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/dockcross .ci/sd4/dockcross-real + + section_end get-dockcross + fi +} +do_get_dockcross + +if [ -n "${dockcross_image:-}" ]; then + # rsync needs to be available, even after Docker container disappears + if [ ! -e .ci/sd4/bs/bin/rsync ]; then + section_begin get-opam-prereqs-in-dockcross 'Get Opam prerequisites (ManyLinux)' + install -d .ci/sd4/bs/bin + # shellcheck disable=SC2016 + .ci/sd4/dockcross --args "${dockcross_run_extra_args:-}" sh -c 'sudo yum install -y rsync && install $(command -v rsync) .ci/sd4/bs/bin' + section_end get-opam-prereqs-in-dockcross + fi +fi + +# Opam prerequisites for using opam (not for installing opam) + +{ + if [ -n "${docker_runner:-}" ]; then + # rsync needs to be available, even after Docker container disappears + if [ ! -e .ci/sd4/bs/bin/rsync.deps ]; then + section_begin get-opam-prereqs-in-docker 'Get Opam prerequisites (Linux Docker)' + install -d .ci/sd4/bs/bin + ${docker_runner} sh -c ' + apt-get update && + apt-get install -y rsync && + ldd /usr/bin/rsync && + ls -l /lib/i386-linux-gnu/libpopt.so.0 /lib/i386-linux-gnu/libacl.so.1 /lib/i386-linux-gnu/libattr.so.1 && + tar cCfhz / /work/.ci/sd4/bs/bin/deps.tar.gz /usr/bin/rsync /lib/i386-linux-gnu/libpopt.so.0 + ' + touch .ci/sd4/bs/bin/rsync.deps + section_end get-opam-prereqs-in-docker + fi + fi + + # Bundle Opam prerequisites (ManyLinux or Linux Docker) + if [ -n "${docker_runner:-}" ] || [ -n "${dockcross_image:-}" ]; then + # Bundle for consumers of setup-dkml.yml + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/bs/bin/rsync + fi +} + +# Get Opam Cache +do_get_opam_cache() { + if [ "$unix_opam_root_cacheable" = "$unix_opam_root" ]; then return; fi + if [ ! -e "$unix_opam_root_cacheable" ]; then return; fi + section_begin get-opam-cache "Transferring Opam cache to $original_opam_root_cacheable" + echo Starting transfer # need some output or GitLab CI will not display the section duration + transfer_dir "$unix_opam_root_cacheable" "$unix_opam_root" + echo Finished transfer + section_end get-opam-cache +} +do_get_opam_cache + +# Setup Opam + +do_write_opam_scripts() { + case "${FDOPEN_OPAMEXE_BOOTSTRAP:-},$dkml_host_abi" in + true,windows_*) + # With fdopen's opam.exe, 'os-distribution = "cygwinports"'. But native Windows opam.exe has 'os-distribution = "win32"'. + # But on Windows we always want MSYS2 or native Windows libraries, not Cygwin. If cygwinports then + # code like https://github.com/ocaml/opam-repository/blob/08cbb8258bd4bf30cd6f307c958911a29d537b54/packages/conf-pkg-config/conf-pkg-config.2/opam#L36 + # will fail. So always set 'os-distribution = "win32"' on Windows. + PATCH_OS_DISTRIBUTION_WIN32=true + # With fdopen's opam.exe, no 'exe = ".exe"' is set because Cygwin does not need file extensions. + # Native Windows requires a .exe extension. + PATCH_EXE_WIN32=true + ;; + *) + PATCH_OS_DISTRIBUTION_WIN32=false + PATCH_EXE_WIN32=false + ;; + esac + + # --------------------- + # Empty opam repository + # --------------------- + + install -d .ci/sd4/eor + cat >.ci/sd4/eor/repo <.ci/sd4/troubleshoot-opam.sh <&2 +find "\$OPAMROOT"/log -mindepth 1 -maxdepth 1 -name "*.out" ! -name "log-*.out" ! -name "ocaml-variants-*.out" | while read -r dump_on_error_LOG; do + dump_on_error_BLOG=\$(basename "\$dump_on_error_LOG") + printf "\n\n========= [TROUBLESHOOTING] %s ===========\n\n" "\$dump_on_error_BLOG" >&2 + awk -v BLOG="\$dump_on_error_BLOG" '{print "[" BLOG "]", \$0}' "\$dump_on_error_LOG" >&2 +done +printf "\nScroll up to see the [TROUBLESHOOTING] logs that begin at the [START OF TROUBLESHOOTING] line\n" >&2 +EOF + + chmod +x .ci/sd4/troubleshoot-opam.sh + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/troubleshoot-opam.sh + + # --------------- + # Create Opam support scripts (not needed for all platforms) + # The PATH to find opam must work internally in setup-dkml.yml (sd4/bs/bin) and + # by consumers of setup-dkml.yml (sd4/opamexe) + # --------------- + + USER_ID=$(id -u) + GROUP_ID=$(id -g) + USER_NAME=$(id -un) + GROUP_NAME=$(id -gn) + + case "${opam_root}" in + /* | ?:*) # /a/b/c or C:\Windows + validate_supports_docker() { + echo "Docker only supported with relative paths for the opam root, not: ${opam_root}" >&2 + exit 3 + } + ;; + *) # relative path + validate_supports_docker() { + true + } + cat >.ci/sd4/opam-in-docker <&2 +set +e +opam "\$@" +exitcode=\$? +if [ \$troubleshooting = 1 ]; then + [ \$exitcode = 0 ] || "/work/.ci/sd4/troubleshoot-opam.sh" \$OPAMROOT +fi +exit \$exitcode +EOF + chmod +x .ci/sd4/opam-in-docker + ;; + esac + + cat >.ci/sd4/deescalate <.ci/sd4/opam-with-env <&2 + cat .ci/sd4/opam-in-docker >&2 + echo '________________________' >&2 + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/opam-with-env .ci/sd4/opam-in-docker .ci/sd4/edr + + elif [ -n "${docker_runner:-}" ]; then + + cat >.ci/sd4/opam-with-env <&2 + cat .ci/sd4/opam-in-docker >&2 + echo '________________________' >&2 + echo '__ deescalate __' >&2 + cat .ci/sd4/deescalate >&2 + echo '________________' >&2 + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/opam-with-env .ci/sd4/opam-in-docker .ci/sd4/deescalate + + else + + cat >.ci/sd4/opam-with-env <&2 +set +e +opam "\$@" +exitcode=\$? +if [ \$troubleshooting = 1 ]; then + [ \$exitcode = 0 ] || "\${PROJECT_DIR}/.ci/sd4/troubleshoot-opam.sh" \$OPAMROOT +fi +exit \$exitcode +EOF + chmod +x .ci/sd4/opam-with-env + + # Bundle for consumers of setup-dkml.yml + do_tar_rf .ci/sd4/dist/opam-with-env.tar .ci/sd4/opam-with-env + + fi + echo '__ opam-with-env __' >&2 + cat .ci/sd4/opam-with-env >&2 + echo '___________________' >&2 + + # ------- + # opamrun + # ------- + + install -d .ci/sd4/opamrun + cat >.ci/sd4/opamrun/opamrun <>"$GITHUB_PATH" + # Special case: GITHUB_PATH does not influence msys2.CMD of msys2/setup-msys2@v2, so place in real MSYS2 PATH + if [ -n "${MSYSTEM:-}" ]; then + install -d /usr/local/bin + install .ci/sd4/opamrun/opamrun /usr/local/bin/opamrun + fi +fi + +# Place opamrun in the immediate PATH +PATH="$setup_WORKSPACE/.ci/sd4/opamrun:$PATH" + +# Complicated Opam sequence is because: +# 1. Opam's default curl does not work on Windows, +# and `opam init` does not provide a way to change it (TODO: need +# a PR!). +# 2. We have to separate the Opam download cache from the other Opam +# caches +if [ ! -s "$opam_root/.ci.root-init" ]; then # non-empty init file so can be cached irrespective of existence + section_begin opam-init 'Initialize opam root' + + # Clear any partial previous attempt + rm -rf "$opam_root" + + case "$dkml_host_abi,${in_docker:-}" in + windows_*,*) + eor=$(cygpath -am "$setup_WORKSPACE"/.ci/sd4/eor) + opamrun init --disable-sandboxing --no-setup --kind local --bare "$eor" + case "$(opamrun --version)" in + 2.0.*) echo 'download-command: wget' >>"$opam_root/config" ;; + *) opamrun option --yes --global download-command=wget ;; + esac + ;; + *,true) + opamrun init --disable-sandboxing --no-setup --kind local --bare "/work/.ci/sd4/eor" + ;; + *) + opamrun init --disable-sandboxing --no-setup --kind local --bare "$setup_WORKSPACE/.ci/sd4/eor" + ;; + esac + echo yes > "$opam_root/.ci.root-init" + + section_end opam-init +fi + +section_begin opam-vars "Summary: opam global variables" +opamrun --no-troubleshooting var --global || true +section_end opam-vars + +# Build OCaml + +do_switch_create() { + do_switch_create_NAME=$1 + shift + + section_begin "switch-create-$do_switch_create_NAME" "Create opam switch '$do_switch_create_NAME'" + # Create, or recreate, the Opam switch. The Opam switch should not be + # cached except for the compiler (confer docs for setup-ocaml GitHub + # Action) which is the 'dkml' switch (or the 'two' switch). + # Check if the switch name is present in the Opam root (which may come from cache) + NOMINALLY_PRESENT=false + if opamrun switch list --short | grep "^${do_switch_create_NAME}\$"; then NOMINALLY_PRESENT=true; fi + + # Check if the switch is actually present in case of cache incoherence + # or corrupt Opam state that could result in: + # Error: No config file found for switch dkml. Switch broken? + if [ $NOMINALLY_PRESENT = true ] && [ ! -e "$opam_root/$do_switch_create_NAME/.opam-switch/switch-config" ]; then + # Remove the switch name from Opam root, and any partial switch state. + # Ignore inevitable warnings/failure about missing switch. + opamrun --no-troubleshooting switch remove "$do_switch_create_NAME" --yes || true + rm -rf "${opam_root:?}/$do_switch_create_NAME" + NOMINALLY_PRESENT=false + fi + + if [ $NOMINALLY_PRESENT = false ]; then + opamrun switch create "$do_switch_create_NAME" --empty --yes + fi + section_end "switch-create-$do_switch_create_NAME" +} +do_switch_create dkml +if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_switch_create two +else + section_begin "switch-create-two" "Create empty opam switch 'two'" + # Always create a secondary switch ... just empty. Avoid problems with cache content missing + # and idempotency. + opamrun --no-troubleshooting switch remove two --yes || true + rm -rf "$opam_root/two" + opamrun switch create two --empty --yes + section_end "switch-create-two" +fi + +do_switch_active() { + section_begin "switch-active" "Set dkml as active switch" + opamrun switch set dkml --yes + section_end "switch-active" +} +do_switch_active + +do_opam_repositories_add() { + section_begin "opam-repo-add" "Add 'diskuv' opam repository" + if ! opamrun --no-troubleshooting repository list -s | grep '^diskuv'; then + opamrun repository add diskuv "git+https://github.com/diskuv/diskuv-opam-repository.git#${DISKUV_OPAM_REPOSITORY:-$DEFAULT_DISKUV_OPAM_REPOSITORY_TAG}" --yes --dont-select + fi + section_end "opam-repo-add" +} +do_opam_repositories_add + +do_opam_repositories_config() { + do_opam_repositories_config_NAME=$1 + shift + + section_begin "opam-repo-$do_opam_repositories_config_NAME" "Attach repositories to $do_opam_repositories_config_NAME" + + if [ ! -s "$opam_root/.ci.$do_opam_repositories_config_NAME.repo-init" ]; then # non-empty init file so can be cached irrespective of existence + opamrun --no-troubleshooting repository remove default --switch "$do_opam_repositories_config_NAME" --yes || true + opamrun --no-troubleshooting repository remove diskuv --switch "$do_opam_repositories_config_NAME" --yes || true + opamrun repository add default --switch "$do_opam_repositories_config_NAME" --yes + opamrun repository add diskuv --switch "$do_opam_repositories_config_NAME" --yes + echo yes > "$opam_root/.ci.$do_opam_repositories_config_NAME.repo-init" + fi + + section_end "opam-repo-$do_opam_repositories_config_NAME" +} +do_opam_repositories_config dkml +if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_opam_repositories_config two +fi + +do_opam_repositories_update() { + section_begin "opam-repo-update" "Update opam repositories" + # The default repository may be the initial 'eor' (empty) repository + opamrun repository set-url default https://opam.ocaml.org --yes + # Always set the `diskuv` repository url since it can change + opamrun repository set-url diskuv "git+https://github.com/diskuv/diskuv-opam-repository.git#${DISKUV_OPAM_REPOSITORY:-$DEFAULT_DISKUV_OPAM_REPOSITORY_TAG}" --yes --dont-select + # Update both `default` and `diskuv` Opam repositories + opamrun update default diskuv + section_end "opam-repo-update" +} +do_opam_repositories_update + +do_pins() { + do_pins_NAME=$1 + shift + + # dkml-base-compiler + + if [ "${DKML_COMPILER:-}" != '@repository@' ] && [ -z "${DKML_COMPILER:-}" ] && [ -z "${OCAML_COMPILER:-}" ]; then + section_begin checkout-dkml-base-compiler "Pin dkml-base-compiler to default ${DEFAULT_DKML_COMPILER} (neither dkml-base-compiler nor OCAML_COMPILER specified) for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action dkml-base-compiler "https://github.com/diskuv/dkml-compiler.git#${DEFAULT_DKML_COMPILER}" + section_end checkout-dkml-base-compiler + elif [ "${DKML_COMPILER:-}" != '@repository@' ] && [ -n "${DKML_COMPILER:-}" ] && [ -z "${OCAML_COMPILER:-}" ]; then + section_begin checkout-dkml-base-compiler "Pin dkml-base-compiler to $DKML_COMPILER (dkml-base-compiler specified; no OCAML_COMPILER specified) for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action dkml-base-compiler "https://github.com/diskuv/dkml-compiler.git#${DKML_COMPILER}" + section_end checkout-dkml-base-compiler + elif [ -n "${OCAML_COMPILER:-}" ]; then + # Validate OCAML_COMPILER (OCAML_COMPILER specified) + case "${OCAML_COMPILER:-}" in + 4.12.1) true ;; + *) + echo "OCAML_COMPILER version ${OCAML_COMPILER:-} is not supported" >&2 + exit 109 + ;; + esac + + section_begin checkout-dkml-base-compiler "Pin dkml-base-compiler (OCAML_COMPILER specified) for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action dkml-base-compiler "https://github.com/diskuv/dkml-compiler.git#${OCAML_COMPILER}-v${DKML_VERSION}" + section_end checkout-dkml-base-compiler + fi + + # conf-dkml-cross-toolchain + + if [ "${CONF_DKML_CROSS_TOOLCHAIN:-}" != '@repository@' ]; then + section_begin checkout-conf-dkml-cross-toolchain "Pin conf-dkml-cross-toolchain for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action conf-dkml-cross-toolchain "https://github.com/diskuv/conf-dkml-cross-toolchain.git#$CONF_DKML_CROSS_TOOLCHAIN" + section_end checkout-conf-dkml-cross-toolchain + fi + + # patches necessary for Windows in diskuv-opam-repository + # + # - ocamlfind and ocamlbuild + # + # - dune-configurator (and hence Dune) + # Dune 2.9.1 and 3.0.2 will fail to build jst-config.v0.14.1 because for jst-config/discover/discover.ml Dune does: + # cl -nologo -O2 -Gy- -MD -I Z:/.opam_root-cached-8/installer-ocaml/lib/ocaml -o C:\Users\beckf\AppData\Local\Temp\build_f18aec_dune\ocaml-configurator4d3858\c-test-31\test.obj -c C:\Users\beckf\AppData\Local\Temp\build_f18aec_dune\ocaml-configurator4d3858\c-test-31\test.c advapi32.lib ws2_32.lib version.lib + # instead of + # cl -nologo -O2 -Gy- -MD -I Z:/.opam_root-cached-8/installer-ocaml/lib/ocaml /FoC:\Users\beckf\AppData\Local\Temp\build_f18aec_dune\ocaml-configurator4d3858\c-test-31\test.obj -c C:\Users\beckf\AppData\Local\Temp\build_f18aec_dune\ocaml-configurator4d3858\c-test-31\test.c advapi32.lib ws2_32.lib version.lib + # with the (irrelevant) test.c file: + # #include + # #include + # + # #ifdef ARCH_BIG_ENDIAN + # const char *s0 = "BEGIN-0-true-END"; + # #else + # const char *s0 = "BEGIN-0-false-END"; + # #endif + # + # #ifdef ARCH_SIXTYFOUR + # const char *s1 = "BEGIN-1-true-END"; + # #else + # const char *s1 = "BEGIN-1-false-END"; + # #endif + # + # The actual problem is dune-configurator ... we only have patches in Diskuv + # repository up until 2.9.3. Need to upstream fix the problem. + # + # - ppx_expect; only patch is for v0.14.1. Need to upstream fix the problem. + # - base; patches for v0.14.1/2/3. Need to upstream fix the problem. + section_begin "opam-pins-$do_pins_NAME" "Opam pins for $do_pins_NAME switch" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version base "${PIN_BASE}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version bigstringaf "${PIN_BIGSTRINGAF}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version core_kernel "${PIN_CORE_KERNEL}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ctypes "${PIN_CTYPES}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ctypes-foreign "${PIN_CTYPES_FOREIGN}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version curly "${PIN_CURLY}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version digestif "${PIN_DIGESTIF}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version dkml-apps "${PIN_DKML_APPS}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version dune "${PIN_DUNE}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version dune-configurator "${PIN_DUNE_CONFIGURATOR}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ocamlbuild "${PIN_OCAMLBUILD}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ocamlfind "${PIN_OCAMLFIND}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ocp-indent "${PIN_OCP_INDENT}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ppx_expect "${PIN_PPX_EXPECT}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version ptime "${PIN_PTIME}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version time_now "${PIN_TIME_NOW}" + opamrun pin add --switch "$do_pins_NAME" --yes --no-action -k version with-dkml "${PIN_WITH_DKML}" + section_end "opam-pins-$do_pins_NAME" +} + +do_pins dkml +if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_pins two +fi + +do_use_vsstudio() { + do_use_vsstudio_NAME=$1 + shift + case "$dkml_host_abi" in + windows_*) + section_begin "use-vsstudio-$do_use_vsstudio_NAME" "Use Visual Studio in dkml-* Opam packages (Windows) for $do_use_vsstudio_NAME switch" + + # shellcheck disable=SC2153 + E_VS_DIR=$(escape_arg_as_ocaml_string "$VS_DIR") + # shellcheck disable=SC2153 + E_VS_VCVARSVER=$(escape_arg_as_ocaml_string "$VS_VCVARSVER") + # shellcheck disable=SC2153 + E_VS_WINSDKVER=$(escape_arg_as_ocaml_string "$VS_WINSDKVER") + # shellcheck disable=SC2153 + E_VS_MSVSPREFERENCE=$(escape_arg_as_ocaml_string "$VS_MSVSPREFERENCE") + # shellcheck disable=SC2153 + E_VS_CMAKEGENERATOR=$(escape_arg_as_ocaml_string "$VS_CMAKEGENERATOR") + + case "$(opamrun --version)" in + 2.0.*) + if [ "${in_docker}" = "true" ]; then + echo Opam 2.0 support in dockcross to use a portable opam var prefix not yet implemented + exit 67 + fi + OP=$(opamrun var prefix --switch "$do_use_vsstudio_NAME") + OPSC=$OP/.opam-switch/switch-config + if grep setenv: "$OPSC"; then + echo "INFO: Updating switch-config. Old was:" + awk '{print ">> " $0}' "$OPSC" + + awk '$1=="setenv:"{x=1} x==0{print} x==1 && $0=="]"{x=0}' "$OPSC" >"$OPSC".trimmed + mv "$OPSC".trimmed "$OPSC" + fi + echo 'setenv: [' >>"$OPSC" + echo ' [DKML_COMPILE_SPEC = "1"]' >>"$OPSC" + echo ' [DKML_COMPILE_TYPE = "VS"]' >>"$OPSC" + echo " [DKML_COMPILE_VS_DIR = \"$E_VS_DIR\"]" >>"$OPSC" + echo " [DKML_COMPILE_VS_VCVARSVER = \"$E_VS_VCVARSVER\"]" >>"$OPSC" + echo " [DKML_COMPILE_VS_WINSDKVER = \"$E_VS_WINSDKVER\"]" >>"$OPSC" + echo " [DKML_COMPILE_VS_MSVSPREFERENCE = \"$E_VS_MSVSPREFERENCE\"]" >>"$OPSC" + echo " [DKML_COMPILE_VS_CMAKEGENERATOR = \"$E_VS_CMAKEGENERATOR\"]" >>"$OPSC" + echo " [DKML_HOST_ABI = \"${dkml_host_abi}\"]" >>"$OPSC" + echo ']' >>"$OPSC" + cat "$OPSC" >&2 # print + ;; + *) + opamrun option --switch "$do_use_vsstudio_NAME" setenv= # reset + opamrun option --switch "$do_use_vsstudio_NAME" setenv+='DKML_COMPILE_SPEC = "1"' + opamrun option --switch "$do_use_vsstudio_NAME" setenv+='DKML_COMPILE_TYPE = "VS"' + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_DIR = \"$E_VS_DIR\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_VCVARSVER = \"$E_VS_VCVARSVER\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_WINSDKVER = \"$E_VS_WINSDKVER\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_MSVSPREFERENCE = \"$E_VS_MSVSPREFERENCE\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_COMPILE_VS_CMAKEGENERATOR = \"$E_VS_CMAKEGENERATOR\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv+="DKML_HOST_ABI = \"${dkml_host_abi}\"" + opamrun option --switch "$do_use_vsstudio_NAME" setenv # print + ;; + esac + + # shellcheck disable=SC2016 + opamrun exec --switch "$do_use_vsstudio_NAME" -- sh -c 'echo $VCToolsRedistDir' + + section_end "use-vsstudio-$do_use_vsstudio_NAME" + ;; + esac +} +if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_use_vsstudio two +fi +do_use_vsstudio dkml + +# Because dune.X.Y.Z+shim (and any user DKML packages) requires DKML installed (after all, it is just +# a with-dkml.exe shim), we need either dkmlvars-v2.sexp or DKML environment +# variables. Confer: Dkml_runtimelib.Dkml_context.get_dkmlversion +# +# grep matches either: +# [... [DiskuvOCamlVersion = "1.0.1"] ...] +# DiskuvOCamlVersion = "1.0.1" +do_setenv() { + do_setenv_SWITCH=$1 + shift + opamrun option --switch "$do_setenv_SWITCH" setenv > ".ci/sd4/setenv.$do_setenv_SWITCH.txt" + if ! grep -q '\(^|\[\)DiskuvOCamlVarsVersion ' ".ci/sd4/setenv.$do_setenv_SWITCH.txt"; then + opamrun option --switch "$do_setenv_SWITCH" setenv+='DiskuvOCamlVarsVersion = "2"' + fi + if ! grep -q '\(^|\[\)DiskuvOCamlVersion ' ".ci/sd4/setenv.$do_setenv_SWITCH.txt"; then + opamrun option --switch "$do_setenv_SWITCH" setenv+="DiskuvOCamlVersion = \"$DKML_VERSION\"" + fi + case "${dkml_host_abi}" in + windows_*) + if ! grep -q '\(^|\[\)DiskuvOCamlMSYS2Dir ' ".ci/sd4/setenv.$do_setenv_SWITCH.txt"; then + if [ -x /usr/bin/cygpath ]; then + MSYS2_DIR_NATIVE=$(/usr/bin/cygpath -aw /) + else + # If we are already inside MSYS2 then MSYSTEM_PREFIX should be set. But cygpath should be there as well!! + echo "FATAL: Could not locate MSYS2: there was no cygpath" >&2 + exit 3 + fi + MSYS2_DIR_NATIVE_ESCAPED=$(printf "%s" "$MSYS2_DIR_NATIVE" | sed 's/\\/\\\\/g') + opamrun option --switch "$do_setenv_SWITCH" setenv+="DiskuvOCamlMSYS2Dir = \"$MSYS2_DIR_NATIVE_ESCAPED\"" + fi + esac +} +if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_setenv two +fi +do_setenv dkml + +do_install_compiler() { + do_install_compiler_NAME=$1 + shift + section_begin "install-compiler-$do_install_compiler_NAME" "Install OCaml compiler for $do_install_compiler_NAME switch" + opamrun pin list --switch "$do_install_compiler_NAME" + # shellcheck disable=SC2086 + opamrun upgrade --switch "$do_install_compiler_NAME" --yes dkml-base-compiler conf-dkml-cross-toolchain ${ocaml_options:-} + section_end "install-compiler-$do_install_compiler_NAME" +} +do_install_compiler dkml +if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_install_compiler two +fi + +do_summary() { + do_summary_NAME=$1 + shift + section_begin "summary-$do_summary_NAME" "Summary: $do_summary_NAME switch" + opamrun var --switch "$do_summary_NAME" + opamrun exec --switch "$do_summary_NAME" -- ocamlc -config + section_end "summary-$do_summary_NAME" +} +do_summary dkml +if [ "${SECONDARY_SWITCH:-}" = "true" ]; then + do_summary two +fi + +end_of_script + +sh .ci/sd4/run-checkout-code.sh PC_PROJECT_DIR "${PC_PROJECT_DIR}" +sh .ci/sd4/run-setup-dkml.sh PC_PROJECT_DIR "${PC_PROJECT_DIR}" + +# shellcheck disable=SC2154 +echo " +Finished setup. + +To continue your testing, run: + export dkml_host_abi='${dkml_host_abi}' + export abi_pattern='${abi_pattern}' + export opam_root='${opam_root}' + export exe_ext='${exe_ext:-}' + +Now you can use 'opamrun' to do opam commands like: + + PATH=\"$PWD/.ci/sd4/opamrun:\$PATH\" opamrun install XYZ.opam + PATH=\"$PWD/.ci/sd4/opamrun:\$PATH\" opamrun -it exec -- bash + sh ci/build-test.sh +" \ No newline at end of file