From 7033e76bd6291dfe5541715d81fc79d7a0a84243 Mon Sep 17 00:00:00 2001 From: Trey Dockendorf Date: Sat, 9 Nov 2024 09:46:46 -0500 Subject: [PATCH 1/8] Upgrade Ubuntu and Debian to NodeJS 20 --- .github/workflows/e2e.yaml | 8 ++++++++ .github/workflows/release.yaml | 4 ++++ .gitlab-ci.yml | 8 ++++---- lib/ood_packaging/build_box/docker-image/Dockerfile.erb | 2 -- lib/ood_packaging/version.rb | 2 +- packages/ondemand-release-latest/deb/debian/rules | 8 +------- packages/ondemand-release/deb/debian/rules | 8 +------- 7 files changed, 19 insertions(+), 21 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 0ba7889..92df282 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -32,6 +32,10 @@ jobs: arch: ppc64le - dist: ubuntu-22.04 arch: ppc64le + - dist: ubuntu-24.04 + arch: ppc64le + - dist: debian-12 + arch: ppc64le runs-on: ${{ matrix.os }} name: Buildbox tests steps: @@ -276,6 +280,10 @@ jobs: arch: ppc64le - dist: ubuntu-22.04 arch: ppc64le + - dist: ubuntu-24.04 + arch: ppc64le + - dist: debian-12 + arch: ppc64le runs-on: ${{ matrix.os }} name: Package ${{ matrix.package }} dist=${{ matrix.dist }} arch=${{ matrix.arch }} steps: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a623cf8..809d30a 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -29,6 +29,10 @@ jobs: arch: ppc64le - dist: ubuntu-22.04 arch: ppc64le + - dist: ubuntu-24.04 + arch: ppc64le + - dist: debian-12 + arch: ppc64le name: Release buildbox runs-on: ubuntu-latest steps: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 350e2cd..625a247 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -32,9 +32,9 @@ build-image: - OOD_PACKAGING_DIST: [el8] OOD_PACKAGING_ARCH: [x86_64, aarch64, ppc64le] OOD_PACKAGING_GPG_PRIVATE_KEY: /systems/osc_certs/gpg/ondemand/ondemand.sec - - OOD_PACKAGING_DIST: [el9, debian-12] + - OOD_PACKAGING_DIST: [el9] OOD_PACKAGING_ARCH: [x86_64, aarch64, ppc64le] - - OOD_PACKAGING_DIST: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04] + - OOD_PACKAGING_DIST: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04, debian-12] OOD_PACKAGING_ARCH: [x86_64, aarch64] - OOD_PACKAGING_DIST: [amzn2023] OOD_PACKAGING_ARCH: [x86_64, aarch64] @@ -132,9 +132,9 @@ build-passenger: - OOD_PACKAGING_DIST: [el8] OOD_PACKAGING_ARCH: [x86_64, aarch64, ppc64le] OOD_PACKAGING_GPG_PRIVATE_KEY: /systems/osc_certs/gpg/ondemand/ondemand.sec - - OOD_PACKAGING_DIST: [el9, debian-12] + - OOD_PACKAGING_DIST: [el9] OOD_PACKAGING_ARCH: [x86_64, aarch64, ppc64le] - - OOD_PACKAGING_DIST: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04] + - OOD_PACKAGING_DIST: [ubuntu-20.04, ubuntu-22.04, ubuntu-24.04, debian-12] OOD_PACKAGING_ARCH: [x86_64, aarch64] - OOD_PACKAGING_DIST: [amzn2023] # Amazon aarch64 takes hours so build by hand for now diff --git a/lib/ood_packaging/build_box/docker-image/Dockerfile.erb b/lib/ood_packaging/build_box/docker-image/Dockerfile.erb index 0068530..8e92709 100644 --- a/lib/ood_packaging/build_box/docker-image/Dockerfile.erb +++ b/lib/ood_packaging/build_box/docker-image/Dockerfile.erb @@ -32,10 +32,8 @@ RUN apt update -y && apt install -y apt-transport-https ca-certificates \ init debhelper devscripts dh-make build-essential apt-cudf lintian equivs \ sudo rake wget curl ruby bundler && \ apt clean all -y -<% if ['focal', 'jammy'].include?(codename) && arch != 'ppc64le' %> RUN echo "deb https://deb.nodesource.com/node_<%= nodejs_version %>.x nodistro main" > /etc/apt/sources.list.d/nodesource.list RUN curl -s https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor > /etc/apt/trusted.gpg.d/nodesource.gpg -<% end %> <% end -%> RUN ln -s /dev/null /etc/systemd/system/proc-sys-fs-binfmt_misc.automount COPY . /build diff --git a/lib/ood_packaging/version.rb b/lib/ood_packaging/version.rb index b4fdb4f..b861896 100644 --- a/lib/ood_packaging/version.rb +++ b/lib/ood_packaging/version.rb @@ -10,7 +10,7 @@ module OodPackaging }, 'ondemand-release-latest' => { '(ubuntu|debian)' => '4', - 'default' => '1-8' + 'default' => '1-9' }, 'ondemand-runtime' => '4.0.0', 'scl-utils' => '2.0.3', diff --git a/packages/ondemand-release-latest/deb/debian/rules b/packages/ondemand-release-latest/deb/debian/rules index 74e2021..30228a6 100755 --- a/packages/ondemand-release-latest/deb/debian/rules +++ b/packages/ondemand-release-latest/deb/debian/rules @@ -4,7 +4,7 @@ export DH_VERBOSE = 1 export CODENAME = $(shell lsb_release -sc 2>/dev/null) export WEB_DESTDIR = $(CURDIR)/debian/ondemand-release-web-latest export DEB_BUILD_OPTIONS=nocheck -export NODEREPO=node_18.x +export NODEREPO=node_20.x %: dh $@ @@ -20,14 +20,8 @@ override_dh_auto_install: mkdir -p $(WEB_DESTDIR)/etc/apt/trusted.gpg.d echo "deb https://apt.osc.edu/ondemand/latest/web/apt $(CODENAME) main" > $(WEB_DESTDIR)/etc/apt/sources.list.d/ondemand-web.list install -m 644 -D $(CURDIR)/ondemand.gpg $(WEB_DESTDIR)/etc/apt/trusted.gpg.d/ondemand-web.gpg -# Debian 12 (bookworm) and Ubuntu 24.04 (noble) -# have NodeJS 18 in OS repos -ifeq ($(CODENAME),bookworm) -else ifeq ($(CODENAME),noble) -else echo "deb https://deb.nodesource.com/$(NODEREPO) nodistro main" > $(WEB_DESTDIR)/etc/apt/sources.list.d/nodesource.list curl -s https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor > $(WEB_DESTDIR)/etc/apt/trusted.gpg.d/nodesource.gpg -endif override_dh_builddeb: dh_builddeb -- -Zgzip \ No newline at end of file diff --git a/packages/ondemand-release/deb/debian/rules b/packages/ondemand-release/deb/debian/rules index 82d5a68..a79a382 100755 --- a/packages/ondemand-release/deb/debian/rules +++ b/packages/ondemand-release/deb/debian/rules @@ -7,7 +7,7 @@ export DEB_BUILD_OPTIONS=nocheck export VERSION_MAJOR = $(shell echo "$(VERSION)" | cut -d'.' -f1) export VERSION_MINOR = $(shell echo "$(VERSION)" | cut -d'.' -f2) export REPO = $(VERSION_MAJOR).$(VERSION_MINOR) -export NODEREPO=node_18.x +export NODEREPO=node_20.x %: dh $@ @@ -23,14 +23,8 @@ override_dh_auto_install: mkdir -p $(WEB_DESTDIR)/etc/apt/trusted.gpg.d echo "deb https://apt.osc.edu/ondemand/$(REPO)/web/apt $(CODENAME) main" > $(WEB_DESTDIR)/etc/apt/sources.list.d/ondemand-web.list install -m 644 -D $(CURDIR)/ondemand.gpg $(WEB_DESTDIR)/etc/apt/trusted.gpg.d/ondemand-web.gpg -# Debian 12 (bookworm) and Ubuntu 24.04 (noble) -# have NodeJS 18 in OS repos -ifeq ($(CODENAME),bookworm) -else ifeq ($(CODENAME),noble) -else echo "deb https://deb.nodesource.com/$(NODEREPO) nodistro main" > $(WEB_DESTDIR)/etc/apt/sources.list.d/nodesource.list curl -s https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor > $(WEB_DESTDIR)/etc/apt/trusted.gpg.d/nodesource.gpg -endif override_dh_builddeb: dh_builddeb -- -Zgzip \ No newline at end of file From 78d173df5a1858bcaaab7f2064a34b1da03d694a Mon Sep 17 00:00:00 2001 From: Trey Dockendorf Date: Sat, 9 Nov 2024 09:54:26 -0500 Subject: [PATCH 2/8] Bump version, fix ondemand-release-latest for Ubuntu/Debian --- Gemfile.lock | 2 +- lib/ood_packaging/version.rb | 6 +++--- spec/ood_packaging/ood_packaging_spec.rb | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 82a7aa1..70d0ffc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - ood_packaging (0.16.0) + ood_packaging (0.16.1) rake (~> 13.0.1) GEM diff --git a/lib/ood_packaging/version.rb b/lib/ood_packaging/version.rb index b861896..342a324 100644 --- a/lib/ood_packaging/version.rb +++ b/lib/ood_packaging/version.rb @@ -2,15 +2,15 @@ # Version code for OodPackaging module OodPackaging - VERSION = '0.16.0' + VERSION = '0.16.1' PACKAGE_VERSION = { 'ondemand-release' => { '(ubuntu|debian)' => '4.0.0', 'default' => '4.0' }, 'ondemand-release-latest' => { - '(ubuntu|debian)' => '4', - 'default' => '1-9' + '(ubuntu|debian)' => '5', + 'default' => '1-8' }, 'ondemand-runtime' => '4.0.0', 'scl-utils' => '2.0.3', diff --git a/spec/ood_packaging/ood_packaging_spec.rb b/spec/ood_packaging/ood_packaging_spec.rb index caeb055..b749868 100644 --- a/spec/ood_packaging/ood_packaging_spec.rb +++ b/spec/ood_packaging/ood_packaging_spec.rb @@ -8,6 +8,6 @@ end it 'gets package version for dist ubuntu-20.04' do - expect(described_class.package_version('ondemand-release-latest', 'ubuntu-20.04')).to eq('4') + expect(described_class.package_version('ondemand-release-latest', 'ubuntu-24.04')).to eq('5') end end From 7ded8c927857259046e7e467847f7bdd59580f1a Mon Sep 17 00:00:00 2001 From: Trey Dockendorf Date: Sat, 9 Nov 2024 10:25:30 -0500 Subject: [PATCH 3/8] Upgrade amzn2023 to NodeJS 20 --- .../build_box/docker-image/Dockerfile.erb | 3 ++- lib/ood_packaging/utils.rb | 10 ++++++++++ packages/ondemand-runtime/rpm/ondemand-runtime.spec | 4 ++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/lib/ood_packaging/build_box/docker-image/Dockerfile.erb b/lib/ood_packaging/build_box/docker-image/Dockerfile.erb index 8e92709..c3bf2c7 100644 --- a/lib/ood_packaging/build_box/docker-image/Dockerfile.erb +++ b/lib/ood_packaging/build_box/docker-image/Dockerfile.erb @@ -18,7 +18,8 @@ RUN dnf config-manager --set-enabled crb && dnf clean all && rm -rf /var/cache/d <% if dist =~ /^el/ -%> RUN dnf module enable -y ruby:<%= ruby_version %> nodejs:<%= nodejs_version %> && dnf clean all && rm -rf /var/cache/dnf/* <% end -%> -RUN dnf install -y --allowerasing systemd ruby nodejs sudo which wget \ +RUN dnf install -y --allowerasing systemd sudo which wget \ + <%= ruby_package %> <%= nodejs_package %> \ gcc-c++ gcc make patch shadow-utils gnupg2 \ rpm-build rpmdevtools rpm-sign <%= dist =~ /^el/ ? 'scl-utils-build' : '' %> && \ dnf clean all && rm -rf /var/cache/dnf/* diff --git a/lib/ood_packaging/utils.rb b/lib/ood_packaging/utils.rb index 5ed1f4a..c4033ac 100644 --- a/lib/ood_packaging/utils.rb +++ b/lib/ood_packaging/utils.rb @@ -66,10 +66,20 @@ def ruby_version '3.3' end + def ruby_package + return "ruby#{ruby_version}" if dist == 'amzn2023' + 'ruby' + end + def nodejs_version '20' end + def nodejs_package + return "nodejs#{nodejs_version}" if dist == 'amzn2023' + 'nodejs' + end + def ctr_scripts_dir '/ondemand-packaging' end diff --git a/packages/ondemand-runtime/rpm/ondemand-runtime.spec b/packages/ondemand-runtime/rpm/ondemand-runtime.spec index b491f31..933b9c6 100644 --- a/packages/ondemand-runtime/rpm/ondemand-runtime.spec +++ b/packages/ondemand-runtime/rpm/ondemand-runtime.spec @@ -12,7 +12,7 @@ %endif %if 0%{?amzn} == 2023 %global ruby ruby3.2 -%global nodejs nodejs +%global nodejs nodejs20 %global apache httpd %global ruby_version 3.2 %endif @@ -95,7 +95,7 @@ Requires: %{nodejs} >= 1:20.0, %{nodejs} < 1:21.0 Requires: npm %endif %if 0%{?amzn} == 2023 -Requires: %{nodejs} >= 1:18.0, %{nodejs} < 1:19.0 +Requires: %{nodejs} >= 1:20.0, %{nodejs} < 1:21.0 Requires: npm %endif From 9eb3449dd469c0858dab571ae4159f2878cefb73 Mon Sep 17 00:00:00 2001 From: Trey Dockendorf Date: Sat, 9 Nov 2024 10:29:04 -0500 Subject: [PATCH 4/8] Upgrade runtime to 4.0.1 --- lib/ood_packaging/utils.rb | 2 ++ lib/ood_packaging/version.rb | 2 +- packages/passenger/rpm/passenger.spec | 2 +- release-manifest.yaml | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/ood_packaging/utils.rb b/lib/ood_packaging/utils.rb index c4033ac..53cf227 100644 --- a/lib/ood_packaging/utils.rb +++ b/lib/ood_packaging/utils.rb @@ -68,6 +68,7 @@ def ruby_version def ruby_package return "ruby#{ruby_version}" if dist == 'amzn2023' + 'ruby' end @@ -77,6 +78,7 @@ def nodejs_version def nodejs_package return "nodejs#{nodejs_version}" if dist == 'amzn2023' + 'nodejs' end diff --git a/lib/ood_packaging/version.rb b/lib/ood_packaging/version.rb index 342a324..daeb64c 100644 --- a/lib/ood_packaging/version.rb +++ b/lib/ood_packaging/version.rb @@ -12,7 +12,7 @@ module OodPackaging '(ubuntu|debian)' => '5', 'default' => '1-8' }, - 'ondemand-runtime' => '4.0.0', + 'ondemand-runtime' => '4.0.1', 'scl-utils' => '2.0.3', 'passenger' => '6.0.23', 'cjose' => '0.6.1', diff --git a/packages/passenger/rpm/passenger.spec b/packages/passenger/rpm/passenger.spec index 69d2eb3..abdc56a 100644 --- a/packages/passenger/rpm/passenger.spec +++ b/packages/passenger/rpm/passenger.spec @@ -6,7 +6,7 @@ %define release_version %{package_release} %define ngix_release_version 1 -%define runtime_version 4.0.0 +%define runtime_version 4.0.1 %global nginx_user %{?scl_prefix}nginx %global nginx_group %{nginx_user} diff --git a/release-manifest.yaml b/release-manifest.yaml index e45e60f..553a50e 100644 --- a/release-manifest.yaml +++ b/release-manifest.yaml @@ -1,6 +1,6 @@ major: '4.0' full: '4.0.0' -runtime: '4.0.0' +runtime: '4.0.1' # release ondemand-release: From 622c55bb3254eb3fc74c8525ec3e7c7f1f064d02 Mon Sep 17 00:00:00 2001 From: Trey Dockendorf Date: Sat, 9 Nov 2024 10:46:12 -0500 Subject: [PATCH 5/8] Fix Ruby version of amzn2023 --- lib/ood_packaging/utils.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/ood_packaging/utils.rb b/lib/ood_packaging/utils.rb index 53cf227..6d2d06d 100644 --- a/lib/ood_packaging/utils.rb +++ b/lib/ood_packaging/utils.rb @@ -63,6 +63,8 @@ def ondemand_rpm_repo_baseurl end def ruby_version + return '3.2' if dist == 'amzn2023' + '3.3' end From b66ff93ccf35502947c9c0b27955288f44bd1995 Mon Sep 17 00:00:00 2001 From: Trey Dockendorf Date: Sat, 9 Nov 2024 12:39:29 -0500 Subject: [PATCH 6/8] Stop using buildx, as seems this maybe causing frequent failures on Gitlab CI --- lib/ood_packaging/build_box.rb | 2 +- spec/ood_packaging/build_box_spec.rb | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/ood_packaging/build_box.rb b/lib/ood_packaging/build_box.rb index c924eef..9844363 100644 --- a/lib/ood_packaging/build_box.rb +++ b/lib/ood_packaging/build_box.rb @@ -155,7 +155,7 @@ def scripts def build_command if container_runtime == 'docker' - ['buildx', 'build'] + ['build'] else ['build'] end diff --git a/spec/ood_packaging/build_box_spec.rb b/spec/ood_packaging/build_box_spec.rb index f09973b..34e6528 100644 --- a/spec/ood_packaging/build_box_spec.rb +++ b/spec/ood_packaging/build_box_spec.rb @@ -27,7 +27,7 @@ context 'when building for el8' do it 'executes el8 build command' do expected_cmd = [ - 'docker buildx build', '--platform linux/amd64', + 'docker build', '--platform linux/amd64', "--tag ohiosupercomputer/ood-buildbox-el8-x86_64:#{OodPackaging::VERSION}", '--output', 'type=docker', '-f /tmp/dockerfile /fake-builddir' ] @@ -41,7 +41,7 @@ it 'executes ubuntu-20.04 build command' do expected_cmd = [ - 'docker buildx build', '--platform linux/amd64', + 'docker build', '--platform linux/amd64', "--tag ohiosupercomputer/ood-buildbox-ubuntu-20.04-x86_64:#{OodPackaging::VERSION}", '--output', 'type=docker', '-f /tmp/dockerfile /fake-builddir' ] @@ -55,7 +55,7 @@ it 'executes aarch64 build command' do expected_cmd = [ - 'docker buildx build', '--platform linux/arm64', + 'docker build', '--platform linux/arm64', "--tag ohiosupercomputer/ood-buildbox-el8-aarch64:#{OodPackaging::VERSION}", '--output', 'type=docker', '-f /tmp/dockerfile /fake-builddir' ] From f0fca590ab785aa777ce8a94bbbac9ed406a9102 Mon Sep 17 00:00:00 2001 From: Trey Dockendorf Date: Sat, 9 Nov 2024 12:39:45 -0500 Subject: [PATCH 7/8] Setup nodejs and npm alternatives for Amazon image --- lib/ood_packaging/build_box/docker-image/Dockerfile.erb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/ood_packaging/build_box/docker-image/Dockerfile.erb b/lib/ood_packaging/build_box/docker-image/Dockerfile.erb index c3bf2c7..f5ada6c 100644 --- a/lib/ood_packaging/build_box/docker-image/Dockerfile.erb +++ b/lib/ood_packaging/build_box/docker-image/Dockerfile.erb @@ -23,6 +23,10 @@ RUN dnf install -y --allowerasing systemd sudo which wget \ gcc-c++ gcc make patch shadow-utils gnupg2 \ rpm-build rpmdevtools rpm-sign <%= dist =~ /^el/ ? 'scl-utils-build' : '' %> && \ dnf clean all && rm -rf /var/cache/dnf/* +<% if dist == 'amzn2023' -%> +RUN alternatives --install /usr/bin/node node /usr/bin/node-<%= nodejs_version %> 1 && \ + alternatives --install /usr/bin/npm npm /usr/bin/npm-<%= nodejs_version %> 1 +<% end -%> <% elsif dist =~ /^ubuntu|debian/ -%> ENV DEBIAN_FRONTEND=noninteractive RUN apt update -y && apt upgrade -y && apt clean all -y From 934167cbde13d8016447fb54c78a46621abf054f Mon Sep 17 00:00:00 2001 From: Trey Dockendorf Date: Sat, 9 Nov 2024 13:03:55 -0500 Subject: [PATCH 8/8] Add buildx back, didn't solve parallel docker build issues --- lib/ood_packaging/build_box.rb | 2 +- spec/ood_packaging/build_box_spec.rb | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/ood_packaging/build_box.rb b/lib/ood_packaging/build_box.rb index 9844363..c924eef 100644 --- a/lib/ood_packaging/build_box.rb +++ b/lib/ood_packaging/build_box.rb @@ -155,7 +155,7 @@ def scripts def build_command if container_runtime == 'docker' - ['build'] + ['buildx', 'build'] else ['build'] end diff --git a/spec/ood_packaging/build_box_spec.rb b/spec/ood_packaging/build_box_spec.rb index 34e6528..f09973b 100644 --- a/spec/ood_packaging/build_box_spec.rb +++ b/spec/ood_packaging/build_box_spec.rb @@ -27,7 +27,7 @@ context 'when building for el8' do it 'executes el8 build command' do expected_cmd = [ - 'docker build', '--platform linux/amd64', + 'docker buildx build', '--platform linux/amd64', "--tag ohiosupercomputer/ood-buildbox-el8-x86_64:#{OodPackaging::VERSION}", '--output', 'type=docker', '-f /tmp/dockerfile /fake-builddir' ] @@ -41,7 +41,7 @@ it 'executes ubuntu-20.04 build command' do expected_cmd = [ - 'docker build', '--platform linux/amd64', + 'docker buildx build', '--platform linux/amd64', "--tag ohiosupercomputer/ood-buildbox-ubuntu-20.04-x86_64:#{OodPackaging::VERSION}", '--output', 'type=docker', '-f /tmp/dockerfile /fake-builddir' ] @@ -55,7 +55,7 @@ it 'executes aarch64 build command' do expected_cmd = [ - 'docker build', '--platform linux/arm64', + 'docker buildx build', '--platform linux/arm64', "--tag ohiosupercomputer/ood-buildbox-el8-aarch64:#{OodPackaging::VERSION}", '--output', 'type=docker', '-f /tmp/dockerfile /fake-builddir' ]