From 3f34c6f2048f2d739339c7f4a3782c0638bb3380 Mon Sep 17 00:00:00 2001 From: Valentin Volkl Date: Thu, 12 Dec 2024 22:35:19 +0100 Subject: [PATCH 1/2] makedist: update cvmfs package repo urls with fallback Adds a fallback URL for the cvmfs package download. makedist currently looks for the latest package in the listing returned by package servers; since the s3 bucket we are mainly using (and which is the most reliable) does not support listings, this mechanism now relies on reading a small file .latest_cvmfs_version_prod (https://cvmrepo.web.cern.ch/cvmrepo/.latest_cvmfs_version_prod) containing a string with the latest version. I'll add this to our testing to ensure it is kept up to date. Obviously this won't work if you wanted to add mirrors maintained by a third party - one possibility would be to hardcode the latest version in the script. The downloads for the non-default config packages is kept the same, except for a small fix for the egi download. The ecsft.cern.ch url is removed alltogether, as this is the most likely one to cause issues. --- makedist | 93 +++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 61 insertions(+), 32 deletions(-) diff --git a/makedist b/makedist index 0f78f94..2de008b 100755 --- a/makedist +++ b/makedist @@ -75,6 +75,17 @@ distroname() { fi } +# return the distroname as encountered in yum package repo URLS, +# i.e https://cvmrepo.web.cern.ch/cvmrepo/yum/cvmfs/EL/9/ +distroname_yum() { + case " $1 " in + *" rhel "*) echo EL;; + *" suse "*) echo suse;; + *) echo "Distro name $1 not supported" >&2 + exit 2;; + esac +} + distroversion() { if [ -n "$MACHTYPE" ]; then echo "$MACHTYPE"|sed 's/^[^0-9]*\([0-9]*\)-.*/\1/' @@ -100,6 +111,7 @@ distroarch() { } DISTRO="`distroname`" +DISTRO_YUM="`distroname_yum $DISTRO`" VERS="`distroversion`" ARCH="`distroarch`" @@ -118,7 +130,7 @@ EL=$VERS MACH=el$EL if [ "$DISTRO" = "suse" ]; then EL=7 # we get some suse stuff from rhel7 - MACH=sle + MACH=sle$VERS fi HERE="$(cd `dirname $0` && pwd)" @@ -185,10 +197,19 @@ case $1 in EXT="/Packages/c" fi REPO=release - BASEURL="https://repo.opensciencegrid.org/osg/$REL/el$EL/$REPO/x86_64$EXT";; + BASEURL="https://repo.opensciencegrid.org/osg/$REL/el$EL/$REPO/x86_64$EXT" + BASELIST="`curl -Ls $BASEURL/|grep "cvmfs-"|sed 's/.*href="//;s/".*//'`";; egi) OS=centos$EL - BASEURL="https://repository.egi.eu/sw/production/umd/4/$OS/x86_64/updates";; + if [ $EL -gt 8 ]; then + BASEURL="https://repository.egi.eu/sw/production/umd/5/al9/release/x86_64/" + elif [ $EL -eq 7 ]; then + BASEURL="https://repository.egi.eu/sw/production/umd/4/centos7/x86_64/updates" + else + echo "ERROR: unsupported OS for egi! only centos7, almalinux9 supported." >&2 + exit 1 + fi + BASELIST="`curl -Ls $BASEURL/|grep "cvmfs-"|sed 's/.*href="//;s/".*//'`";; default|none) INCLUDEHELPER=false BASEURL="https://cvmrepo.web.cern.ch/cvmrepo/yum/cvmfs/EL/$EL/x86_64";; @@ -196,6 +217,7 @@ case $1 in esac DISTTYPE=$1 + if [ -d $DIST ]; then echo "$DIST already exists" >&2 exit 1 @@ -234,58 +256,64 @@ if [ "$ARCH" != "x86_64" ]; then # we're looking at x86_64 version repositories for the config rpm INCLUDEHELPER=false fi - -URLS="" -BASELIST="`curl -Ls $BASEURL/|grep "cvmfs-"|sed 's/.*href="//;s/".*//'`" -CVMFSPKG="`echo "$BASELIST"|grep "^cvmfs-[0-9]"|tail -1`" -if [ -z "$CVMFSPKG" ]; then - if [ "$DISTTYPE" = egi ] && [ "$EL" = 8 ]; then - echo "egi's UMD does not yet support rhel8" 2>&1 - else - echo "No cvmfs package found from $BASEURL" >&2 - fi +if [ "$DISTTYPE" = egi ] && [ "$EL" = 8 ]; then + echo "egi's UMD does not yet support rhel8" 2>&1 exit 1 fi + +URLS="" CVMFSURL="" +CVMFSRPMURLS="" if [ "$ARCH" = "ppc64le" ]; then # Grab cvmfs package from copr, using yumdownloader to calculate # the URL - CVMFSRPMURL="$(getcoprurl cvmfs)" - if [ -z "$CVMFSRPMURL" ]; then + CVMFSRPMURLS="$(getcoprurl cvmfs)" + if [ -z "$CVMFSRPMURLS" ]; then echo "Failed to get $ARCH cvmfs rpm from copr" >&2 exit 1 fi - CVMFSURL="$(dirname $CVMFSRPMURL)" - CVMFSRPMNAME="$(basename $CVMFSRPMURL)" + CVMFSURL="$(dirname $CVMFSRPMURLS)" + CVMFSRPMNAME="$(basename $CVMFSRPMURLS)" CVMFSVERSION="`echo "$CVMFSRPMNAME" 's/.*cvmfs-\([^-]*\)-.*/\1/'`" else - # Now that we can figure out the latest cvmfs version, download the - # corresponding cvmfs packages from CERN instead - CVMFSVERSION="`echo "$CVMFSPKG"|sed 's/.*cvmfs-\([^-]*\)-.*/\1/'`" - CVMFSURL="https://ecsft.cern.ch/dist/cvmfs/cvmfs-$CVMFSVERSION" - CVMFSRPMNAME="`curl -Ls $CVMFSURL/|grep "cvmfs-$CVMFSVERSION-.*$MACH.*\.$ARCH"|sed 's/.*href="//;s/".*//'|tail -1`" - if [ -z "$CVMFSRPMNAME" ]; then - echo "No matching cvmfs rpm found at $CVMFSURL" >&2 + CVMFS_BASEURL_MIRROR1="https://cvmrepo.s3.cern.ch/cvmrepo" + CVMFS_BASEURL_MIRROR2="https://cvmrepo.web.cern.ch/cvmrepo" + CVMFS_BASEURL=$CVMFS_BASEURL_MIRROR1 + CVMFSVERSION="`curl -Ls $CVMFS_BASEURL/.latest_cvmfs_version_prod`" + if [ -z "CVMFSVERSION" ]; then + echo "WARN: Couldn't find .latest_cvmfs_version_prod in mirror $CVMFS_BASEURL, switching over.." >&2 + CVMFS_BASEURL="$CVMFS_BASE_URL_MIRROR2" + CVMFSVERSION="`curl -Ls $CVMFS_BASEURL/.latest_cvmfs_version_prod`" + if [ -z "CVMFSVERSION" ]; then + echo "ERROR: Couldn't find .latest_cvmfs_version_prod in mirror $CVMFS_BASEURL, and no mirrors left! " >&2 exit 1 + fi fi - CVMFSRPMURL="$CVMFSURL/$CVMFSRPMNAME" + CVMFSURL="$CVMFS_BASEURL/yum/cvmfs/$DISTRO_YUM/$VERS/$ARCH" + CVMFSRPMNAME="cvmfs-$CVMFSVERSION-1.$MACH.$ARCH.rpm" + CVMFSRPMURLS="$CVMFSURL/$CVMFSRPMNAME" fi -URLS="$CVMFSRPMURL" MINORVERSION="`echo "$CVMFSVERSION"|cut -d. -f2`" if [ "$MINORVERSION" -ge 10 ]; then # include cvmfs-libs - URLS="$URLS $CVMFSURL/`echo $CVMFSRPMNAME|sed 's/cvmfs-/cvmfs-libs-/'`" + CVMFSRPMURLS="$CVMFSRPMURLS $CVMFSURL/`echo $CVMFSRPMNAME|sed 's/cvmfs-/cvmfs-libs-/'`" fi +URLS="$URLS $CVMFSRPMURLS" + if [ "$DISTTYPE" != "none" ]; then - CONFIGREPO="`echo "$BASELIST"|grep "^cvmfs-config-$DISTTYPE-[0-9]"|tail -1`" - if [ -z "$CONFIGREPO" ]; then - echo "cvmfs-config not found at $BASEURL!" >&2 - exit 1 + if [ "$DISTTYPE" != "default" ]; then + CONFIGREPO="`echo "$BASELIST"|grep "^cvmfs-config-$DISTTYPE-[0-9]"|tail -1`" + if [ -z "$CONFIGREPO" ]; then + echo "cvmfs-config not found at $BASEURL!" >&2 + exit 1 + fi + URLS="$URLS $BASEURL/$CONFIGREPO" + else + URLS="$URLS $CVMFS_BASEURL/yum/cvmfs-config-default-latest.noarch.rpm" fi - URLS="$URLS $BASEURL/$CONFIGREPO" fi # return the url of the latest version of a package given @@ -327,6 +355,7 @@ fi FUSESUBDIR=false CVMFSFUSE3URL="$CVMFSURL/`echo $CVMFSRPMNAME|sed 's/cvmfs-/cvmfs-fuse3-/'`" +CVMFSRPMURLS="$CVMFSRPMURLS $CVMFSFUSE3URL" if $SING; then URLS="$URLS $CVMFSFUSE3URL" if [ "$EL" -lt 8 ]; then From 642e7a482d16e05d4b2ae694998676e5dd710674 Mon Sep 17 00:00:00 2001 From: Valentin Volkl Date: Thu, 9 Jan 2025 10:18:13 +0100 Subject: [PATCH 2/2] CVMFS_BASEURL still needed for config-default package --- makedist | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/makedist b/makedist index 2de008b..e1a9fcd 100755 --- a/makedist +++ b/makedist @@ -265,6 +265,9 @@ fi URLS="" CVMFSURL="" CVMFSRPMURLS="" +CVMFS_BASEURL_MIRROR1="https://cvmrepo.s3.cern.ch/cvmrepo" +CVMFS_BASEURL_MIRROR2="https://cvmrepo.web.cern.ch/cvmrepo" +CVMFS_BASEURL=$CVMFS_BASEURL_MIRROR1 if [ "$ARCH" = "ppc64le" ]; then # Grab cvmfs package from copr, using yumdownloader to calculate # the URL @@ -277,9 +280,6 @@ if [ "$ARCH" = "ppc64le" ]; then CVMFSRPMNAME="$(basename $CVMFSRPMURLS)" CVMFSVERSION="`echo "$CVMFSRPMNAME" 's/.*cvmfs-\([^-]*\)-.*/\1/'`" else - CVMFS_BASEURL_MIRROR1="https://cvmrepo.s3.cern.ch/cvmrepo" - CVMFS_BASEURL_MIRROR2="https://cvmrepo.web.cern.ch/cvmrepo" - CVMFS_BASEURL=$CVMFS_BASEURL_MIRROR1 CVMFSVERSION="`curl -Ls $CVMFS_BASEURL/.latest_cvmfs_version_prod`" if [ -z "CVMFSVERSION" ]; then echo "WARN: Couldn't find .latest_cvmfs_version_prod in mirror $CVMFS_BASEURL, switching over.." >&2