Skip to content

Commit

Permalink
Merge pull request #33 from dpmatthews/cylc8
Browse files Browse the repository at this point in the history
Support Cylc 8
  • Loading branch information
dpmatthews authored Jun 12, 2024
2 parents fcbdaf8 + 73fe2e7 commit 87ec2bd
Show file tree
Hide file tree
Showing 18 changed files with 267 additions and 54 deletions.
12 changes: 9 additions & 3 deletions .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,7 @@ jobs:
steps:
- name: Install pre requisites
run: sudo apt install virtualbox vagrant
#run: sudo apt update && sudo apt install virtualbox vagrant


- name: Open up network interfaces for VM
run: |
sudo mkdir -p /etc/vbox/
Expand All @@ -37,6 +36,12 @@ jobs:
- name: Show Vagrant version
run: vagrant --version

# - name: Free up space
# run: |
# df -h
# sudo rm -rf /usr/share/dotnet /usr/local/lib/android
# df -h

- name: Run vagrant up
run: |
export VAGRANT_VAGRANTFILE=Vagrantfile.${{ matrix.os }}
Expand All @@ -45,11 +50,12 @@ jobs:
- name: Check for errors
run: |
export VAGRANT_VAGRANTFILE=Vagrantfile.${{ matrix.os }}
vagrant up
vagrant ssh -c "grep 'errors occurred during installation' /var/log/install.log || true"
vagrant ssh -c "grep -q 'No errors detected' /var/log/install.log && \
cylc version --long && \
CYLC_VERSION=8 cylc version --long && \
rose version && \
CYLC_VERSION=8 rose version && \
fcm version && \
svn --version && \
gfortran --version | grep 4.8.5 || fcm test-battery t/fcm-make && \
Expand Down
7 changes: 7 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,13 @@ Once the installation is complete the VM will shutdown.
Run the command `vagrant up` to launch the VM.
A separate window should open containing a lightweight Linux desktop environment ([LXDE](http://lxde.org/)) with a terminal already opened.

Both Cylc 7 and Cylc 8 are installed ([see the migration guide](https://cylc.github.io/cylc-doc/stable/html/7-to-8/index.html)).
Cylc 7 is currently the default.
To use Cylc 8, run the following command before running any Cylc or Rose commands:
```
export CYLC_VERSION=8
```

If your VM includes support for the Met Office Science Repository Service then you will be prompted for your password (and also your user name the first time you use the VM).
If you get your username or password wrong and Subversion fails to connect, just run `mosrs-cache-password` to try again.

Expand Down
2 changes: 1 addition & 1 deletion Vagrantfile.ubuntu-1804-headless
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.vm.box = "bento/ubuntu-18.04"
# Remove "desktop" from the args below if only accessing via SSH
# Remove "mosrs" from the args below if not accessing the Met Office Science Repository Service
config.vm.provision :shell, path: "install.sh", args: "ubuntu 1804 desktop mosrs"
config.vm.provision :shell, path: "install.sh", args: "ubuntu 1804 desktop mosrs skip-shutdown"
config.ssh.forward_x11 = true

config.vm.provider "virtualbox" do |v|
Expand Down
2 changes: 1 addition & 1 deletion Vagrantfile.ubuntu-2204-headless
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
config.vm.box = "bento/ubuntu-22.04"
# Remove "desktop" from the args below if only accessing via SSH
# Remove "mosrs" from the args below if not accessing the Met Office Science Repository Service
config.vm.provision :shell, path: "install.sh", args: "ubuntu 2204 desktop mosrs"
config.vm.provision :shell, path: "install.sh", args: "ubuntu 2204 desktop mosrs skip-shutdown"
config.ssh.forward_x11 = true

config.vm.provider "virtualbox" do |v|
Expand Down
6 changes: 3 additions & 3 deletions home/Desktop/docs.desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[Desktop Entry]
Name=Local Docs
Exec=firefox http://localhost/
Name=Documentation
Exec=xdg-open http://localhost/
Type=Application
Icon=firefox
Icon=browser
28 changes: 17 additions & 11 deletions install-base.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@ fi

#### Install FCM dependencies & configuration
if [[ $dist == ubuntu ]]; then
apt-get install -q -y subversion firefox tkcvs tk libxml-parser-perl || error
apt-get install -q -y subversion chromium-browser tkcvs tk libxml-parser-perl || error
xdg-settings set default-web-browser chromium-browser.desktop
apt-get install -q -y m4 libconfig-inifiles-perl libdbi-perl g++ libsvn-perl || error
apt-get install -q -y xxdiff || error
elif [[ $dist == redhat ]]; then
Expand Down Expand Up @@ -120,10 +121,17 @@ fi
# Add the Cylc wrapper scripts
dos2unix -n /vagrant/usr/local/bin/cylc /usr/local/bin/cylc
cd /usr/local/bin
ln -sf cylc isodatetime
ln -sf cylc gcylc
# Configure additional copyable environment variables
mkdir -p /opt/metomi-site/conf
dos2unix -n /vagrant/opt/metomi-site/conf/global.rc /opt/metomi-site/conf/global.rc
mkdir -p /opt/metomi-site/etc/cylc/flow/8
dos2unix -n /vagrant/opt/metomi-site/etc/cylc/flow/8/global.cylc /opt/metomi-site/etc/cylc/flow/8/global.cylc
# Insecure workaround for browser permissions error
# See https://stackoverflow.com/questions/70753768/jupyter-notebook-access-to-the-file-was-denied
mkdir -p /opt/metomi-site/etc/cylc/uiserver
dos2unix -n /vagrant/opt/metomi-site/etc/cylc/uiserver/jupyter_config.py /opt/metomi-site/etc/cylc/uiserver/jupyter_config.py

#### Install Rose dependencies & configuration
if [[ $dist == ubuntu ]]; then
Expand Down Expand Up @@ -155,16 +163,18 @@ elif [[ $dist == redhat ]]; then
fi
fi
# Add the Rose wrapper scripts
dos2unix -n /vagrant/usr/local/bin/rose /usr/local/bin/rose
cd /usr/local/bin
ln -sf rose rosie
ln -sf cylc rose
ln -sf cylc rosie
# Configure Rose
mkdir -p /opt/metomi-site/etc
if [[ $dist == ubuntu ]]; then
dos2unix -n /vagrant/opt/metomi-site/etc/rose.conf /opt/metomi-site/etc/rose.conf
elif [[ $dist == redhat ]]; then
dos2unix -n /vagrant/opt/metomi-site/etc/rose.conf.redhat /opt/metomi-site/etc/rose.conf
fi
mkdir -p /opt/metomi-site/etc/rose
dos2unix -n /vagrant/opt/metomi-site/etc/rose/rose.conf /opt/metomi-site/etc/rose/rose.conf

#### Install latest versions of FCM, Cylc & Rose
if [[ $dist == ubuntu ]]; then
Expand All @@ -173,10 +183,14 @@ if [[ $dist == ubuntu ]]; then
fi
dos2unix -n /vagrant/usr/local/bin/install-fcm /usr/local/bin/install-fcm
dos2unix -n /vagrant/usr/local/bin/install-cylc7 /usr/local/bin/install-cylc7
dos2unix -n /vagrant/usr/local/bin/install-cylc8 /usr/local/bin/install-cylc8
dos2unix -n /vagrant/usr/local/bin/install-rose /usr/local/bin/install-rose
/usr/local/bin/install-fcm --set-default || error
/usr/local/bin/install-cylc7 --set-default --make-docs || error
/usr/local/bin/install-cylc8 || error
/usr/local/bin/install-rose --set-default --make-docs || error
# Set the default to Cylc 7
ln -sf /opt/cylc-7 /opt/cylc

#### Configure syntax highlighting & bash completion
sudo -u $(logname) mkdir -p /home/vagrant/.local/share/gtksourceview-3.0/language-specs/
Expand All @@ -196,11 +210,6 @@ fi
echo "[[ -f /opt/rose/etc/rose-bash-completion ]] && . /opt/rose/etc/rose-bash-completion" >>/home/vagrant/.bashrc
echo "[[ -f /opt/cylc/conf/cylc-bash-completion ]] && . /opt/cylc/conf/cylc-bash-completion" >>/home/vagrant/.bashrc

#### Configure firefox as the default PDF viewer
sudo -u $(logname) mkdir -p /home/vagrant/.local/share/applications
sudo -u $(logname) bash -c 'echo "[Added Associations]" >/home/vagrant/.local/share/applications/mimeapps.list'
sudo -u $(logname) bash -c 'echo "application/pdf=firefox.desktop;" >>/home/vagrant/.local/share/applications/mimeapps.list'

#### Configure cylc review & rosie web services (with a local rosie repository)
if [[ $dist == ubuntu ]]; then
if [[ $release != 2204 ]]; then
Expand Down Expand Up @@ -244,9 +253,6 @@ dos2unix -n /vagrant/var/www/html/index.html /var/www/html/index.html
if [[ $dist == ubuntu ]]; then
ln -sf /opt/metomi-site/etc/httpd/rosie-wsgi.conf /etc/apache2/conf-enabled/rosie-wsgi.conf
ln -sf /opt/metomi-site/etc/httpd/svn.conf /etc/apache2/conf-enabled/svn.conf
if [[ $release == 2204 ]]; then
echo "WSGIPythonPath /usr/local/lib/python2.7/dist-packages:/opt/rose/lib/python" >> /etc/apache2/conf-enabled/rosie-wsgi.conf
fi
service apache2 restart || error
elif [[ $dist == redhat ]]; then
ln -sf /opt/metomi-site/etc/httpd/rosie-wsgi.conf /etc/httpd/conf.d/rosie-wsgi.conf
Expand Down
4 changes: 2 additions & 2 deletions install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -85,10 +85,10 @@ else
echo "[INFO] No errors detected"
fi

if [[ $collections =~ desktop ]]; then
rm -f /etc/nologin
if [[ $collections =~ desktop && ! $collections =~ skip-shutdown ]]; then
echo Shutting down the system.
echo Please run vagrant up to restart it.
rm /etc/nologin
sudo shutdown -h now
else
echo Please run vagrant ssh to connect.
Expand Down
5 changes: 5 additions & 0 deletions opt/metomi-site/etc/cylc/flow/8/global.cylc
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
[install]
source dirs = ~/cylc-src, ~/roses
[platforms]
[[localhost]]
copyable environment variables = FCM_VERSION
1 change: 1 addition & 0 deletions opt/metomi-site/etc/cylc/uiserver/jupyter_config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
c.ServerApp.use_redirect_file = False
6 changes: 6 additions & 0 deletions opt/metomi-site/etc/rose.conf
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,12 @@ prefixes-ws-default=tmp
prefix-location.tmp=http://localhost/svn/roses-tmp
prefix-ws.tmp=http://localhost/rosie/tmp

prefix-location.mi=https://code.metoffice.gov.uk/svn/roses-mi
prefix-web.mi=https://code.metoffice.gov.uk/trac/roses-mi/intertrac/source:
prefix-ws.mi=https://code.metoffice.gov.uk/rosie/mi
prefix-location.o=https://code.metoffice.gov.uk/svn/roses-o
prefix-web.o=https://code.metoffice.gov.uk/trac/roses-o/intertrac/source:
prefix-ws.o=https://code.metoffice.gov.uk/rosie/o
prefix-location.u=https://code.metoffice.gov.uk/svn/roses-u
prefix-web.u=https://code.metoffice.gov.uk/trac/roses-u/intertrac/source:
prefix-ws.u=https://code.metoffice.gov.uk/rosie/u
6 changes: 6 additions & 0 deletions opt/metomi-site/etc/rose.conf.redhat
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,12 @@ prefixes-ws-default=tmp
prefix-location.tmp=http://localhost/svn/roses-tmp
prefix-ws.tmp=http://localhost/rosie/tmp

prefix-location.mi=https://code.metoffice.gov.uk/svn/roses-mi
prefix-web.mi=https://code.metoffice.gov.uk/trac/roses-mi/intertrac/source:
prefix-ws.mi=https://code.metoffice.gov.uk/rosie/mi
prefix-location.o=https://code.metoffice.gov.uk/svn/roses-o
prefix-web.o=https://code.metoffice.gov.uk/trac/roses-o/intertrac/source:
prefix-ws.o=https://code.metoffice.gov.uk/rosie/o
prefix-location.u=https://code.metoffice.gov.uk/svn/roses-u
prefix-web.u=https://code.metoffice.gov.uk/trac/roses-u/intertrac/source:
prefix-ws.u=https://code.metoffice.gov.uk/rosie/u
29 changes: 29 additions & 0 deletions opt/metomi-site/etc/rose/rose.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
meta-path=/home/vagrant/etc/rose-meta

[rose-mpi-launch]
launcher-fileopts.mpiexec=-file $ROSE_COMMAND_FILE
launcher-list=mpiexec
launcher-preopts.mpiexec=-n $NPROC

[rose-stem]
automatic-options=SITE=vm

[rosie-db]
db.tmp=sqlite:////srv/svn/rosie-tmp-db.sqlite
repos.tmp=/srv/svn/roses-tmp

[rosie-id]
prefix-default=tmp
prefixes-ws-default=tmp
prefix-location.tmp=http://localhost/svn/roses-tmp
prefix-ws.tmp=http://localhost/rosie/tmp

prefix-location.mi=https://code.metoffice.gov.uk/svn/roses-mi
prefix-web.mi=https://code.metoffice.gov.uk/trac/roses-mi/intertrac/source:
prefix-ws.mi=https://code.metoffice.gov.uk/rosie/mi
prefix-location.o=https://code.metoffice.gov.uk/svn/roses-o
prefix-web.o=https://code.metoffice.gov.uk/trac/roses-o/intertrac/source:
prefix-ws.o=https://code.metoffice.gov.uk/rosie/o
prefix-location.u=https://code.metoffice.gov.uk/svn/roses-u
prefix-web.u=https://code.metoffice.gov.uk/trac/roses-u/intertrac/source:
prefix-ws.u=https://code.metoffice.gov.uk/rosie/u
101 changes: 94 additions & 7 deletions usr/local/bin/cylc
Original file line number Diff line number Diff line change
@@ -1,10 +1,97 @@
#!/bin/bash
if [[ -z ${CYLC_HOME:-} ]]; then
CYLC_HOME_ROOT=${CYLC_HOME_ROOT:-/opt}
CYLC_HOME=$CYLC_HOME_ROOT/cylc
if [[ -n ${CYLC_VERSION:-} && -d $CYLC_HOME_ROOT/cylc-$CYLC_VERSION ]]; then
CYLC_HOME=$CYLC_HOME_ROOT/cylc-$CYLC_VERSION
#!/usr/bin/env bash

# Centrally installed Cylc releases:
CYLC_HOME_ROOT="${CYLC_HOME_ROOT:-/opt}"
# Global config locations:
export CYLC_SITE_CONF_PATH=${CYLC_HOME_ROOT}/metomi-site/etc/cylc
export ROSE_SITE_CONF_PATH=${CYLC_HOME_ROOT}/metomi-site/etc/rose

if [[ -z "${CYLC_HOME}" ]]; then
if [[ -z "${CYLC_ENV_NAME}" ]]; then
if [[ -n "${CYLC_VERSION}" ]]; then
CYLC_ENV_NAME="cylc-$CYLC_VERSION"
else
# Use default version (symlink)
CYLC_ENV_NAME="cylc"
fi
fi
# CYLC_VERSION gets set to the actual version of Cylc used within workflows.
# Therefore we export CYLC_ENV_NAME to ensure it gets used rather than
# CYLC_VERSION in subsequent calls to the wrapper.
export CYLC_ENV_NAME
for ROOT in "${CYLC_HOME_ROOT}" "${CYLC_HOME_ROOT_ALT}"; do
if [[ -d "${ROOT}/${CYLC_ENV_NAME}" ]]; then
CYLC_HOME="${ROOT}/${CYLC_ENV_NAME}"
# If CYLC_HOME is a symlink then replace it with the real path and
# set CYLC_ENV_NAME to the linked environment to ensure changes to
# symlinked environments can't affect running workflows.
if [[ -L ${CYLC_HOME} ]]; then
CYLC_HOME=$(readlink -f "${CYLC_HOME}")
CYLC_ENV_NAME=${CYLC_HOME##*/}
fi
break
fi
done
if [[ -z "${CYLC_HOME}" ]]; then
MSG="ERROR: $CYLC_ENV_NAME not found in $CYLC_HOME_ROOT"
if [[ -n "${CYLC_HOME_ROOT_ALT}" ]]; then
MSG="${MSG} or ${CYLC_HOME_ROOT_ALT}"
fi
echo 1>&2 "$MSG"
exit 1
fi
fi

# Legacy support for Rose
if [[ ${0##*/} =~ ^ros ]]; then
# Prior to Cylc 8, Rose used a standalone installation
if [[ -n "${CYLC_ENV_NAME}" ]]; then
ROSE_HOME_ROOT="${ROSE_HOME_ROOT:-$CYLC_HOME_ROOT}"
if [[ ${CYLC_ENV_NAME:-} =~ ^cylc-7 ]]; then
# Cylc 7: Use ROSE_HOME / ROSE_VERSION to select the installation
if [[ -z "${ROSE_HOME:-}" ]]; then
if [[ -n "${ROSE_VERSION:-}" ]]; then
CYLC_HOME="${ROSE_HOME_ROOT}/rose-${ROSE_VERSION}"
else
# Use default version (symlink)
CYLC_HOME="${ROSE_HOME_ROOT}/rose"
fi
else
CYLC_HOME="${ROSE_HOME}"
fi
elif [[ ${1:-} == "edit" || ${1:-} == "config-edit" || \
${0##*/} == "rosie" ]]; then
# Cylc 8: Use Rose 2019.01 to run "rose config-edit" or "rosie"
CYLC_HOME="${ROSE_HOME_ROOT}/rose-2019.01"
fi
elif [[ -z "${CYLC_ENV_NAME}" ]]; then
# If CYLC_HOME was set externally, use ROSE_HOME if it is set
CYLC_HOME="${ROSE_HOME:-$CYLC_HOME}"
fi
fi

exec $CYLC_HOME/bin/$(basename $0) "$@"
# Legacy support for cylc review
if [[ ${0##*/} == "cylc" && ${1:-} == "review" && \
! ${CYLC_ENV_NAME:-} =~ ^cylc-7 ]]; then
# Cylc 8: Use Cylc 7 to run "review"
CYLC_HOME="${CYLC_HOME_ROOT}/cylc-7"
fi

if [[ ! -x "${CYLC_HOME}/bin/${0##*/}" ]]; then
echo 1>&2 "ERROR: ${0##*/} not found in ${CYLC_HOME}"
exit 1
fi

# Set PATH when running cylc hub so that configurable-http-proxy can find node
if [[ ${0##*/} == "cylc" && ${1:-} == "hub" ]]; then
PATH=${CYLC_HOME}/bin:${PATH}
fi

# Run the executable from the selected installation.
# n.b. If CYLC_HOME points to a binary inside a
# ``conda/envs/bin directory`` this is sufficient without using
# ``conda activate``. We avoid ``conda activate`` because:
# - It's slow.
# - Subsequent environment changes (conda activate or deactivate)
# are not straightforward.
exec "${CYLC_HOME}/bin/${0##*/}" "$@"
24 changes: 17 additions & 7 deletions usr/local/bin/fcm
Original file line number Diff line number Diff line change
@@ -1,9 +1,19 @@
#!/bin/bash
if [[ -z ${FCM_HOME:-} ]]; then
FCM_HOME_ROOT=${FCM_HOME_ROOT:-/opt}
FCM_HOME=$FCM_HOME_ROOT/fcm
if [[ -n ${FCM_VERSION:-} && -d $FCM_HOME_ROOT/fcm-$FCM_VERSION ]]; then
FCM_HOME=$FCM_HOME_ROOT/fcm-$FCM_VERSION
#!/usr/bin/env bash

# Centrally installed FCM releases:
FCM_HOME_ROOT="${FCM_HOME_ROOT:-/opt}"

if [[ -z "${FCM_HOME}" ]]; then
if [[ -n "${FCM_VERSION}" ]]; then
FCM_HOME="${FCM_HOME_ROOT}/fcm-${FCM_VERSION}"
else
# Use default version (symlink)
FCM_HOME="${FCM_HOME_ROOT}/fcm"
fi
fi
exec $FCM_HOME/bin/$(basename $0) "$@"
if [[ ! -x "${FCM_HOME}/bin/fcm" ]]; then
echo 1>&2 "ERROR: fcm not found in ${FCM_HOME}"
exit 1
fi

exec "${FCM_HOME}/bin/fcm" "$@"
8 changes: 4 additions & 4 deletions usr/local/bin/install-cylc7
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ done
CYLC_VERSION=${CYLC_VERSION:-$(curl -L -s -S https://api.github.com/repos/cylc/cylc/releases|\
grep tag_name|sort|grep '"7\.'|tail -1|sed -e 's/^.*: "//' -e 's/".*$//')}
if [[ -z $CYLC_VERSION ]]; then
CYLC_VERSION=7.9.7
CYLC_VERSION=7.9.9
echo "[WARNING] Can't determine Cylc version - defaulting to $CYLC_VERSION"
fi

Expand All @@ -53,7 +53,7 @@ else
ln -sf /opt/metomi-site/conf/global.rc /opt/cylc-$CYLC_VERSION/conf/global.rc
fi
if $set_default; then
echo Setting the default Cylc version to $CYLC_VERSION
rm -f /opt/cylc
ln -sf /opt/cylc-$CYLC_VERSION /opt/cylc
echo Setting the default Cylc 7 version to $CYLC_VERSION
rm -f /opt/cylc-7
ln -sf /opt/cylc-$CYLC_VERSION /opt/cylc-7
fi
Loading

0 comments on commit 87ec2bd

Please sign in to comment.