From 5692063990098fae89c5144326f3a3eeeae4665e Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 30 Nov 2018 08:54:55 -0800 Subject: [PATCH 01/41] Move old local process spawner setup aside --- jupyter-spin/{ => old}/Dockerfile | 0 jupyter-spin/{ => old}/build.sh | 0 jupyter-spin/{ => old}/docker-entrypoint.sh | 0 jupyter-spin/{ => old}/etc/ldap.conf | 0 jupyter-spin/{ => old}/etc/nslcd.conf | 0 jupyter-spin/{ => old}/etc/nsswitch.conf | 0 jupyter-spin/{ => old}/etc/pam.d/jupyterhub | 0 jupyter-spin/{ => old}/etc/pam.d/password-auth | 0 jupyter-spin/{ => old}/jupyterhub_config.py | 0 jupyter-spin/{ => old}/packages2.txt | 0 jupyter-spin/{ => old}/packages3.txt | 0 jupyter-spin/{ => old}/supervisord.conf | 0 12 files changed, 0 insertions(+), 0 deletions(-) rename jupyter-spin/{ => old}/Dockerfile (100%) rename jupyter-spin/{ => old}/build.sh (100%) rename jupyter-spin/{ => old}/docker-entrypoint.sh (100%) rename jupyter-spin/{ => old}/etc/ldap.conf (100%) rename jupyter-spin/{ => old}/etc/nslcd.conf (100%) rename jupyter-spin/{ => old}/etc/nsswitch.conf (100%) rename jupyter-spin/{ => old}/etc/pam.d/jupyterhub (100%) rename jupyter-spin/{ => old}/etc/pam.d/password-auth (100%) rename jupyter-spin/{ => old}/jupyterhub_config.py (100%) rename jupyter-spin/{ => old}/packages2.txt (100%) rename jupyter-spin/{ => old}/packages3.txt (100%) rename jupyter-spin/{ => old}/supervisord.conf (100%) diff --git a/jupyter-spin/Dockerfile b/jupyter-spin/old/Dockerfile similarity index 100% rename from jupyter-spin/Dockerfile rename to jupyter-spin/old/Dockerfile diff --git a/jupyter-spin/build.sh b/jupyter-spin/old/build.sh similarity index 100% rename from jupyter-spin/build.sh rename to jupyter-spin/old/build.sh diff --git a/jupyter-spin/docker-entrypoint.sh b/jupyter-spin/old/docker-entrypoint.sh similarity index 100% rename from jupyter-spin/docker-entrypoint.sh rename to jupyter-spin/old/docker-entrypoint.sh diff --git a/jupyter-spin/etc/ldap.conf b/jupyter-spin/old/etc/ldap.conf similarity index 100% rename from jupyter-spin/etc/ldap.conf rename to jupyter-spin/old/etc/ldap.conf diff --git a/jupyter-spin/etc/nslcd.conf b/jupyter-spin/old/etc/nslcd.conf similarity index 100% rename from jupyter-spin/etc/nslcd.conf rename to jupyter-spin/old/etc/nslcd.conf diff --git a/jupyter-spin/etc/nsswitch.conf b/jupyter-spin/old/etc/nsswitch.conf similarity index 100% rename from jupyter-spin/etc/nsswitch.conf rename to jupyter-spin/old/etc/nsswitch.conf diff --git a/jupyter-spin/etc/pam.d/jupyterhub b/jupyter-spin/old/etc/pam.d/jupyterhub similarity index 100% rename from jupyter-spin/etc/pam.d/jupyterhub rename to jupyter-spin/old/etc/pam.d/jupyterhub diff --git a/jupyter-spin/etc/pam.d/password-auth b/jupyter-spin/old/etc/pam.d/password-auth similarity index 100% rename from jupyter-spin/etc/pam.d/password-auth rename to jupyter-spin/old/etc/pam.d/password-auth diff --git a/jupyter-spin/jupyterhub_config.py b/jupyter-spin/old/jupyterhub_config.py similarity index 100% rename from jupyter-spin/jupyterhub_config.py rename to jupyter-spin/old/jupyterhub_config.py diff --git a/jupyter-spin/packages2.txt b/jupyter-spin/old/packages2.txt similarity index 100% rename from jupyter-spin/packages2.txt rename to jupyter-spin/old/packages2.txt diff --git a/jupyter-spin/packages3.txt b/jupyter-spin/old/packages3.txt similarity index 100% rename from jupyter-spin/packages3.txt rename to jupyter-spin/old/packages3.txt diff --git a/jupyter-spin/supervisord.conf b/jupyter-spin/old/supervisord.conf similarity index 100% rename from jupyter-spin/supervisord.conf rename to jupyter-spin/old/supervisord.conf From 00b2e031ea6a9c34673f6f887832020a611ce6f5 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 30 Nov 2018 08:55:29 -0800 Subject: [PATCH 02/41] Change image name to follow convention --- jupyter-dev/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter-dev/build.sh b/jupyter-dev/build.sh index 81a8225..8edcc4e 100644 --- a/jupyter-dev/build.sh +++ b/jupyter-dev/build.sh @@ -5,4 +5,4 @@ branch=$(git symbolic-ref --short HEAD) docker build \ --build-arg branch=$branch \ --no-cache \ - --tag registry.spin.nersc.gov/das/jupyterhub-jupyter-dev.$branch:latest . + --tag registry.spin.nersc.gov/das/jupyter-dev-nersc-web.$branch:latest . From 4136c8966f6f1ba7c22f3531961419189c46a400 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 30 Nov 2018 08:56:28 -0800 Subject: [PATCH 03/41] Current setup is for Gerty --- jupyter-dev/jupyterhub_config.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/jupyter-dev/jupyterhub_config.py b/jupyter-dev/jupyterhub_config.py index 9a5a9aa..b9197d4 100644 --- a/jupyter-dev/jupyterhub_config.py +++ b/jupyter-dev/jupyterhub_config.py @@ -18,7 +18,8 @@ def comma_split(string): ip = requests.get('https://v4.ifconfig.co/json').json()['ip'] -bindir = '/global/common/cori/software/python/3.6-anaconda-5.2/bin/' +#bindir = '/global/common/cori/software/python/3.6-anaconda-5.2/bin/' +bindir = '/global/common/gerty/software/python/3.6-anaconda-5.2/bin/' if 'BASE_PATH' in os.environ: bindir = os.path.join(os.environ['BASE_PATH'], 'bin') @@ -978,7 +979,8 @@ def comma_split(string): # SSHSpawner(Spawner) configuration #------------------------------------------------------------------------------ -c.SSHSpawner.remote_hosts = ['cori19-224.nersc.gov'] +#c.SSHSpawner.remote_hosts = ['cori19-224.nersc.gov'] +c.SSHSpawner.remote_hosts = ['gert01-224.nersc.gov'] c.SSHSpawner.remote_port = '22' c.SSHSpawner.hub_api_url = "http://{}:8081/hub/api".format(ip) c.SSHSpawner.path = bindir + ':/global/common/cori/das/jupyterhub/:/usr/common/usg/bin:/usr/bin:/bin' @@ -1000,4 +1002,4 @@ async def my_quota(spawner): e.my_message = "There is insufficient space in your home directory; please clear up some files and try again." raise e -c.Spawner.pre_spawn_hook = my_quota \ No newline at end of file +c.Spawner.pre_spawn_hook = my_quota From 6663d3114329306c1daa36038cce528d46b2bd9c Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 30 Nov 2018 08:57:56 -0800 Subject: [PATCH 04/41] Move to image subdirectory "web" --- jupyter-dev/{ => web}/Dockerfile | 0 jupyter-dev/{ => web}/build.sh | 0 jupyter-dev/{ => web}/docker-compose.yml | 0 jupyter-dev/{ => web}/docker-entrypoint.sh | 0 jupyter-dev/{ => web}/hub-scripts/flush-certs.sh | 0 jupyter-dev/{ => web}/hub-scripts/kill-cori.sh | 0 jupyter-dev/{ => web}/hub-scripts/scram-user.sh | 0 jupyter-dev/{ => web}/hub-scripts/test-user.sh | 0 jupyter-dev/{ => web}/jupyterhub_config.py | 0 jupyter-dev/{ => web}/templates/error.html | 0 jupyter-dev/{ => web}/templates/login.html | 0 11 files changed, 0 insertions(+), 0 deletions(-) rename jupyter-dev/{ => web}/Dockerfile (100%) rename jupyter-dev/{ => web}/build.sh (100%) rename jupyter-dev/{ => web}/docker-compose.yml (100%) rename jupyter-dev/{ => web}/docker-entrypoint.sh (100%) rename jupyter-dev/{ => web}/hub-scripts/flush-certs.sh (100%) rename jupyter-dev/{ => web}/hub-scripts/kill-cori.sh (100%) rename jupyter-dev/{ => web}/hub-scripts/scram-user.sh (100%) rename jupyter-dev/{ => web}/hub-scripts/test-user.sh (100%) rename jupyter-dev/{ => web}/jupyterhub_config.py (100%) rename jupyter-dev/{ => web}/templates/error.html (100%) rename jupyter-dev/{ => web}/templates/login.html (100%) diff --git a/jupyter-dev/Dockerfile b/jupyter-dev/web/Dockerfile similarity index 100% rename from jupyter-dev/Dockerfile rename to jupyter-dev/web/Dockerfile diff --git a/jupyter-dev/build.sh b/jupyter-dev/web/build.sh similarity index 100% rename from jupyter-dev/build.sh rename to jupyter-dev/web/build.sh diff --git a/jupyter-dev/docker-compose.yml b/jupyter-dev/web/docker-compose.yml similarity index 100% rename from jupyter-dev/docker-compose.yml rename to jupyter-dev/web/docker-compose.yml diff --git a/jupyter-dev/docker-entrypoint.sh b/jupyter-dev/web/docker-entrypoint.sh similarity index 100% rename from jupyter-dev/docker-entrypoint.sh rename to jupyter-dev/web/docker-entrypoint.sh diff --git a/jupyter-dev/hub-scripts/flush-certs.sh b/jupyter-dev/web/hub-scripts/flush-certs.sh similarity index 100% rename from jupyter-dev/hub-scripts/flush-certs.sh rename to jupyter-dev/web/hub-scripts/flush-certs.sh diff --git a/jupyter-dev/hub-scripts/kill-cori.sh b/jupyter-dev/web/hub-scripts/kill-cori.sh similarity index 100% rename from jupyter-dev/hub-scripts/kill-cori.sh rename to jupyter-dev/web/hub-scripts/kill-cori.sh diff --git a/jupyter-dev/hub-scripts/scram-user.sh b/jupyter-dev/web/hub-scripts/scram-user.sh similarity index 100% rename from jupyter-dev/hub-scripts/scram-user.sh rename to jupyter-dev/web/hub-scripts/scram-user.sh diff --git a/jupyter-dev/hub-scripts/test-user.sh b/jupyter-dev/web/hub-scripts/test-user.sh similarity index 100% rename from jupyter-dev/hub-scripts/test-user.sh rename to jupyter-dev/web/hub-scripts/test-user.sh diff --git a/jupyter-dev/jupyterhub_config.py b/jupyter-dev/web/jupyterhub_config.py similarity index 100% rename from jupyter-dev/jupyterhub_config.py rename to jupyter-dev/web/jupyterhub_config.py diff --git a/jupyter-dev/templates/error.html b/jupyter-dev/web/templates/error.html similarity index 100% rename from jupyter-dev/templates/error.html rename to jupyter-dev/web/templates/error.html diff --git a/jupyter-dev/templates/login.html b/jupyter-dev/web/templates/login.html similarity index 100% rename from jupyter-dev/templates/login.html rename to jupyter-dev/web/templates/login.html From 13fa953c4fcd2309207ca4bc27cfeb299702d94c Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 30 Nov 2018 08:58:39 -0800 Subject: [PATCH 05/41] Move to match stack name --- {jupyter-dev => jupyter-dev-nersc}/web/Dockerfile | 0 {jupyter-dev => jupyter-dev-nersc}/web/build.sh | 0 {jupyter-dev => jupyter-dev-nersc}/web/docker-compose.yml | 0 {jupyter-dev => jupyter-dev-nersc}/web/docker-entrypoint.sh | 0 {jupyter-dev => jupyter-dev-nersc}/web/hub-scripts/flush-certs.sh | 0 {jupyter-dev => jupyter-dev-nersc}/web/hub-scripts/kill-cori.sh | 0 {jupyter-dev => jupyter-dev-nersc}/web/hub-scripts/scram-user.sh | 0 {jupyter-dev => jupyter-dev-nersc}/web/hub-scripts/test-user.sh | 0 {jupyter-dev => jupyter-dev-nersc}/web/jupyterhub_config.py | 0 {jupyter-dev => jupyter-dev-nersc}/web/templates/error.html | 0 {jupyter-dev => jupyter-dev-nersc}/web/templates/login.html | 0 11 files changed, 0 insertions(+), 0 deletions(-) rename {jupyter-dev => jupyter-dev-nersc}/web/Dockerfile (100%) rename {jupyter-dev => jupyter-dev-nersc}/web/build.sh (100%) rename {jupyter-dev => jupyter-dev-nersc}/web/docker-compose.yml (100%) rename {jupyter-dev => jupyter-dev-nersc}/web/docker-entrypoint.sh (100%) rename {jupyter-dev => jupyter-dev-nersc}/web/hub-scripts/flush-certs.sh (100%) rename {jupyter-dev => jupyter-dev-nersc}/web/hub-scripts/kill-cori.sh (100%) rename {jupyter-dev => jupyter-dev-nersc}/web/hub-scripts/scram-user.sh (100%) rename {jupyter-dev => jupyter-dev-nersc}/web/hub-scripts/test-user.sh (100%) rename {jupyter-dev => jupyter-dev-nersc}/web/jupyterhub_config.py (100%) rename {jupyter-dev => jupyter-dev-nersc}/web/templates/error.html (100%) rename {jupyter-dev => jupyter-dev-nersc}/web/templates/login.html (100%) diff --git a/jupyter-dev/web/Dockerfile b/jupyter-dev-nersc/web/Dockerfile similarity index 100% rename from jupyter-dev/web/Dockerfile rename to jupyter-dev-nersc/web/Dockerfile diff --git a/jupyter-dev/web/build.sh b/jupyter-dev-nersc/web/build.sh similarity index 100% rename from jupyter-dev/web/build.sh rename to jupyter-dev-nersc/web/build.sh diff --git a/jupyter-dev/web/docker-compose.yml b/jupyter-dev-nersc/web/docker-compose.yml similarity index 100% rename from jupyter-dev/web/docker-compose.yml rename to jupyter-dev-nersc/web/docker-compose.yml diff --git a/jupyter-dev/web/docker-entrypoint.sh b/jupyter-dev-nersc/web/docker-entrypoint.sh similarity index 100% rename from jupyter-dev/web/docker-entrypoint.sh rename to jupyter-dev-nersc/web/docker-entrypoint.sh diff --git a/jupyter-dev/web/hub-scripts/flush-certs.sh b/jupyter-dev-nersc/web/hub-scripts/flush-certs.sh similarity index 100% rename from jupyter-dev/web/hub-scripts/flush-certs.sh rename to jupyter-dev-nersc/web/hub-scripts/flush-certs.sh diff --git a/jupyter-dev/web/hub-scripts/kill-cori.sh b/jupyter-dev-nersc/web/hub-scripts/kill-cori.sh similarity index 100% rename from jupyter-dev/web/hub-scripts/kill-cori.sh rename to jupyter-dev-nersc/web/hub-scripts/kill-cori.sh diff --git a/jupyter-dev/web/hub-scripts/scram-user.sh b/jupyter-dev-nersc/web/hub-scripts/scram-user.sh similarity index 100% rename from jupyter-dev/web/hub-scripts/scram-user.sh rename to jupyter-dev-nersc/web/hub-scripts/scram-user.sh diff --git a/jupyter-dev/web/hub-scripts/test-user.sh b/jupyter-dev-nersc/web/hub-scripts/test-user.sh similarity index 100% rename from jupyter-dev/web/hub-scripts/test-user.sh rename to jupyter-dev-nersc/web/hub-scripts/test-user.sh diff --git a/jupyter-dev/web/jupyterhub_config.py b/jupyter-dev-nersc/web/jupyterhub_config.py similarity index 100% rename from jupyter-dev/web/jupyterhub_config.py rename to jupyter-dev-nersc/web/jupyterhub_config.py diff --git a/jupyter-dev/web/templates/error.html b/jupyter-dev-nersc/web/templates/error.html similarity index 100% rename from jupyter-dev/web/templates/error.html rename to jupyter-dev-nersc/web/templates/error.html diff --git a/jupyter-dev/web/templates/login.html b/jupyter-dev-nersc/web/templates/login.html similarity index 100% rename from jupyter-dev/web/templates/login.html rename to jupyter-dev-nersc/web/templates/login.html From ad1105b946f64416d0ce5584992420664b1e6bfc Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 30 Nov 2018 08:59:34 -0800 Subject: [PATCH 06/41] Rename to stack name --- {jupyter-spin => jupyter-nersc}/old/Dockerfile | 0 {jupyter-spin => jupyter-nersc}/old/build.sh | 0 {jupyter-spin => jupyter-nersc}/old/docker-entrypoint.sh | 0 {jupyter-spin => jupyter-nersc}/old/etc/ldap.conf | 0 {jupyter-spin => jupyter-nersc}/old/etc/nslcd.conf | 0 {jupyter-spin => jupyter-nersc}/old/etc/nsswitch.conf | 0 {jupyter-spin => jupyter-nersc}/old/etc/pam.d/jupyterhub | 0 {jupyter-spin => jupyter-nersc}/old/etc/pam.d/password-auth | 0 {jupyter-spin => jupyter-nersc}/old/jupyterhub_config.py | 0 {jupyter-spin => jupyter-nersc}/old/packages2.txt | 0 {jupyter-spin => jupyter-nersc}/old/packages3.txt | 0 {jupyter-spin => jupyter-nersc}/old/supervisord.conf | 0 12 files changed, 0 insertions(+), 0 deletions(-) rename {jupyter-spin => jupyter-nersc}/old/Dockerfile (100%) rename {jupyter-spin => jupyter-nersc}/old/build.sh (100%) rename {jupyter-spin => jupyter-nersc}/old/docker-entrypoint.sh (100%) rename {jupyter-spin => jupyter-nersc}/old/etc/ldap.conf (100%) rename {jupyter-spin => jupyter-nersc}/old/etc/nslcd.conf (100%) rename {jupyter-spin => jupyter-nersc}/old/etc/nsswitch.conf (100%) rename {jupyter-spin => jupyter-nersc}/old/etc/pam.d/jupyterhub (100%) rename {jupyter-spin => jupyter-nersc}/old/etc/pam.d/password-auth (100%) rename {jupyter-spin => jupyter-nersc}/old/jupyterhub_config.py (100%) rename {jupyter-spin => jupyter-nersc}/old/packages2.txt (100%) rename {jupyter-spin => jupyter-nersc}/old/packages3.txt (100%) rename {jupyter-spin => jupyter-nersc}/old/supervisord.conf (100%) diff --git a/jupyter-spin/old/Dockerfile b/jupyter-nersc/old/Dockerfile similarity index 100% rename from jupyter-spin/old/Dockerfile rename to jupyter-nersc/old/Dockerfile diff --git a/jupyter-spin/old/build.sh b/jupyter-nersc/old/build.sh similarity index 100% rename from jupyter-spin/old/build.sh rename to jupyter-nersc/old/build.sh diff --git a/jupyter-spin/old/docker-entrypoint.sh b/jupyter-nersc/old/docker-entrypoint.sh similarity index 100% rename from jupyter-spin/old/docker-entrypoint.sh rename to jupyter-nersc/old/docker-entrypoint.sh diff --git a/jupyter-spin/old/etc/ldap.conf b/jupyter-nersc/old/etc/ldap.conf similarity index 100% rename from jupyter-spin/old/etc/ldap.conf rename to jupyter-nersc/old/etc/ldap.conf diff --git a/jupyter-spin/old/etc/nslcd.conf b/jupyter-nersc/old/etc/nslcd.conf similarity index 100% rename from jupyter-spin/old/etc/nslcd.conf rename to jupyter-nersc/old/etc/nslcd.conf diff --git a/jupyter-spin/old/etc/nsswitch.conf b/jupyter-nersc/old/etc/nsswitch.conf similarity index 100% rename from jupyter-spin/old/etc/nsswitch.conf rename to jupyter-nersc/old/etc/nsswitch.conf diff --git a/jupyter-spin/old/etc/pam.d/jupyterhub b/jupyter-nersc/old/etc/pam.d/jupyterhub similarity index 100% rename from jupyter-spin/old/etc/pam.d/jupyterhub rename to jupyter-nersc/old/etc/pam.d/jupyterhub diff --git a/jupyter-spin/old/etc/pam.d/password-auth b/jupyter-nersc/old/etc/pam.d/password-auth similarity index 100% rename from jupyter-spin/old/etc/pam.d/password-auth rename to jupyter-nersc/old/etc/pam.d/password-auth diff --git a/jupyter-spin/old/jupyterhub_config.py b/jupyter-nersc/old/jupyterhub_config.py similarity index 100% rename from jupyter-spin/old/jupyterhub_config.py rename to jupyter-nersc/old/jupyterhub_config.py diff --git a/jupyter-spin/old/packages2.txt b/jupyter-nersc/old/packages2.txt similarity index 100% rename from jupyter-spin/old/packages2.txt rename to jupyter-nersc/old/packages2.txt diff --git a/jupyter-spin/old/packages3.txt b/jupyter-nersc/old/packages3.txt similarity index 100% rename from jupyter-spin/old/packages3.txt rename to jupyter-nersc/old/packages3.txt diff --git a/jupyter-spin/old/supervisord.conf b/jupyter-nersc/old/supervisord.conf similarity index 100% rename from jupyter-spin/old/supervisord.conf rename to jupyter-nersc/old/supervisord.conf From 2a78b0406a0adb2453fe3700c35902334f4eccd6 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 30 Nov 2018 09:01:28 -0800 Subject: [PATCH 07/41] Add and set name to follow image name conventions --- jupyter-nersc/web/build.sh | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 jupyter-nersc/web/build.sh diff --git a/jupyter-nersc/web/build.sh b/jupyter-nersc/web/build.sh new file mode 100644 index 0000000..b72f6ff --- /dev/null +++ b/jupyter-nersc/web/build.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +branch=$(git symbolic-ref --short HEAD) + +docker build \ + --build-arg branch=$branch \ + --no-cache \ + --tag registry.spin.nersc.gov/das/jupyter-nersc-web.$branch:latest . From 81ca1657b45600fcfab7d73ee6336c0514c1ac23 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 30 Nov 2018 09:04:44 -0800 Subject: [PATCH 08/41] Move more for symmetry than anything else --- {base => jupyter-base}/Dockerfile | 0 {base => jupyter-base}/build.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {base => jupyter-base}/Dockerfile (100%) rename {base => jupyter-base}/build.sh (100%) diff --git a/base/Dockerfile b/jupyter-base/Dockerfile similarity index 100% rename from base/Dockerfile rename to jupyter-base/Dockerfile diff --git a/base/build.sh b/jupyter-base/build.sh similarity index 100% rename from base/build.sh rename to jupyter-base/build.sh From 105b182f9f7e586bd18a68d3e7eafeffc74cb323 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 30 Nov 2018 09:06:19 -0800 Subject: [PATCH 09/41] Fix-up name to match other images --- jupyter-base/build.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jupyter-base/build.sh b/jupyter-base/build.sh index 4749334..bcae179 100644 --- a/jupyter-base/build.sh +++ b/jupyter-base/build.sh @@ -1,4 +1,7 @@ #!/bin/bash branch=$(git symbolic-ref --short HEAD) -docker build --no-cache -t registry.spin.nersc.gov/das/jupyterhub-base.$branch:latest . + +docker build \ + --no-cache \ + --tag registry.spin.nersc.gov/das/jupyter-base.$branch:latest . From b92769091891aba627d8353fbdfbe4fb1dfe9591 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 30 Nov 2018 09:09:57 -0800 Subject: [PATCH 10/41] Update to match base image name --- jupyter-dev-nersc/web/Dockerfile | 2 +- jupyter-localhost/Dockerfile | 2 +- jupyter-sshspawner/app/Dockerfile | 2 +- jupyter-sshspawner/web/Dockerfile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/jupyter-dev-nersc/web/Dockerfile b/jupyter-dev-nersc/web/Dockerfile index 3edaeaf..7b20602 100644 --- a/jupyter-dev-nersc/web/Dockerfile +++ b/jupyter-dev-nersc/web/Dockerfile @@ -1,6 +1,6 @@ ARG branch=unknown -FROM registry.spin.nersc.gov/das/jupyterhub-base.${branch}:latest +FROM registry.spin.nersc.gov/das/jupyter-base.${branch}:latest LABEL maintainer="Rollin Thomas " WORKDIR /srv diff --git a/jupyter-localhost/Dockerfile b/jupyter-localhost/Dockerfile index e37a2bf..0145036 100644 --- a/jupyter-localhost/Dockerfile +++ b/jupyter-localhost/Dockerfile @@ -1,4 +1,4 @@ -FROM registry.spin.nersc.gov/das/jupyterhub-base:latest +FROM registry.spin.nersc.gov/das/jupyter-base:latest MAINTAINER Rollin Thomas # Python 3 Anaconda and additional packages diff --git a/jupyter-sshspawner/app/Dockerfile b/jupyter-sshspawner/app/Dockerfile index e21dd2f..698d650 100644 --- a/jupyter-sshspawner/app/Dockerfile +++ b/jupyter-sshspawner/app/Dockerfile @@ -1,4 +1,4 @@ -FROM registry.spin.nersc.gov/das/jupyterhub-base:latest +FROM registry.spin.nersc.gov/das/jupyter-base:latest MAINTAINER Rollin Thomas WORKDIR /tmp diff --git a/jupyter-sshspawner/web/Dockerfile b/jupyter-sshspawner/web/Dockerfile index 849e301..1109e07 100644 --- a/jupyter-sshspawner/web/Dockerfile +++ b/jupyter-sshspawner/web/Dockerfile @@ -1,4 +1,4 @@ -FROM registry.spin.nersc.gov/das/jupyterhub-jupyter-dev.deploy-18-10:latest +FROM registry.spin.nersc.gov/das/jupyter-dev-nersc.deploy-18-10:latest MAINTAINER Rollin Thomas # JupyterHub components From 6d74bc07d7bb97f118ba5c409e7aa05536050adc Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 30 Nov 2018 09:12:06 -0800 Subject: [PATCH 11/41] Add Dockerfile --- jupyter-nersc/web/Dockerfile | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 jupyter-nersc/web/Dockerfile diff --git a/jupyter-nersc/web/Dockerfile b/jupyter-nersc/web/Dockerfile new file mode 100644 index 0000000..7b20602 --- /dev/null +++ b/jupyter-nersc/web/Dockerfile @@ -0,0 +1,36 @@ +ARG branch=unknown + +FROM registry.spin.nersc.gov/das/jupyter-base.${branch}:latest +LABEL maintainer="Rollin Thomas " +WORKDIR /srv + +# Authenticator and spawner + +RUN \ + pip install git+https://github.com/nersc/sshapiauthenticator.git && \ + pip install git+https://github.com/nersc/sshspawner.git + +# MFA-enabled login template + +ADD templates templates + +# Install announcement service components + +RUN \ + cp /tmp/jupyterhub/examples/service-announcement/announcement.py . && \ + cp /tmp/jupyterhub/examples/service-announcement/templates/page.html templates/. + +# Hub scripts + +ADD hub-scripts/*.sh /srv/ + +# Volume for user cert/key files + +VOLUME /certs + +# Entrypoint and command + +ADD docker-entrypoint.sh jupyterhub_config.py /srv/ +RUN chmod +x docker-entrypoint.sh +ENTRYPOINT ["./docker-entrypoint.sh"] +CMD ["jupyterhub", "--debug"] From adfde8cf8f62f71b24af2ad9d04a09e23fd848a0 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 30 Nov 2018 09:12:49 -0800 Subject: [PATCH 12/41] Add entrypoint --- jupyter-nersc/web/docker-entrypoint.sh | 29 ++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 jupyter-nersc/web/docker-entrypoint.sh diff --git a/jupyter-nersc/web/docker-entrypoint.sh b/jupyter-nersc/web/docker-entrypoint.sh new file mode 100644 index 0000000..057050a --- /dev/null +++ b/jupyter-nersc/web/docker-entrypoint.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# file_env VAR [DEFAULT] +# ---------------------- +# Treat the value of VAR_FILE as the path to a secrets file and initialize VAR +# with the contents of that file. From postgres docker-entrypoint.sh. + +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then + echo >&2 "error: both $var and $fileVar are set (but are exclusive)" + exit 1 + fi + local val="$def" + if [ "${!var:-}" ]; then + val="${!var}" + elif [ "${!fileVar:-}" ]; then + val="$(< "${!fileVar}")" + fi + export "$var"="$val" + unset "$fileVar" +} + +file_env 'POSTGRES_PASSWORD' +file_env 'SKEY' + +exec "$@" From a84c6d143ad50103f65afeb97d3cee8153448ed4 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 30 Nov 2018 09:36:06 -0800 Subject: [PATCH 13/41] Couple scripts to help test/clean up --- jupyter-nersc/web/hub-scripts/flush-certs.sh | 9 +++++++++ jupyter-nersc/web/hub-scripts/test-user.sh | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 jupyter-nersc/web/hub-scripts/flush-certs.sh create mode 100644 jupyter-nersc/web/hub-scripts/test-user.sh diff --git a/jupyter-nersc/web/hub-scripts/flush-certs.sh b/jupyter-nersc/web/hub-scripts/flush-certs.sh new file mode 100644 index 0000000..5c1e2cf --- /dev/null +++ b/jupyter-nersc/web/hub-scripts/flush-certs.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# Get rid of any cert files older than 2 weeks + +limit=14 + +find /certs -type f -name '*.key' -mtime +$limit -exec rm {} \; +find /certs -type f -name '*.key-cert.pub' -mtime +$limit -exec rm {} \; +find /certs -type f -name '*.key.pub' -mtime +$limit -exec rm {} \; diff --git a/jupyter-nersc/web/hub-scripts/test-user.sh b/jupyter-nersc/web/hub-scripts/test-user.sh new file mode 100644 index 0000000..5dbab43 --- /dev/null +++ b/jupyter-nersc/web/hub-scripts/test-user.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Test user's ability to ssh + +hostname=app +username=$1 +cert=/certs/$username.key +echo $username $cert +if [ ! -f $cert ]; then + echo " ... no cert for $username" + exit 1 +fi +/usr/bin/ssh \ + -i $cert \ + -l $username \ + -o PreferredAuthentications=publickey \ + -o StrictHostKeyChecking=no \ + -p 22 \ + $hostname From 4946aa924b5436c46b394cd15f4b8b756386f21c Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 30 Nov 2018 09:37:13 -0800 Subject: [PATCH 14/41] Our templates --- jupyter-nersc/web/templates/error.html | 12 ++++ jupyter-nersc/web/templates/login.html | 99 ++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 jupyter-nersc/web/templates/error.html create mode 100644 jupyter-nersc/web/templates/login.html diff --git a/jupyter-nersc/web/templates/error.html b/jupyter-nersc/web/templates/error.html new file mode 100644 index 0000000..25aa9a5 --- /dev/null +++ b/jupyter-nersc/web/templates/error.html @@ -0,0 +1,12 @@ +{% extends "templates/error.html" %} + +{% block error_detail %} +{% if exception and exception.my_message %} +
+

{{ exception.my_message }}

+
+{% else %} +{{ super() }} +{% endif %} + +{% endblock error_detail %} \ No newline at end of file diff --git a/jupyter-nersc/web/templates/login.html b/jupyter-nersc/web/templates/login.html new file mode 100644 index 0000000..92d57dc --- /dev/null +++ b/jupyter-nersc/web/templates/login.html @@ -0,0 +1,99 @@ +{% extends "page.html" %} {# This extends page.html, which has announcements in it. #} +{% if announcement_login %} + {% set announcement = announcement_login %} +{% endif %} + +{% block login_widget %} +{% endblock %} + +{% block main %} + +{% block login %} +
+{% if custom_html %} +{{ custom_html | safe }} +{% elif login_service %} + +{% else %} +
+
+ Sign in +
+
+ + + + {% if login_error %} + + {% endif %} + + + + + + + + + +
+
+{% endif %} +
+{% endblock login %} + +{% endblock %} + +{% block script %} +{{ super() }} + + + +{% endblock %} From c152762fa4d746f58ce4f59baed9580cf6a22d54 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 30 Nov 2018 09:37:57 -0800 Subject: [PATCH 15/41] Add config, targetting app container --- jupyter-nersc/web/jupyterhub_config.py | 983 +++++++++++++++++++++++++ 1 file changed, 983 insertions(+) create mode 100644 jupyter-nersc/web/jupyterhub_config.py diff --git a/jupyter-nersc/web/jupyterhub_config.py b/jupyter-nersc/web/jupyterhub_config.py new file mode 100644 index 0000000..193d34e --- /dev/null +++ b/jupyter-nersc/web/jupyterhub_config.py @@ -0,0 +1,983 @@ +# Configuration file for jupyterhub. + +import os +import sys + +import requests + +def comma_split(string): + """Handle env variables that may be None, empty string, or have spaces""" + try: + stripped = string.strip() + except AttributeError: + return list() + if stripped: + return [s.strip() for s in stripped.split(",")] + else: + return list() + +ip = requests.get('https://v4.ifconfig.co/json').json()['ip'] + +bindir = '/opt/anaconda3/bin/' +if 'BASE_PATH' in os.environ: + bindir = os.path.join(os.environ['BASE_PATH'], 'bin') + +#------------------------------------------------------------------------------ +# Application(SingletonConfigurable) configuration +#------------------------------------------------------------------------------ + +## This is an application. + +## The date format used by logging formatters for %(asctime)s +#c.Application.log_datefmt = '%Y-%m-%d %H:%M:%S' + +## The Logging format template +#c.Application.log_format = '[%(name)s]%(highlevel)s %(message)s' + +## Set the log level by value or name. +#c.Application.log_level = 30 + +#------------------------------------------------------------------------------ +# JupyterHub(Application) configuration +#------------------------------------------------------------------------------ + +## An Application for starting a Multi-User Jupyter Notebook server. + +## Maximum number of concurrent servers that can be active at a time. +# +# Setting this can limit the total resources your users can consume. +# +# An active server is any server that's not fully stopped. It is considered +# active from the time it has been requested until the time that it has +# completely stopped. +# +# If this many user servers are active, users will not be able to launch new +# servers until a server is shutdown. Spawn requests will be rejected with a 429 +# error asking them to try again. +# +# If set to 0, no limit is enforced. +#c.JupyterHub.active_server_limit = 0 + +## Duration (in seconds) to determine the number of active users. +#c.JupyterHub.active_user_window = 1800 + +## Grant admin users permission to access single-user servers. +# +# Users should be properly informed if this is enabled. +#c.JupyterHub.admin_access = False + +## DEPRECATED since version 0.7.2, use Authenticator.admin_users instead. +#c.JupyterHub.admin_users = set() + +## Allow named single-user servers per user +#c.JupyterHub.allow_named_servers = False + +## Answer yes to any questions (e.g. confirm overwrite) +#c.JupyterHub.answer_yes = False + +## PENDING DEPRECATION: consider using service_tokens +# +# Dict of token:username to be loaded into the database. +# +# Allows ahead-of-time generation of API tokens for use by externally managed +# services, which authenticate as JupyterHub users. +# +# Consider using service_tokens for general services that talk to the JupyterHub +# API. +#c.JupyterHub.api_tokens = {} + +## Class for authenticating users. +# +# This should be a class with the following form: +# +# - constructor takes one kwarg: `config`, the IPython config object. +# +# with an authenticate method that: +# +# - is a coroutine (asyncio or tornado) +# - returns username on success, None on failure +# - takes two arguments: (handler, data), +# where `handler` is the calling web.RequestHandler, +# and `data` is the POST form data from the login page. +#c.JupyterHub.authenticator_class = 'jupyterhub.auth.PAMAuthenticator' +c.JupyterHub.authenticator_class = 'sshapiauthenticator.auth.SSHAPIAuthenticator' + +## The base URL of the entire application. +# +# Add this to the beginning of all JupyterHub URLs. Use base_url to run +# JupyterHub within an existing website. +# +# .. deprecated: 0.9 +# Use JupyterHub.bind_url +#c.JupyterHub.base_url = '/' + +## The public facing URL of the whole JupyterHub application. +# +# This is the address on which the proxy will bind. Sets protocol, ip, base_url +#c.JupyterHub.bind_url = 'http://:8000' + +## Whether to shutdown the proxy when the Hub shuts down. +# +# Disable if you want to be able to teardown the Hub while leaving the proxy +# running. +# +# Only valid if the proxy was starting by the Hub process. +# +# If both this and cleanup_servers are False, sending SIGINT to the Hub will +# only shutdown the Hub, leaving everything else running. +# +# The Hub should be able to resume from database state. +#c.JupyterHub.cleanup_proxy = True + +## Whether to shutdown single-user servers when the Hub shuts down. +# +# Disable if you want to be able to teardown the Hub while leaving the single- +# user servers running. +# +# If both this and cleanup_proxy are False, sending SIGINT to the Hub will only +# shutdown the Hub, leaving everything else running. +# +# The Hub should be able to resume from database state. +#c.JupyterHub.cleanup_servers = True + +## Maximum number of concurrent users that can be spawning at a time. +# +# Spawning lots of servers at the same time can cause performance problems for +# the Hub or the underlying spawning system. Set this limit to prevent bursts of +# logins from attempting to spawn too many servers at the same time. +# +# This does not limit the number of total running servers. See +# active_server_limit for that. +# +# If more than this many users attempt to spawn at a time, their requests will +# be rejected with a 429 error asking them to try again. Users will have to wait +# for some of the spawning services to finish starting before they can start +# their own. +# +# If set to 0, no limit is enforced. +#c.JupyterHub.concurrent_spawn_limit = 100 + +## The config file to load +#c.JupyterHub.config_file = 'jupyterhub_config.py' + +## DEPRECATED: does nothing +#c.JupyterHub.confirm_no_ssl = False + +## Number of days for a login cookie to be valid. Default is two weeks. +#c.JupyterHub.cookie_max_age_days = 14 +c.JupyterHub.cookie_max_age_days = 0.5 + +## The cookie secret to use to encrypt cookies. +# +# Loaded from the JPY_COOKIE_SECRET env variable by default. +# +# Should be exactly 256 bits (32 bytes). +#c.JupyterHub.cookie_secret = b'' + +## File in which to store the cookie secret. +#c.JupyterHub.cookie_secret_file = 'jupyterhub_cookie_secret' + +## The location of jupyterhub data files (e.g. /usr/local/share/jupyterhub) +#c.JupyterHub.data_files_path = '/opt/anaconda3/share/jupyterhub' + +## Include any kwargs to pass to the database connection. See +# sqlalchemy.create_engine for details. +#c.JupyterHub.db_kwargs = {} + +## url for the database. e.g. `sqlite:///jupyterhub.sqlite` +#c.JupyterHub.db_url = 'sqlite:///jupyterhub.sqlite' + +## log all database transactions. This has A LOT of output +#c.JupyterHub.debug_db = False + +## DEPRECATED since version 0.8: Use ConfigurableHTTPProxy.debug +#c.JupyterHub.debug_proxy = False + +## The default URL for users when they arrive (e.g. when user directs to "/") +# +# By default, redirects users to their own server. +#c.JupyterHub.default_url = '' + +## Register extra tornado Handlers for jupyterhub. +# +# Should be of the form ``("", Handler)`` +# +# The Hub prefix will be added, so `/my-page` will be served at `/hub/my-page`. +#c.JupyterHub.extra_handlers = [] + +## DEPRECATED: use output redirection instead, e.g. +# +# jupyterhub &>> /var/log/jupyterhub.log +#c.JupyterHub.extra_log_file = '' + +## Extra log handlers to set on JupyterHub logger +#c.JupyterHub.extra_log_handlers = [] + +## Generate default config file +#c.JupyterHub.generate_config = False + +## The URL on which the Hub will listen. This is a private URL for internal +# communication. Typically set in combination with hub_connect_url. If a unix +# socket, hub_connect_url **must** also be set. +# +# For example: +# +# "http://127.0.0.1:8081" +# "unix+http://%2Fsrv%2Fjupyterhub%2Fjupyterhub.sock" +# +# .. versionadded:: 0.9 +#c.JupyterHub.hub_bind_url = '' + +## The ip or hostname for proxies and spawners to use for connecting to the Hub. +# +# Use when the bind address (`hub_ip`) is 0.0.0.0 or otherwise different from +# the connect address. +# +# Default: when `hub_ip` is 0.0.0.0, use `socket.gethostname()`, otherwise use +# `hub_ip`. +# +# Note: Some spawners or proxy implementations might not support hostnames. +# Check your spawner or proxy documentation to see if they have extra +# requirements. +# +# .. versionadded:: 0.8 +#c.JupyterHub.hub_connect_ip = '' + +## DEPRECATED +# +# Use hub_connect_url +# +# .. versionadded:: 0.8 +# +# .. deprecated:: 0.9 +# Use hub_connect_url +#c.JupyterHub.hub_connect_port = 0 + +## The URL for connecting to the Hub. Spawners, services, and the proxy will use +# this URL to talk to the Hub. +# +# Only needs to be specified if the default hub URL is not connectable (e.g. +# using a unix+http:// bind url). +# +# .. seealso:: +# JupyterHub.hub_connect_ip +# JupyterHub.hub_bind_url +# +# .. versionadded:: 0.9 +#c.JupyterHub.hub_connect_url = '' + +## The ip address for the Hub process to *bind* to. +# +# By default, the hub listens on localhost only. This address must be accessible +# from the proxy and user servers. You may need to set this to a public ip or '' +# for all interfaces if the proxy or user servers are in containers or on a +# different host. +# +# See `hub_connect_ip` for cases where the bind and connect address should +# differ, or `hub_bind_url` for setting the full bind URL. +#c.JupyterHub.hub_ip = '127.0.0.1' +c.JupyterHub.hub_ip = '0.0.0.0' + +## The internal port for the Hub process. +# +# This is the internal port of the hub itself. It should never be accessed +# directly. See JupyterHub.port for the public port to use when accessing +# jupyterhub. It is rare that this port should be set except in cases of port +# conflict. +# +# See also `hub_ip` for the ip and `hub_bind_url` for setting the full bind URL. +#c.JupyterHub.hub_port = 8081 + +## The public facing ip of the whole JupyterHub application (specifically +# referred to as the proxy). +# +# This is the address on which the proxy will listen. The default is to listen +# on all interfaces. This is the only address through which JupyterHub should be +# accessed by users. +# +# .. deprecated: 0.9 +# Use JupyterHub.bind_url +#c.JupyterHub.ip = '' + +## Supply extra arguments that will be passed to Jinja environment. +#c.JupyterHub.jinja_environment_options = {} + +## Interval (in seconds) at which to update last-activity timestamps. +#c.JupyterHub.last_activity_interval = 300 + +## Dict of 'group': ['usernames'] to load at startup. +# +# This strictly *adds* groups and users to groups. +# +# Loading one set of groups, then starting JupyterHub again with a different set +# will not remove users or groups from previous launches. That must be done +# through the API. +#c.JupyterHub.load_groups = {} + +## Specify path to a logo image to override the Jupyter logo in the banner. +#c.JupyterHub.logo_file = '' + +## File to write PID Useful for daemonizing JupyterHub. +#c.JupyterHub.pid_file = '' + +## The public facing port of the proxy. +# +# This is the port on which the proxy will listen. This is the only port through +# which JupyterHub should be accessed by users. +# +# .. deprecated: 0.9 +# Use JupyterHub.bind_url +#c.JupyterHub.port = 8000 + +## DEPRECATED since version 0.8 : Use ConfigurableHTTPProxy.api_url +#c.JupyterHub.proxy_api_ip = '' + +## DEPRECATED since version 0.8 : Use ConfigurableHTTPProxy.api_url +#c.JupyterHub.proxy_api_port = 0 + +## DEPRECATED since version 0.8: Use ConfigurableHTTPProxy.auth_token +#c.JupyterHub.proxy_auth_token = '' + +## Interval (in seconds) at which to check if the proxy is running. +#c.JupyterHub.proxy_check_interval = 30 + +## Select the Proxy API implementation. +#c.JupyterHub.proxy_class = 'jupyterhub.proxy.ConfigurableHTTPProxy' + +## DEPRECATED since version 0.8. Use ConfigurableHTTPProxy.command +#c.JupyterHub.proxy_cmd = [] + +## Redirect user to server (if running), instead of control panel. +#c.JupyterHub.redirect_to_server = True + +## Purge and reset the database. +#c.JupyterHub.reset_db = False + +## Interval (in seconds) at which to check connectivity of services with web +# endpoints. +#c.JupyterHub.service_check_interval = 60 + +## Dict of token:servicename to be loaded into the database. +# +# Allows ahead-of-time generation of API tokens for use by externally managed +# services. +#c.JupyterHub.service_tokens = {} + +## List of service specification dictionaries. +# +# A service +# +# For instance:: +# +# services = [ +# { +# 'name': 'cull_idle', +# 'command': ['/path/to/cull_idle_servers.py'], +# }, +# { +# 'name': 'formgrader', +# 'url': 'http://127.0.0.1:1234', +# 'api_token': 'super-secret', +# 'environment': +# } +# ] +#c.JupyterHub.services = [] +c.JupyterHub.services = [ + { + 'name': 'cull-idle', + 'admin': True, + 'command': 'cull_idle_servers.py --timeout=43200'.split(), + }, + { + 'name': 'announcement', + 'url': 'http://127.0.0.1:8888', + 'command': ["python", "-m", "announcement"], + } +] + +## The class to use for spawning single-user servers. +# +# Should be a subclass of Spawner. +#c.JupyterHub.spawner_class = 'jupyterhub.spawner.LocalProcessSpawner' +c.JupyterHub.spawner_class = 'sshspawner.sshspawner.SSHSpawner' + +## Path to SSL certificate file for the public facing interface of the proxy +# +# When setting this, you should also set ssl_key +#c.JupyterHub.ssl_cert = '' + +## Path to SSL key file for the public facing interface of the proxy +# +# When setting this, you should also set ssl_cert +#c.JupyterHub.ssl_key = '' + +## Host to send statsd metrics to. An empty string (the default) disables sending +# metrics. +#c.JupyterHub.statsd_host = '' + +## Port on which to send statsd metrics about the hub +#c.JupyterHub.statsd_port = 8125 + +## Prefix to use for all metrics sent by jupyterhub to statsd +#c.JupyterHub.statsd_prefix = 'jupyterhub' + +## Run single-user servers on subdomains of this host. +# +# This should be the full `https://hub.domain.tld[:port]`. +# +# Provides additional cross-site protections for javascript served by single- +# user servers. +# +# Requires `.hub.domain.tld` to resolve to the same host as +# `hub.domain.tld`. +# +# In general, this is most easily achieved with wildcard DNS. +# +# When using SSL (i.e. always) this also requires a wildcard SSL certificate. +#c.JupyterHub.subdomain_host = '' + +## Paths to search for jinja templates, before using the default templates. +#c.JupyterHub.template_paths = [] +c.JupyterHub.template_paths = ["templates"] + +## Extra variables to be passed into jinja templates +#c.JupyterHub.template_vars = {} + +## Extra settings overrides to pass to the tornado application. +#c.JupyterHub.tornado_settings = {} + +## Trust user-provided tokens (via JupyterHub.service_tokens) to have good +# entropy. +# +# If you are not inserting additional tokens via configuration file, this flag +# has no effect. +# +# In JupyterHub 0.8, internally generated tokens do not pass through additional +# hashing because the hashing is costly and does not increase the entropy of +# already-good UUIDs. +# +# User-provided tokens, on the other hand, are not trusted to have good entropy +# by default, and are passed through many rounds of hashing to stretch the +# entropy of the key (i.e. user-provided tokens are treated as passwords instead +# of random keys). These keys are more costly to check. +# +# If your inserted tokens are generated by a good-quality mechanism, e.g. +# `openssl rand -hex 32`, then you can set this flag to True to reduce the cost +# of checking authentication tokens. +#c.JupyterHub.trust_user_provided_tokens = False + +## Upgrade the database automatically on start. +# +# Only safe if database is regularly backed up. Only SQLite databases will be +# backed up to a local file automatically. +#c.JupyterHub.upgrade_db = False + +#------------------------------------------------------------------------------ +# Spawner(LoggingConfigurable) configuration +#------------------------------------------------------------------------------ + +## Base class for spawning single-user notebook servers. +# +# Subclass this, and override the following methods: +# +# - load_state - get_state - start - stop - poll +# +# As JupyterHub supports multiple users, an instance of the Spawner subclass is +# created for each user. If there are 20 JupyterHub users, there will be 20 +# instances of the subclass. + +## Extra arguments to be passed to the single-user server. +# +# Some spawners allow shell-style expansion here, allowing you to use +# environment variables here. Most, including the default, do not. Consult the +# documentation for your spawner to verify! +#c.Spawner.args = [] + +## The command used for starting the single-user server. +# +# Provide either a string or a list containing the path to the startup script +# command. Extra arguments, other than this path, should be provided via `args`. +# +# This is usually set if you want to start the single-user server in a different +# python environment (with virtualenv/conda) than JupyterHub itself. +# +# Some spawners allow shell-style expansion here, allowing you to use +# environment variables. Most, including the default, do not. Consult the +# documentation for your spawner to verify! +#c.Spawner.cmd = ['jupyterhub-singleuser'] +c.Spawner.cmd = [os.path.join(bindir, 'jupyter-labhub')] + +## Maximum number of consecutive failures to allow before shutting down +# JupyterHub. +# +# This helps JupyterHub recover from a certain class of problem preventing +# launch in contexts where the Hub is automatically restarted (e.g. systemd, +# docker, kubernetes). +# +# A limit of 0 means no limit and consecutive failures will not be tracked. +#c.Spawner.consecutive_failure_limit = 0 + +## Minimum number of cpu-cores a single-user notebook server is guaranteed to +# have available. +# +# If this value is set to 0.5, allows use of 50% of one CPU. If this value is +# set to 2, allows use of up to 2 CPUs. +# +# **This is a configuration setting. Your spawner must implement support for the +# limit to work.** The default spawner, `LocalProcessSpawner`, does **not** +# implement this support. A custom spawner **must** add support for this setting +# for it to be enforced. +#c.Spawner.cpu_guarantee = None + +## Maximum number of cpu-cores a single-user notebook server is allowed to use. +# +# If this value is set to 0.5, allows use of 50% of one CPU. If this value is +# set to 2, allows use of up to 2 CPUs. +# +# The single-user notebook server will never be scheduled by the kernel to use +# more cpu-cores than this. There is no guarantee that it can access this many +# cpu-cores. +# +# **This is a configuration setting. Your spawner must implement support for the +# limit to work.** The default spawner, `LocalProcessSpawner`, does **not** +# implement this support. A custom spawner **must** add support for this setting +# for it to be enforced. +#c.Spawner.cpu_limit = None + +## Enable debug-logging of the single-user server +#c.Spawner.debug = False + +## The URL the single-user server should start in. +# +# `{username}` will be expanded to the user's username +# +# Example uses: +# +# - You can set `notebook_dir` to `/` and `default_url` to `/tree/home/{username}` to allow people to +# navigate the whole filesystem from their notebook server, but still start in their home directory. +# - Start with `/notebooks` instead of `/tree` if `default_url` points to a notebook instead of a directory. +# - You can set this to `/lab` to have JupyterLab start by default, rather than Jupyter Notebook. +#c.Spawner.default_url = '' +c.Spawner.default_url = '/lab/tree/global/homes/{username[0]}/{username}' + +## Disable per-user configuration of single-user servers. +# +# When starting the user's single-user server, any config file found in the +# user's $HOME directory will be ignored. +# +# Note: a user could circumvent this if the user modifies their Python +# environment, such as when they have their own conda environments / virtualenvs +# / containers. +#c.Spawner.disable_user_config = False + +## Whitelist of environment variables for the single-user server to inherit from +# the JupyterHub process. +# +# This whitelist is used to ensure that sensitive information in the JupyterHub +# process's environment (such as `CONFIGPROXY_AUTH_TOKEN`) is not passed to the +# single-user server's process. +#c.Spawner.env_keep = ['PATH', 'PYTHONPATH', 'CONDA_ROOT', 'CONDA_DEFAULT_ENV', 'VIRTUAL_ENV', 'LANG', 'LC_ALL'] + +## Extra environment variables to set for the single-user server's process. +# +# Environment variables that end up in the single-user server's process come from 3 sources: +# - This `environment` configurable +# - The JupyterHub process' environment variables that are whitelisted in `env_keep` +# - Variables to establish contact between the single-user notebook and the hub (such as JUPYTERHUB_API_TOKEN) +# +# The `environment` configurable should be set by JupyterHub administrators to +# add installation specific environment variables. It is a dict where the key is +# the name of the environment variable, and the value can be a string or a +# callable. If it is a callable, it will be called with one parameter (the +# spawner instance), and should return a string fairly quickly (no blocking +# operations please!). +# +# Note that the spawner class' interface is not guaranteed to be exactly same +# across upgrades, so if you are using the callable take care to verify it +# continues to work after upgrades! +#c.Spawner.environment = {} +c.Spawner.environment = {"OMP_NUM_THREADS" : "2"} + +## Timeout (in seconds) before giving up on a spawned HTTP server +# +# Once a server has successfully been spawned, this is the amount of time we +# wait before assuming that the server is unable to accept connections. +#c.Spawner.http_timeout = 30 + +## The IP address (or hostname) the single-user server should listen on. +# +# The JupyterHub proxy implementation should be able to send packets to this +# interface. +#c.Spawner.ip = '' +c.Spawner.ip = '0.0.0.0' + +## Minimum number of bytes a single-user notebook server is guaranteed to have +# available. +# +# Allows the following suffixes: +# - K -> Kilobytes +# - M -> Megabytes +# - G -> Gigabytes +# - T -> Terabytes +# +# **This is a configuration setting. Your spawner must implement support for the +# limit to work.** The default spawner, `LocalProcessSpawner`, does **not** +# implement this support. A custom spawner **must** add support for this setting +# for it to be enforced. +#c.Spawner.mem_guarantee = None + +## Maximum number of bytes a single-user notebook server is allowed to use. +# +# Allows the following suffixes: +# - K -> Kilobytes +# - M -> Megabytes +# - G -> Gigabytes +# - T -> Terabytes +# +# If the single user server tries to allocate more memory than this, it will +# fail. There is no guarantee that the single-user notebook server will be able +# to allocate this much memory - only that it can not allocate more than this. +# +# **This is a configuration setting. Your spawner must implement support for the +# limit to work.** The default spawner, `LocalProcessSpawner`, does **not** +# implement this support. A custom spawner **must** add support for this setting +# for it to be enforced. +#c.Spawner.mem_limit = None + +## Path to the notebook directory for the single-user server. +# +# The user sees a file listing of this directory when the notebook interface is +# started. The current interface does not easily allow browsing beyond the +# subdirectories in this directory's tree. +# +# `~` will be expanded to the home directory of the user, and {username} will be +# replaced with the name of the user. +# +# Note that this does *not* prevent users from accessing files outside of this +# path! They can do so with many other means. +#c.Spawner.notebook_dir = '' +c.Spawner.notebook_dir = '/' + +## An HTML form for options a user can specify on launching their server. +# +# The surrounding `
` element and the submit button are already provided. +# +# For example: +# +# .. code:: html +# +# Set your key: +# +#
+# Choose a letter: +# +# +# The data from this form submission will be passed on to your spawner in +# `self.user_options` +# +# Instead of a form snippet string, this could also be a callable that takes as +# one parameter the current spawner instance and returns a string. The callable +# will be called asynchronously if it returns a future, rather than a str. Note +# that the interface of the spawner class is not deemed stable across versions, +# so using this functionality might cause your JupyterHub upgrades to break. +#c.Spawner.options_form = traitlets.Undefined + +## Interval (in seconds) on which to poll the spawner for single-user server's +# status. +# +# At every poll interval, each spawner's `.poll` method is called, which checks +# if the single-user server is still running. If it isn't running, then +# JupyterHub modifies its own state accordingly and removes appropriate routes +# from the configurable proxy. +#c.Spawner.poll_interval = 30 +c.Spawner.poll_interval = 900 + +## The port for single-user servers to listen on. +# +# Defaults to `0`, which uses a randomly allocated port number each time. +# +# If set to a non-zero value, all Spawners will use the same port, which only +# makes sense if each server is on a different address, e.g. in containers. +# +# New in version 0.7. +#c.Spawner.port = 0 + +## An optional hook function that you can implement to do work after the spawner +# stops. +# +# This can be set independent of any concrete spawner implementation. +#c.Spawner.post_stop_hook = None + +## An optional hook function that you can implement to do some bootstrapping work +# before the spawner starts. For example, create a directory for your user or +# load initial content. +# +# This can be set independent of any concrete spawner implementation. +# +# Example:: +# +# from subprocess import check_call +# def my_hook(spawner): +# username = spawner.user.name +# check_call(['./examples/bootstrap-script/bootstrap.sh', username]) +# +# c.Spawner.pre_spawn_hook = my_hook +#c.Spawner.pre_spawn_hook = None + +## Timeout (in seconds) before giving up on starting of single-user server. +# +# This is the timeout for start to return, not the timeout for the server to +# respond. Callers of spawner.start will assume that startup has failed if it +# takes longer than this. start should return when the server process is started +# and its location is known. +#c.Spawner.start_timeout = 60 + +#------------------------------------------------------------------------------ +# LocalProcessSpawner(Spawner) configuration +#------------------------------------------------------------------------------ + +## A Spawner that uses `subprocess.Popen` to start single-user servers as local +# processes. +# +# Requires local UNIX users matching the authenticated users to exist. Does not +# work on Windows. +# +# This is the default spawner for JupyterHub. +# +# Note: This spawner does not implement CPU / memory guarantees and limits. + +## Seconds to wait for single-user server process to halt after SIGINT. +# +# If the process has not exited cleanly after this many seconds, a SIGTERM is +# sent. +#c.LocalProcessSpawner.interrupt_timeout = 10 + +## Seconds to wait for process to halt after SIGKILL before giving up. +# +# If the process does not exit cleanly after this many seconds of SIGKILL, it +# becomes a zombie process. The hub process will log a warning and then give up. +#c.LocalProcessSpawner.kill_timeout = 5 + +## Extra keyword arguments to pass to Popen +# +# when spawning single-user servers. +# +# For example:: +# +# popen_kwargs = dict(shell=True) +#c.LocalProcessSpawner.popen_kwargs = {} + +## Specify a shell command to launch. +# +# The single-user command will be appended to this list, so it sould end with +# `-c` (for bash) or equivalent. +# +# For example:: +# +# c.LocalProcessSpawner.shell_cmd = ['bash', '-l', '-c'] +# +# to launch with a bash login shell, which would set up the user's own complete +# environment. +# +# .. warning:: +# +# Using shell_cmd gives users control over PATH, etc., +# which could change what the jupyterhub-singleuser launch command does. +# Only use this for trusted users. +#c.LocalProcessSpawner.shell_cmd = [] + +## Seconds to wait for single-user server process to halt after SIGTERM. +# +# If the process does not exit cleanly after this many seconds of SIGTERM, a +# SIGKILL is sent. +#c.LocalProcessSpawner.term_timeout = 5 + +#------------------------------------------------------------------------------ +# Authenticator(LoggingConfigurable) configuration +#------------------------------------------------------------------------------ + +## Base class for implementing an authentication provider for JupyterHub + +## Set of users that will have admin rights on this JupyterHub. +# +# Admin users have extra privileges: +# - Use the admin panel to see list of users logged in +# - Add / remove users in some authenticators +# - Restart / halt the hub +# - Start / stop users' single-user servers +# - Can access each individual users' single-user server (if configured) +# +# Admin access should be treated the same way root access is. +# +# Defaults to an empty set, in which case no user has admin access. +#c.Authenticator.admin_users = set() +c.Authenticator.admin_users = set(comma_split(os.environ.get("ADMINS"))) + +## Automatically begin the login process +# +# rather than starting with a "Login with..." link at `/hub/login` +# +# To work, `.login_url()` must give a URL other than the default `/hub/login`, +# such as an oauth handler or another automatic login handler, registered with +# `.get_handlers()`. +# +# .. versionadded:: 0.8 +#c.Authenticator.auto_login = False + +## Blacklist of usernames that are not allowed to log in. +# +# Use this with supported authenticators to restrict which users can not log in. +# This is an additional blacklist that further restricts users, beyond whatever +# restrictions the authenticator has in place. +# +# If empty, does not perform any additional restriction. +# +# .. versionadded: 0.9 +#c.Authenticator.blacklist = set() + +## Enable persisting auth_state (if available). +# +# auth_state will be encrypted and stored in the Hub's database. This can +# include things like authentication tokens, etc. to be passed to Spawners as +# environment variables. +# +# Encrypting auth_state requires the cryptography package. +# +# Additionally, the JUPYTERHUB_CRYPT_KEY environment variable must contain one +# (or more, separated by ;) 32B encryption keys. These can be either base64 or +# hex-encoded. +# +# If encryption is unavailable, auth_state cannot be persisted. +# +# New in JupyterHub 0.8 +#c.Authenticator.enable_auth_state = False + +## Dictionary mapping authenticator usernames to JupyterHub users. +# +# Primarily used to normalize OAuth user names to local users. +#c.Authenticator.username_map = {} + +## Regular expression pattern that all valid usernames must match. +# +# If a username does not match the pattern specified here, authentication will +# not be attempted. +# +# If not set, allow any username. +#c.Authenticator.username_pattern = '' + +## Whitelist of usernames that are allowed to log in. +# +# Use this with supported authenticators to restrict which users can log in. +# This is an additional whitelist that further restricts users, beyond whatever +# restrictions the authenticator has in place. +# +# If empty, does not perform any additional restriction. +#c.Authenticator.whitelist = set() +c.Authenticator.whitelist = set(comma_split(os.environ.get("WHITELIST"))) + +#------------------------------------------------------------------------------ +# LocalAuthenticator(Authenticator) configuration +#------------------------------------------------------------------------------ + +## Base class for Authenticators that work with local Linux/UNIX users +# +# Checks for local users, and can attempt to create them if they exist. + +## The command to use for creating users as a list of strings +# +# For each element in the list, the string USERNAME will be replaced with the +# user's username. The username will also be appended as the final argument. +# +# For Linux, the default value is: +# +# ['adduser', '-q', '--gecos', '""', '--disabled-password'] +# +# To specify a custom home directory, set this to: +# +# ['adduser', '-q', '--gecos', '""', '--home', '/customhome/USERNAME', '-- +# disabled-password'] +# +# This will run the command: +# +# adduser -q --gecos "" --home /customhome/river --disabled-password river +# +# when the user 'river' is created. +#c.LocalAuthenticator.add_user_cmd = [] + +## If set to True, will attempt to create local system users if they do not exist +# already. +# +# Supports Linux and BSD variants only. +#c.LocalAuthenticator.create_system_users = False + +## Whitelist all users from this UNIX group. +# +# This makes the username whitelist ineffective. +#c.LocalAuthenticator.group_whitelist = set() + +#------------------------------------------------------------------------------ +# PAMAuthenticator(LocalAuthenticator) configuration +#------------------------------------------------------------------------------ + +## Authenticate local UNIX users with PAM + +## Whether to check the user's account status via PAM during authentication. +# +# The PAM account stack performs non-authentication based account management. +# It is typically used to restrict/permit access to a service and this step is +# needed to access the host's user access control. +# +# Disabling this can be dangerous as authenticated but unauthorized users may be +# granted access and, therefore, arbitrary execution on the system. +#c.PAMAuthenticator.check_account = True + +## The text encoding to use when communicating with PAM +#c.PAMAuthenticator.encoding = 'utf8' + +## Whether to open a new PAM session when spawners are started. +# +# This may trigger things like mounting shared filsystems, loading credentials, +# etc. depending on system configuration, but it does not always work. +# +# If any errors are encountered when opening/closing PAM sessions, this is +# automatically set to False. +#c.PAMAuthenticator.open_sessions = True + +## The name of the PAM service to use for authentication +#c.PAMAuthenticator.service = 'login' + +#------------------------------------------------------------------------------ +# CryptKeeper(SingletonConfigurable) configuration +#------------------------------------------------------------------------------ + +## Encapsulate encryption configuration +# +# Use via the encryption_config singleton below. + +## +#c.CryptKeeper.keys = [] + +## The number of threads to allocate for encryption +#c.CryptKeeper.n_threads = 2 + +#------------------------------------------------------------------------------ +# SSHAPIAuthenticator(Authenticator) configuration +#------------------------------------------------------------------------------ + +c.SSHAPIAuthenticator.server = 'https://sshproxy.nersc.gov/create_pair/jupyter/' +c.SSHAPIAuthenticator.skey = os.environ.get('SKEY') +c.SSHAPIAuthenticator.cert_path = '/certs' + +#------------------------------------------------------------------------------ +# SSHSpawner(Spawner) configuration +#------------------------------------------------------------------------------ + +c.SSHSpawner.remote_hosts = ['app'] +c.SSHSpawner.remote_port = '22' +c.SSHSpawner.hub_api_url = "http://{}:8081/hub/api".format(ip) +c.SSHSpawner.path = bindir + ':/usr/bin:/bin' +c.SSHSpawner.remote_port_command = bindir + 'python /opt/anaconda3/bin/get_port.py' +c.SSHSpawner.ssh_keyfile = '/certs/{username}.key' From 9a26f93bb4c80203794e91dd1fc34309892c779f Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 30 Nov 2018 12:16:44 -0800 Subject: [PATCH 16/41] Target container --- jupyter-nersc/app/Dockerfile | 70 ++++++++++++++++++++++++++ jupyter-nersc/app/NERSC-keys-api | 7 +++ jupyter-nersc/app/docker-entrypoint.sh | 5 ++ jupyter-nersc/app/get_port.py | 8 +++ jupyter-nersc/app/packages3.txt | 10 ++++ 5 files changed, 100 insertions(+) create mode 100644 jupyter-nersc/app/Dockerfile create mode 100644 jupyter-nersc/app/NERSC-keys-api create mode 100644 jupyter-nersc/app/docker-entrypoint.sh create mode 100644 jupyter-nersc/app/get_port.py create mode 100644 jupyter-nersc/app/packages3.txt diff --git a/jupyter-nersc/app/Dockerfile b/jupyter-nersc/app/Dockerfile new file mode 100644 index 0000000..f7186a4 --- /dev/null +++ b/jupyter-nersc/app/Dockerfile @@ -0,0 +1,70 @@ +ARG branch=unknown + +FROM registry.spin.nersc.gov/das/jupyter-base.${branch}:latest +LABEL maintainer="Rollin Thomas " + +# Additional Ubuntu packages +# TODO: resolve zsh thing that Rollin ran into b/c datatran + +RUN \ + apt-get --yes install \ + ldap-utils \ + libnss-ldapd \ + libpam-ldap \ + nscd \ + openssh-server \ + zsh + +# Python 3 Anaconda and additional packages + +ADD packages3.txt /tmp/packages3.txt +RUN \ + /opt/anaconda3/bin/conda update --yes conda && \ + /opt/anaconda3/bin/conda install --yes anaconda && \ + /opt/anaconda3/bin/conda install --file /tmp/packages3.txt && \ + /opt/anaconda3/bin/ipython kernel install && \ + /opt/anaconda3/bin/conda clean --yes --all + +### # Python 2 Anaconda and additional packages +### +### ADD packages2.txt /tmp/packages2.txt +### RUN \ +### wget -q -O /tmp/miniconda2.sh https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh && \ +### bash /tmp/miniconda2.sh -f -b -p /opt/anaconda2 && \ +### rm /tmp/miniconda2.sh && \ +### /opt/anaconda2/bin/conda update --yes conda && \ +### /opt/anaconda2/bin/conda install --yes anaconda && \ +### /opt/anaconda2/bin/conda install --file /tmp/packages2.txt && \ +### /opt/anaconda2/bin/ipython kernel install && \ +### /opt/anaconda2/bin/conda clean --yes --all + +# SSHD + +RUN \ + mkdir -p /var/run/sshd && \ + echo "AuthorizedKeysCommand /usr/lib/nersc-ssh-keys/NERSC-keys-api" >> /etc/ssh/sshd_config && \ + echo "AuthorizedKeysCommandUser nobody" >> /etc/ssh/sshd_config && \ + echo "TrustedUserCAKeys /etc/user_ca.pub" >> /etc/ssh/sshd_config + +# PAM, LDAP + +COPY etc/ /etc/ + +# GPFS + +RUN \ + mkdir /global && \ + ln -sf /global/u1 /global/homes && \ + ln -sf /global/project /project && \ + ln -s /global/common/datatran /usr/common && \ + echo "datatran" > /etc/clustername + +ADD docker-entrypoint.sh /srv/ +ADD NERSC-keys-api /usr/lib/nersc-ssh-keys/ +ADD get_port.py /opt/anaconda3/bin/ +RUN chmod a+x /usr/lib/nersc-ssh-keys/NERSC-keys-api + +WORKDIR /srv +RUN chmod +x docker-entrypoint.sh +ENTRYPOINT [ "./docker-entrypoint.sh" ] +CMD [ "/usr/sbin/sshd", "-p", "22", "-D" ] diff --git a/jupyter-nersc/app/NERSC-keys-api b/jupyter-nersc/app/NERSC-keys-api new file mode 100644 index 0000000..6718bd5 --- /dev/null +++ b/jupyter-nersc/app/NERSC-keys-api @@ -0,0 +1,7 @@ +#!/bin/sh + +URL=https://sshproxy.nersc.gov/get_keys +USER=$1 + + +curl $URL/$USER diff --git a/jupyter-nersc/app/docker-entrypoint.sh b/jupyter-nersc/app/docker-entrypoint.sh new file mode 100644 index 0000000..6e500b8 --- /dev/null +++ b/jupyter-nersc/app/docker-entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +service nslcd start + +exec "$@" diff --git a/jupyter-nersc/app/get_port.py b/jupyter-nersc/app/get_port.py new file mode 100644 index 0000000..ab26463 --- /dev/null +++ b/jupyter-nersc/app/get_port.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python + +import socket + +sock = socket.socket() +sock.bind(('', 0)) +print(sock.getsockname()[1]) +sock.close() diff --git a/jupyter-nersc/app/packages3.txt b/jupyter-nersc/app/packages3.txt new file mode 100644 index 0000000..3bf09a0 --- /dev/null +++ b/jupyter-nersc/app/packages3.txt @@ -0,0 +1,10 @@ +astropy +basemap +biopython +blaze +jupyter +jupyterlab +netcdf4 +#r-essentials +rpy2 +ujson From 217ef2aedce0f6e4d3f6d27c54558a55eaef2871 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 30 Nov 2018 12:24:05 -0800 Subject: [PATCH 17/41] Adding configs --- jupyter-nersc/app/etc/ldap.conf | 12 +++++++++++ jupyter-nersc/app/etc/nslcd.conf | 23 ++++++++++++++++++++ jupyter-nersc/app/etc/nsswitch.conf | 21 ++++++++++++++++++ jupyter-nersc/app/etc/pam.d/jupyterhub | 13 ++++++++++++ jupyter-nersc/app/etc/pam.d/password-auth | 26 +++++++++++++++++++++++ jupyter-nersc/app/etc/user_ca.pub | 1 + 6 files changed, 96 insertions(+) create mode 100644 jupyter-nersc/app/etc/ldap.conf create mode 100644 jupyter-nersc/app/etc/nslcd.conf create mode 100644 jupyter-nersc/app/etc/nsswitch.conf create mode 100644 jupyter-nersc/app/etc/pam.d/jupyterhub create mode 100644 jupyter-nersc/app/etc/pam.d/password-auth create mode 100644 jupyter-nersc/app/etc/user_ca.pub diff --git a/jupyter-nersc/app/etc/ldap.conf b/jupyter-nersc/app/etc/ldap.conf new file mode 100644 index 0000000..39af084 --- /dev/null +++ b/jupyter-nersc/app/etc/ldap.conf @@ -0,0 +1,12 @@ +# +# LDAP Defaults +# + +# See ldap.conf(5) for details +# This file should be world readable but not world writable. + + +URI ldaps://ldapcrt.nersc.gov +BASE ou=datatran,ou=Host,o=ldapsvc,dc=nersc,dc=gov +TLS_CACERTDIR /etc/openldap/cacerts +TLS_REQCERT never diff --git a/jupyter-nersc/app/etc/nslcd.conf b/jupyter-nersc/app/etc/nslcd.conf new file mode 100644 index 0000000..ba8efe6 --- /dev/null +++ b/jupyter-nersc/app/etc/nslcd.conf @@ -0,0 +1,23 @@ +# $Id: nslcd.conf.tmpl 175 2012-10-24 23:13:43Z nsgsrc $ + +tls_reqcert never +uid nslcd +gid nslcd +uri ldaps://ldapcrt.nersc.gov/ +base ou=datatran,ou=host,o=ldapsvc,dc=nersc,dc=gov +ssl on + +# Fix regex to allow for two-letter names +# See: https://bugzilla.redhat.com/show_bug.cgi?id=706860 +validnames /^[a-z0-9._@$()][a-z0-9._@$() \~-]*[a-z0-9._@$()~-]$/i + +tls_cacertdir /etc/openldap/cacerts + +# Don't try to resolve any users in the local user database +nss_initgroups_ignoreusers ALLLOCAL + +# Set idle_timelimit to 4 minutes to avoid "can't contact LDAP server" log +# messages +idle_timelimit 240 + +filter passwd (&(objectClass=posixAccount)(authorizedService=ssh)) diff --git a/jupyter-nersc/app/etc/nsswitch.conf b/jupyter-nersc/app/etc/nsswitch.conf new file mode 100644 index 0000000..3d77354 --- /dev/null +++ b/jupyter-nersc/app/etc/nsswitch.conf @@ -0,0 +1,21 @@ +passwd: files ldap +shadow: files ldap +group: files ldap + +hosts: files dns + +bootparams: nisplus [NOTFOUND=return] files + +ethers: files +netmasks: files +networks: files +protocols: files +rpc: files +services: files + +netgroup: nisplus + +publickey: nisplus + +automount: files nisplus +aliases: files nisplus diff --git a/jupyter-nersc/app/etc/pam.d/jupyterhub b/jupyter-nersc/app/etc/pam.d/jupyterhub new file mode 100644 index 0000000..a8a86f4 --- /dev/null +++ b/jupyter-nersc/app/etc/pam.d/jupyterhub @@ -0,0 +1,13 @@ +#%PAM-1.0 +auth required pam_sepermit.so +auth include password-auth +account required pam_nologin.so +account include password-auth +password include password-auth +# pam_selinux.so close should be the first session rule +session required pam_selinux.so close +session optional pam_loginuid.so +# pam_selinux.so open should only be followed by sessions to be executed in the user context +session required pam_selinux.so open env_params +session optional pam_keyinit.so force revoke +session include password-auth diff --git a/jupyter-nersc/app/etc/pam.d/password-auth b/jupyter-nersc/app/etc/pam.d/password-auth new file mode 100644 index 0000000..87cf4e7 --- /dev/null +++ b/jupyter-nersc/app/etc/pam.d/password-auth @@ -0,0 +1,26 @@ +#%PAM-1.0 + +# $Id: password-auth.tmpl 668 2013-11-01 20:00:51Z nsgsrc $ + +auth required pam_env.so +auth sufficient pam_unix.so nullok try_first_pass +auth requisite pam_succeed_if.so uid >= 400 quiet +auth sufficient pam_ldap.so use_first_pass +auth required pam_deny.so + +account required pam_access.so +account required pam_unix.so broken_shadow +account sufficient pam_localuser.so +account sufficient pam_succeed_if.so uid < 400 quiet +account [default=bad success=ok user_unknown=ignore] pam_ldap.so +account required pam_permit.so + +password requisite pam_cracklib.so try_first_pass retry=3 type= +password sufficient pam_unix.so sha512 shadow nullok try_first_pass use_authtok +password sufficient pam_ldap.so use_authtok +password required pam_deny.so + +session optional pam_keyinit.so revoke +session required pam_limits.so +session required pam_unix.so +session optional pam_ldap.so diff --git a/jupyter-nersc/app/etc/user_ca.pub b/jupyter-nersc/app/etc/user_ca.pub new file mode 100644 index 0000000..ff89c20 --- /dev/null +++ b/jupyter-nersc/app/etc/user_ca.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA2yKBpvRbdD9MWiu+7wg17vBsKy46AjjuL27DmdpDYiCqRE2mN0om9b0jn4eI91RGykbcRa9wUKJ2qaD0zsD08A8HM+R14H4UsZ5hi7S+xGqscJH7uTmXy5Igo5xEOahS9Z+ecgonDCgWKJnbd/FRu4vITYXrvTlIIGHGRBYj0GzbgLHBzedoMaGNRwhVyadH2SGRaZCgbH+Swevzy0GwYfZJA9zd7EX0jiAClkSYcflIOsygmI3gHv+b35mrvXcHDeQOR/wg8knfpSiFLCkVDpfgnj27Lemzxe6k61Brhv9CUiq+t7WApVDBovhdXZn6pBg+OKeDk1G1OLvRbxJ2bw== dunford@muppet.nersc.gov From a70baf180c177f886baacde628b71052740fcb73 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 30 Nov 2018 12:34:23 -0800 Subject: [PATCH 18/41] Build helper --- jupyter-nersc/app/build.sh | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 jupyter-nersc/app/build.sh diff --git a/jupyter-nersc/app/build.sh b/jupyter-nersc/app/build.sh new file mode 100644 index 0000000..7457d56 --- /dev/null +++ b/jupyter-nersc/app/build.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +branch=$(git symbolic-ref --short HEAD) + +docker build \ + --build-arg branch=$branch \ + --no-cache \ + --tag registry.spin.nersc.gov/das/jupyter-nersc-app.$branch:latest . From f535ef30b22ce21273e05ef19a73e6f771547f1b Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 30 Nov 2018 12:57:49 -0800 Subject: [PATCH 19/41] Version for testing --- jupyter-nersc/app/Dockerfile | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/jupyter-nersc/app/Dockerfile b/jupyter-nersc/app/Dockerfile index f7186a4..17f3fbb 100644 --- a/jupyter-nersc/app/Dockerfile +++ b/jupyter-nersc/app/Dockerfile @@ -17,14 +17,32 @@ RUN \ # Python 3 Anaconda and additional packages -ADD packages3.txt /tmp/packages3.txt +### TESTING ... + RUN \ - /opt/anaconda3/bin/conda update --yes conda && \ - /opt/anaconda3/bin/conda install --yes anaconda && \ - /opt/anaconda3/bin/conda install --file /tmp/packages3.txt && \ - /opt/anaconda3/bin/ipython kernel install && \ + /opt/anaconda3/bin/conda update --yes conda && \ + /opt/anaconda3/bin/conda install --yes \ + ipykernel \ + ipywidgets \ + notebook && \ + /opt/anaconda3/bin/ipython kernel install && \ /opt/anaconda3/bin/conda clean --yes --all +# Typical extension + +RUN \ + /opt/anaconda3/bin/jupyter nbextension enable --sys-prefix --py widgetsnbextension + +### TESTING DONE ... + +### ADD packages3.txt /tmp/packages3.txt +### RUN \ +### /opt/anaconda3/bin/conda update --yes conda && \ +### /opt/anaconda3/bin/conda install --yes anaconda && \ +### /opt/anaconda3/bin/conda install --file /tmp/packages3.txt && \ +### /opt/anaconda3/bin/ipython kernel install && \ +### /opt/anaconda3/bin/conda clean --yes --all + ### # Python 2 Anaconda and additional packages ### ### ADD packages2.txt /tmp/packages2.txt From 01e8b36146ed1eb12430dfc514c7db4f46e5a4bb Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 30 Nov 2018 22:16:07 -0800 Subject: [PATCH 20/41] Pin to 3.6 --- jupyter-base/Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/jupyter-base/Dockerfile b/jupyter-base/Dockerfile index 18d40a7..cacd415 100644 --- a/jupyter-base/Dockerfile +++ b/jupyter-base/Dockerfile @@ -30,10 +30,11 @@ RUN \ RUN \ curl -s -o /tmp/miniconda3.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ - bash /tmp/miniconda3.sh -f -b -p /opt/anaconda3 && \ - rm -rf /tmp/anaconda3.sh && \ - /opt/anaconda3/bin/conda update --yes conda && \ - /opt/anaconda3/bin/conda install --yes \ + bash /tmp/miniconda3.sh -f -b -p /opt/anaconda3 && \ + rm -rf /tmp/miniconda3.sh && \ + echo "python 3.6.*" >> /opt/anaconda3/conda-meta/pinned && \ + /opt/anaconda3/bin/conda update --yes conda && \ + /opt/anaconda3/bin/conda install --yes \ alembic \ decorator \ jinja2 \ From 24a9b3db8df11a22ed33ff47b2b5247eb16de493 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 30 Nov 2018 22:16:32 -0800 Subject: [PATCH 21/41] Obviously need lab! --- jupyter-nersc/app/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/jupyter-nersc/app/Dockerfile b/jupyter-nersc/app/Dockerfile index 17f3fbb..fb4b632 100644 --- a/jupyter-nersc/app/Dockerfile +++ b/jupyter-nersc/app/Dockerfile @@ -24,6 +24,7 @@ RUN \ /opt/anaconda3/bin/conda install --yes \ ipykernel \ ipywidgets \ + jupyterlab \ notebook && \ /opt/anaconda3/bin/ipython kernel install && \ /opt/anaconda3/bin/conda clean --yes --all From 1acade97fa8217f2b04ea6badfe7f7ba013601a8 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 30 Nov 2018 22:16:46 -0800 Subject: [PATCH 22/41] Fix up path --- jupyter-nersc/web/jupyterhub_config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jupyter-nersc/web/jupyterhub_config.py b/jupyter-nersc/web/jupyterhub_config.py index 193d34e..cb086c1 100644 --- a/jupyter-nersc/web/jupyterhub_config.py +++ b/jupyter-nersc/web/jupyterhub_config.py @@ -186,6 +186,9 @@ def comma_split(string): ## url for the database. e.g. `sqlite:///jupyterhub.sqlite` #c.JupyterHub.db_url = 'sqlite:///jupyterhub.sqlite' +c.JupyterHub.db_url = 'postgresql://jupyterhub:{}@db:5432/jupyterhub'.format( + os.getenv('POSTGRES_PASSWORD') +) ## log all database transactions. This has A LOT of output #c.JupyterHub.debug_db = False @@ -978,6 +981,6 @@ def comma_split(string): c.SSHSpawner.remote_hosts = ['app'] c.SSHSpawner.remote_port = '22' c.SSHSpawner.hub_api_url = "http://{}:8081/hub/api".format(ip) -c.SSHSpawner.path = bindir + ':/usr/bin:/bin' +c.SSHSpawner.path = bindir + ':/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' c.SSHSpawner.remote_port_command = bindir + 'python /opt/anaconda3/bin/get_port.py' c.SSHSpawner.ssh_keyfile = '/certs/{username}.key' From 2fbb1402b235fae270549810d8e139045ff37fe8 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 30 Nov 2018 22:19:57 -0800 Subject: [PATCH 23/41] Let --no-cache etc be set on build line --- jupyter-nersc/web/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter-nersc/web/build.sh b/jupyter-nersc/web/build.sh index b72f6ff..b136616 100644 --- a/jupyter-nersc/web/build.sh +++ b/jupyter-nersc/web/build.sh @@ -4,5 +4,5 @@ branch=$(git symbolic-ref --short HEAD) docker build \ --build-arg branch=$branch \ - --no-cache \ + "$@" \ --tag registry.spin.nersc.gov/das/jupyter-nersc-web.$branch:latest . From e19f31094df6e8f398a36f223f5e89eb772f4323 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Fri, 30 Nov 2018 22:20:26 -0800 Subject: [PATCH 24/41] Add args for --no-cache etc --- jupyter-nersc/app/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter-nersc/app/build.sh b/jupyter-nersc/app/build.sh index 7457d56..b408866 100644 --- a/jupyter-nersc/app/build.sh +++ b/jupyter-nersc/app/build.sh @@ -4,5 +4,5 @@ branch=$(git symbolic-ref --short HEAD) docker build \ --build-arg branch=$branch \ - --no-cache \ + "$@" \ --tag registry.spin.nersc.gov/das/jupyter-nersc-app.$branch:latest . From d99758818075c2262df5b01f55e48817fd35d211 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Sun, 2 Dec 2018 15:56:42 -0800 Subject: [PATCH 25/41] Supervisor chages --- jupyter-nersc/app/Dockerfile | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/jupyter-nersc/app/Dockerfile b/jupyter-nersc/app/Dockerfile index fb4b632..a9669a6 100644 --- a/jupyter-nersc/app/Dockerfile +++ b/jupyter-nersc/app/Dockerfile @@ -2,9 +2,9 @@ ARG branch=unknown FROM registry.spin.nersc.gov/das/jupyter-base.${branch}:latest LABEL maintainer="Rollin Thomas " +WORKDIR /srv # Additional Ubuntu packages -# TODO: resolve zsh thing that Rollin ran into b/c datatran RUN \ apt-get --yes install \ @@ -13,7 +13,7 @@ RUN \ libpam-ldap \ nscd \ openssh-server \ - zsh + supervisor # Python 3 Anaconda and additional packages @@ -57,7 +57,12 @@ RUN \ ### /opt/anaconda2/bin/ipython kernel install && \ ### /opt/anaconda2/bin/conda clean --yes --all -# SSHD +# For ssh auth API + +ADD NERSC-keys-api /usr/lib/nersc-ssh-keys/ +RUN chmod a+x /usr/lib/nersc-ssh-keys/NERSC-keys-api + +# For sshd RUN \ mkdir -p /var/run/sshd && \ @@ -65,7 +70,7 @@ RUN \ echo "AuthorizedKeysCommandUser nobody" >> /etc/ssh/sshd_config && \ echo "TrustedUserCAKeys /etc/user_ca.pub" >> /etc/ssh/sshd_config -# PAM, LDAP +# For PAM/LDAP COPY etc/ /etc/ @@ -78,12 +83,11 @@ RUN \ ln -s /global/common/datatran /usr/common && \ echo "datatran" > /etc/clustername -ADD docker-entrypoint.sh /srv/ -ADD NERSC-keys-api /usr/lib/nersc-ssh-keys/ +# Get port script + ADD get_port.py /opt/anaconda3/bin/ -RUN chmod a+x /usr/lib/nersc-ssh-keys/NERSC-keys-api -WORKDIR /srv -RUN chmod +x docker-entrypoint.sh -ENTRYPOINT [ "./docker-entrypoint.sh" ] -CMD [ "/usr/sbin/sshd", "-p", "22", "-D" ] +# Supervisord to launch sshd and nslcd + +ADD supervisord.conf /etc/supervisor/conf.d/supervisord.conf +CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"] From 6c267defba10baf2048f7d32c56a9e5440c5cca9 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Sun, 2 Dec 2018 15:56:55 -0800 Subject: [PATCH 26/41] Remove nslcd --- jupyter-nersc/app/docker-entrypoint.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/jupyter-nersc/app/docker-entrypoint.sh b/jupyter-nersc/app/docker-entrypoint.sh index 6e500b8..9d8ad4d 100644 --- a/jupyter-nersc/app/docker-entrypoint.sh +++ b/jupyter-nersc/app/docker-entrypoint.sh @@ -1,5 +1,2 @@ #!/bin/bash - -service nslcd start - exec "$@" From 041cafba5d5a235bdec34369ce5229d80828db5a Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Sun, 2 Dec 2018 23:26:10 -0800 Subject: [PATCH 27/41] Real image setups --- jupyter-nersc/app/Dockerfile | 64 +++++++++++++++++------------------- 1 file changed, 30 insertions(+), 34 deletions(-) diff --git a/jupyter-nersc/app/Dockerfile b/jupyter-nersc/app/Dockerfile index a9669a6..64e0474 100644 --- a/jupyter-nersc/app/Dockerfile +++ b/jupyter-nersc/app/Dockerfile @@ -17,45 +17,41 @@ RUN \ # Python 3 Anaconda and additional packages -### TESTING ... - +##### RUN \ +##### /opt/anaconda3/bin/conda update --yes conda && \ +##### /opt/anaconda3/bin/conda install --yes \ +##### ipykernel \ +##### ipywidgets \ +##### jupyterlab \ +##### notebook && \ +##### /opt/anaconda3/bin/ipython kernel install && \ +##### /opt/anaconda3/bin/conda clean --yes --all +##### +##### # Typical extension +##### +##### RUN \ +##### /opt/anaconda3/bin/jupyter nbextension enable --sys-prefix --py widgetsnbextension + +ADD packages3.txt /tmp/packages3.txt RUN \ - /opt/anaconda3/bin/conda update --yes conda && \ - /opt/anaconda3/bin/conda install --yes \ - ipykernel \ - ipywidgets \ - jupyterlab \ - notebook && \ - /opt/anaconda3/bin/ipython kernel install && \ + /opt/anaconda3/bin/conda update --yes conda && \ + /opt/anaconda3/bin/conda install --yes anaconda && \ + /opt/anaconda3/bin/conda install --file /tmp/packages3.txt && \ + /opt/anaconda3/bin/ipython kernel install && \ /opt/anaconda3/bin/conda clean --yes --all -# Typical extension +# Python 2 Anaconda and additional packages +ADD packages2.txt /tmp/packages2.txt RUN \ - /opt/anaconda3/bin/jupyter nbextension enable --sys-prefix --py widgetsnbextension - -### TESTING DONE ... - -### ADD packages3.txt /tmp/packages3.txt -### RUN \ -### /opt/anaconda3/bin/conda update --yes conda && \ -### /opt/anaconda3/bin/conda install --yes anaconda && \ -### /opt/anaconda3/bin/conda install --file /tmp/packages3.txt && \ -### /opt/anaconda3/bin/ipython kernel install && \ -### /opt/anaconda3/bin/conda clean --yes --all - -### # Python 2 Anaconda and additional packages -### -### ADD packages2.txt /tmp/packages2.txt -### RUN \ -### wget -q -O /tmp/miniconda2.sh https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh && \ -### bash /tmp/miniconda2.sh -f -b -p /opt/anaconda2 && \ -### rm /tmp/miniconda2.sh && \ -### /opt/anaconda2/bin/conda update --yes conda && \ -### /opt/anaconda2/bin/conda install --yes anaconda && \ -### /opt/anaconda2/bin/conda install --file /tmp/packages2.txt && \ -### /opt/anaconda2/bin/ipython kernel install && \ -### /opt/anaconda2/bin/conda clean --yes --all + wget -q -O /tmp/miniconda2.sh https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh && \ + bash /tmp/miniconda2.sh -f -b -p /opt/anaconda2 && \ + rm /tmp/miniconda2.sh && \ + /opt/anaconda2/bin/conda update --yes conda && \ + /opt/anaconda2/bin/conda install --yes anaconda && \ + /opt/anaconda2/bin/conda install --file /tmp/packages2.txt && \ + /opt/anaconda2/bin/ipython kernel install && \ + /opt/anaconda2/bin/conda clean --yes --all # For ssh auth API From 196e0c5e62883f4312b348f3a607e89e537f7ba3 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Sun, 2 Dec 2018 23:26:19 -0800 Subject: [PATCH 28/41] Bring back r-essentials --- jupyter-nersc/app/packages3.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter-nersc/app/packages3.txt b/jupyter-nersc/app/packages3.txt index 3bf09a0..b447306 100644 --- a/jupyter-nersc/app/packages3.txt +++ b/jupyter-nersc/app/packages3.txt @@ -5,6 +5,6 @@ blaze jupyter jupyterlab netcdf4 -#r-essentials +r-essentials rpy2 ujson From c6d631f4cbb497aa39ec5569333c7831f810e98a Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Sun, 2 Dec 2018 23:26:29 -0800 Subject: [PATCH 29/41] Packages for Python 2 --- jupyter-nersc/app/packages2.txt | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 jupyter-nersc/app/packages2.txt diff --git a/jupyter-nersc/app/packages2.txt b/jupyter-nersc/app/packages2.txt new file mode 100644 index 0000000..6f668cf --- /dev/null +++ b/jupyter-nersc/app/packages2.txt @@ -0,0 +1,9 @@ +basemap +beautiful-soup +biopython +netcdf4 +r-car +r-essentials +rpy2 +theano +ujson From 1c04d876b5fdc7e4b78ad90146b25c084c6a7deb Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Sun, 2 Dec 2018 23:26:37 -0800 Subject: [PATCH 30/41] Supervisor setup --- jupyter-nersc/app/supervisord.conf | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 jupyter-nersc/app/supervisord.conf diff --git a/jupyter-nersc/app/supervisord.conf b/jupyter-nersc/app/supervisord.conf new file mode 100644 index 0000000..9a67e11 --- /dev/null +++ b/jupyter-nersc/app/supervisord.conf @@ -0,0 +1,16 @@ +[supervisord] +nodaemon=true + +[program:sshd] +directory=/srv +command=/usr/sbin/sshd -p 22 -D +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +redirect_stderr=true + +[program:nslcd] +directory=/srv +command=/usr/sbin/nslcd --nofork +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +redirect_stderr=true From f83cfe1095af5f8a60d38072644a1c5734f1aa6f Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Mon, 3 Dec 2018 12:34:10 -0800 Subject: [PATCH 31/41] JupyterHub and Lab stuff --- jupyter-nersc/app/Dockerfile | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/jupyter-nersc/app/Dockerfile b/jupyter-nersc/app/Dockerfile index 64e0474..ac7a60f 100644 --- a/jupyter-nersc/app/Dockerfile +++ b/jupyter-nersc/app/Dockerfile @@ -79,6 +79,21 @@ RUN \ ln -s /global/common/datatran /usr/common && \ echo "datatran" > /etc/clustername +# JupyterHub/lab features + +RUN \ + pip install --no-cache-dir ipympl && \ + jupyter nbextension enable --sys-prefix --py widgetsnbextension && \ + jupyter labextension install \ + @jupyterlab/hub-extension \ + @jupyter-widgets/jupyterlab-manager \ + jupyter-matplotlib \ + @jupyterlab/toc \ + jupyterlab_bokeh + +RUN \ + /opt/anaconda2/bin/jupyter nbextension enable --sys-prefix --py widgetsnbextension + # Get port script ADD get_port.py /opt/anaconda3/bin/ From 2877ceda2b99d4571028a3766935c6229516b47d Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Tue, 4 Dec 2018 09:21:35 -0800 Subject: [PATCH 32/41] Pass through args to docker build --- jupyter-dev-nersc/web/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupyter-dev-nersc/web/build.sh b/jupyter-dev-nersc/web/build.sh index 8edcc4e..284a4f5 100644 --- a/jupyter-dev-nersc/web/build.sh +++ b/jupyter-dev-nersc/web/build.sh @@ -4,5 +4,5 @@ branch=$(git symbolic-ref --short HEAD) docker build \ --build-arg branch=$branch \ - --no-cache \ + "$@" \ --tag registry.spin.nersc.gov/das/jupyter-dev-nersc-web.$branch:latest . From e456f3cc29922700a50a91dd0a311ffbb932abce Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Tue, 4 Dec 2018 09:22:06 -0800 Subject: [PATCH 33/41] Change back to cori --- jupyter-dev-nersc/web/jupyterhub_config.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/jupyter-dev-nersc/web/jupyterhub_config.py b/jupyter-dev-nersc/web/jupyterhub_config.py index b9197d4..f9b0ec2 100644 --- a/jupyter-dev-nersc/web/jupyterhub_config.py +++ b/jupyter-dev-nersc/web/jupyterhub_config.py @@ -18,8 +18,7 @@ def comma_split(string): ip = requests.get('https://v4.ifconfig.co/json').json()['ip'] -#bindir = '/global/common/cori/software/python/3.6-anaconda-5.2/bin/' -bindir = '/global/common/gerty/software/python/3.6-anaconda-5.2/bin/' +bindir = '/global/common/cori/software/python/3.6-anaconda-5.2/bin/' if 'BASE_PATH' in os.environ: bindir = os.path.join(os.environ['BASE_PATH'], 'bin') @@ -979,8 +978,7 @@ def comma_split(string): # SSHSpawner(Spawner) configuration #------------------------------------------------------------------------------ -#c.SSHSpawner.remote_hosts = ['cori19-224.nersc.gov'] -c.SSHSpawner.remote_hosts = ['gert01-224.nersc.gov'] +c.SSHSpawner.remote_hosts = ['cori19-224.nersc.gov'] c.SSHSpawner.remote_port = '22' c.SSHSpawner.hub_api_url = "http://{}:8081/hub/api".format(ip) c.SSHSpawner.path = bindir + ':/global/common/cori/das/jupyterhub/:/usr/common/usg/bin:/usr/bin:/bin' From 5d35e0763cbf0242dfad509c77c5db163fc6d856 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Wed, 5 Dec 2018 15:00:26 -0800 Subject: [PATCH 34/41] Fixes to enable use of fixed sshspawner --- jupyter-dev-nersc/web/jupyterhub_config.py | 31 +++++++++++++++------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/jupyter-dev-nersc/web/jupyterhub_config.py b/jupyter-dev-nersc/web/jupyterhub_config.py index f9b0ec2..cd50701 100644 --- a/jupyter-dev-nersc/web/jupyterhub_config.py +++ b/jupyter-dev-nersc/web/jupyterhub_config.py @@ -18,6 +18,7 @@ def comma_split(string): ip = requests.get('https://v4.ifconfig.co/json').json()['ip'] +#bindir = '/global/common/gerty/software/python/3.6-anaconda-5.2/bin/' bindir = '/global/common/cori/software/python/3.6-anaconda-5.2/bin/' if 'BASE_PATH' in os.environ: bindir = os.path.join(os.environ['BASE_PATH'], 'bin') @@ -978,26 +979,38 @@ def comma_split(string): # SSHSpawner(Spawner) configuration #------------------------------------------------------------------------------ -c.SSHSpawner.remote_hosts = ['cori19-224.nersc.gov'] -c.SSHSpawner.remote_port = '22' c.SSHSpawner.hub_api_url = "http://{}:8081/hub/api".format(ip) c.SSHSpawner.path = bindir + ':/global/common/cori/das/jupyterhub/:/usr/common/usg/bin:/usr/bin:/bin' -c.SSHSpawner.remote_port_command = '/usr/bin/python /global/common/cori/das/jupyterhub/get_port.py' c.SSHSpawner.ssh_keyfile = '/certs/{username}.key' +import asyncssh, random from tornado import web -import asyncio, asyncssh -async def my_quota(spawner): - host = spawner.choose_remote_host() +c.SSHSpawner.remote_hosts = ['cori19-224.nersc.gov'] +#c.SSHSpawner.remote_host = ['gert01-224.nersc.gov'] +c.SSHSpawner.remote_port_command = "python -c 'import socket; s=socket.socket(); s.bind((\"\", 0)); print(s.getsockname()[1]); s.close()'" + +async def setup(spawner): username = spawner.user.name - k = asyncssh.read_private_key(spawner.ssh_keyfile.format(username=username)) - async with asyncssh.connect(host,username=username,client_keys=[k],known_hosts=None) as conn: + remote_host = random.choice(spawner.remote_hosts) + keyfile = spawner.ssh_keyfile.format(username=username) + certfile = keyfile + "-cert.pub" + k = asyncssh.read_private_key(keyfile) + c = asyncssh.read_certificate(certfile) + print(username, remote_host, keyfile, certfile) + async with asyncssh.connect(remote_host, + username=username, + client_keys=[(k,c)], + known_hosts=None) as conn: result = await conn.run("myquota -c") retcode = result.exit_status + result = await conn.run(spawner.remote_port_command) + remote_port = int(result.stdout) if retcode: e = web.HTTPError(507,reason="Insufficient Storage") e.my_message = "There is insufficient space in your home directory; please clear up some files and try again." raise e + spawner.remote_host = remote_host + spawner.port = remote_port -c.Spawner.pre_spawn_hook = my_quota +c.Spawner.pre_spawn_hook = setup From 19221d38e602fdb07ad759f2d1b6e97f88b1ca08 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Wed, 5 Dec 2018 16:21:05 -0800 Subject: [PATCH 35/41] Change to flask app with customizable MESSAGE --- jupyter-off/Dockerfile | 51 +++++++++++++++-- jupyter-off/app.py | 17 ++++++ jupyter-off/nginx.conf | 52 ------------------ jupyter-off/{html => static}/logo.png | Bin ...nglelogo-greytext-orangebody-greymoons.png | Bin jupyter-off/{html => templates}/index.html | 9 +-- 6 files changed, 67 insertions(+), 62 deletions(-) create mode 100644 jupyter-off/app.py delete mode 100644 jupyter-off/nginx.conf rename jupyter-off/{html => static}/logo.png (100%) rename jupyter-off/{html => static}/rectanglelogo-greytext-orangebody-greymoons.png (100%) rename jupyter-off/{html => templates}/index.html (75%) diff --git a/jupyter-off/Dockerfile b/jupyter-off/Dockerfile index bf88d31..72a684e 100644 --- a/jupyter-off/Dockerfile +++ b/jupyter-off/Dockerfile @@ -1,4 +1,47 @@ -FROM nginx:1.13.7-alpine -MAINTAINER Rollin Thomas -COPY html /usr/share/nginx/html -COPY nginx.conf /etc/nginx +FROM ubuntu:16.04 +LABEL maintainer="Rollin Thomas " + +# Base Ubuntu packages + +ENV DEBIAN_FRONTEND noninteractive +ENV LANG C.UTF-8 + +RUN \ + apt-get update && \ + apt-get --yes upgrade && \ + apt-get --yes install \ + bzip2 \ + curl \ + tzdata \ + vim + +# Timezone to Berkeley + +ENV TZ=America/Los_Angeles +RUN \ + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ + echo $TZ > /etc/timezone + +# Add flask and gunicorn + +RUN \ + curl -s -o /tmp/miniconda3.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh && \ + bash /tmp/miniconda3.sh -f -b -p /opt/anaconda3 && \ + rm -rf /tmp/miniconda3.sh && \ + echo "python 3.6.*" >> /opt/anaconda3/conda-meta/pinned && \ + /opt/anaconda3/bin/conda update --yes conda && \ + /opt/anaconda3/bin/conda install --yes \ + flask \ + gunicorn && \ + /opt/anaconda3/bin/conda clean --yes --all + +ENV PATH=/opt/anaconda3/bin:$PATH + +# Application + +WORKDIR /srv +ADD app.py /srv/ +ADD templates /srv/templates +ADD static /srv/static + +CMD ["gunicorn", "app:app", "-b", ":8000", "--name", "app", "--workers=4", "--log-file=-"] diff --git a/jupyter-off/app.py b/jupyter-off/app.py new file mode 100644 index 0000000..967035d --- /dev/null +++ b/jupyter-off/app.py @@ -0,0 +1,17 @@ +import os + +from flask import Flask, render_template + +app = Flask(__name__) + +@app.route('/', defaults={'path': ''}) +@app.route('/') +def catch_all(path): + default_message = "NERSC's Jupyter service is offline. It will return when maintenance is over. Please try again later." + message = os.environ.get("MESSAGE", default_message).strip() + if not message: + message = default_message + return render_template("index.html", message=message), 503 + +if __name__ == '__main__': + app.run(host="0.0.0.0") diff --git a/jupyter-off/nginx.conf b/jupyter-off/nginx.conf deleted file mode 100644 index c83a12f..0000000 --- a/jupyter-off/nginx.conf +++ /dev/null @@ -1,52 +0,0 @@ - -user nginx; -worker_processes 1; - -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; - - -events { - worker_connections 1024; -} - - -http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - access_log /var/log/nginx/access.log main; - - sendfile on; - #tcp_nopush on; - - keepalive_timeout 65; - - #gzip on; - - server { - - listen 80; - listen 443; - listen 8000; - server_name localhost; - - location / { - root /usr/share/nginx/html; - index index.html index.htm; - rewrite ^/(.*\.png)$ /$1 break; - rewrite ^/.*$ /index.html break; - } - - error_page 500 502 503 504 /50x.html; - location = /50x.html { - root /usr/share/nginx/html; - } - - } - -} diff --git a/jupyter-off/html/logo.png b/jupyter-off/static/logo.png similarity index 100% rename from jupyter-off/html/logo.png rename to jupyter-off/static/logo.png diff --git a/jupyter-off/html/rectanglelogo-greytext-orangebody-greymoons.png b/jupyter-off/static/rectanglelogo-greytext-orangebody-greymoons.png similarity index 100% rename from jupyter-off/html/rectanglelogo-greytext-orangebody-greymoons.png rename to jupyter-off/static/rectanglelogo-greytext-orangebody-greymoons.png diff --git a/jupyter-off/html/index.html b/jupyter-off/templates/index.html similarity index 75% rename from jupyter-off/html/index.html rename to jupyter-off/templates/index.html index a3bb0a8..b8c4ea4 100644 --- a/jupyter-off/html/index.html +++ b/jupyter-off/templates/index.html @@ -16,20 +16,17 @@
- +
- +

Jupyter is Offline

-

Sorry for the inconvenience. - NERSC's Jupyter service is offline. - It will return when maintenance is over. - Please try again later.

+

Sorry for the inconvenience. {{ message }}

From 02d9264bd40953b0ccd62c65cc97817b7d8169e6 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Wed, 5 Dec 2018 16:22:13 -0800 Subject: [PATCH 36/41] Tell how to use it --- jupyter-off/README.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 jupyter-off/README.md diff --git a/jupyter-off/README.md b/jupyter-off/README.md new file mode 100644 index 0000000..93f68aa --- /dev/null +++ b/jupyter-off/README.md @@ -0,0 +1,5 @@ + +Jupyter is Offline +================== + +Set the `MESSAGE` environment variable if you want to customize it and upgrade the container! From e9c1130484f645f2abf35596b43bed80d3993abc Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Wed, 5 Dec 2018 16:23:25 -0800 Subject: [PATCH 37/41] Build script --- jupyter-off/build.sh | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 jupyter-off/build.sh diff --git a/jupyter-off/build.sh b/jupyter-off/build.sh new file mode 100644 index 0000000..42e135f --- /dev/null +++ b/jupyter-off/build.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +branch=$(git symbolic-ref --short HEAD) + +docker build \ + "$@" \ + --tag registry.spin.nersc.gov/das/jupyter-off.$branch:latest . From 4d7ec8e29b814982cfa8e8938438ca7fe8d1c7c0 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Wed, 5 Dec 2018 17:00:11 -0800 Subject: [PATCH 38/41] Branch not so important --- jupyter-off/build.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/jupyter-off/build.sh b/jupyter-off/build.sh index 42e135f..4884cac 100644 --- a/jupyter-off/build.sh +++ b/jupyter-off/build.sh @@ -1,7 +1,5 @@ #!/bin/bash -branch=$(git symbolic-ref --short HEAD) - docker build \ "$@" \ - --tag registry.spin.nersc.gov/das/jupyter-off.$branch:latest . + --tag registry.spin.nersc.gov/das/jupyter-off:latest . From e655a450b5f5337525ee0363b40506613c62f6ab Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Thu, 6 Dec 2018 18:03:55 -0800 Subject: [PATCH 39/41] Add shell of death --- jupyter-nersc/app/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/jupyter-nersc/app/Dockerfile b/jupyter-nersc/app/Dockerfile index ac7a60f..8063552 100644 --- a/jupyter-nersc/app/Dockerfile +++ b/jupyter-nersc/app/Dockerfile @@ -8,6 +8,7 @@ WORKDIR /srv RUN \ apt-get --yes install \ + csh \ ldap-utils \ libnss-ldapd \ libpam-ldap \ From a5443d8d19828e1ce52afc0d32100a87d67fdca7 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Thu, 13 Dec 2018 14:35:23 -0800 Subject: [PATCH 40/41] Pre-spawn hook disabled for now --- jupyter-dev-nersc/web/jupyterhub_config.py | 48 +++++++++++----------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/jupyter-dev-nersc/web/jupyterhub_config.py b/jupyter-dev-nersc/web/jupyterhub_config.py index cd50701..2576503 100644 --- a/jupyter-dev-nersc/web/jupyterhub_config.py +++ b/jupyter-dev-nersc/web/jupyterhub_config.py @@ -990,27 +990,27 @@ def comma_split(string): #c.SSHSpawner.remote_host = ['gert01-224.nersc.gov'] c.SSHSpawner.remote_port_command = "python -c 'import socket; s=socket.socket(); s.bind((\"\", 0)); print(s.getsockname()[1]); s.close()'" -async def setup(spawner): - username = spawner.user.name - remote_host = random.choice(spawner.remote_hosts) - keyfile = spawner.ssh_keyfile.format(username=username) - certfile = keyfile + "-cert.pub" - k = asyncssh.read_private_key(keyfile) - c = asyncssh.read_certificate(certfile) - print(username, remote_host, keyfile, certfile) - async with asyncssh.connect(remote_host, - username=username, - client_keys=[(k,c)], - known_hosts=None) as conn: - result = await conn.run("myquota -c") - retcode = result.exit_status - result = await conn.run(spawner.remote_port_command) - remote_port = int(result.stdout) - if retcode: - e = web.HTTPError(507,reason="Insufficient Storage") - e.my_message = "There is insufficient space in your home directory; please clear up some files and try again." - raise e - spawner.remote_host = remote_host - spawner.port = remote_port - -c.Spawner.pre_spawn_hook = setup +# async def setup(spawner): +# username = spawner.user.name +# remote_host = random.choice(spawner.remote_hosts) +# keyfile = spawner.ssh_keyfile.format(username=username) +# certfile = keyfile + "-cert.pub" +# k = asyncssh.read_private_key(keyfile) +# c = asyncssh.read_certificate(certfile) +# print(username, remote_host, keyfile, certfile) +# async with asyncssh.connect(remote_host, +# username=username, +# client_keys=[(k,c)], +# known_hosts=None) as conn: +# result = await conn.run("myquota -c") +# retcode = result.exit_status +# result = await conn.run(spawner.remote_port_command) +# remote_port = int(result.stdout) +# if retcode: +# e = web.HTTPError(507,reason="Insufficient Storage") +# e.my_message = "There is insufficient space in your home directory; please clear up some files and try again." +# raise e +# spawner.remote_host = remote_host +# spawner.port = remote_port +# +# c.Spawner.pre_spawn_hook = setup From fc489b638ba260bfeb481e8ec9ffc1d1016abd03 Mon Sep 17 00:00:00 2001 From: Rollin Thomas Date: Thu, 13 Dec 2018 14:36:28 -0800 Subject: [PATCH 41/41] Remove old --- jupyter-nersc/old/Dockerfile | 84 -- jupyter-nersc/old/build.sh | 8 - jupyter-nersc/old/docker-entrypoint.sh | 8 - jupyter-nersc/old/etc/ldap.conf | 12 - jupyter-nersc/old/etc/nslcd.conf | 25 - jupyter-nersc/old/etc/nsswitch.conf | 21 - jupyter-nersc/old/etc/pam.d/jupyterhub | 13 - jupyter-nersc/old/etc/pam.d/password-auth | 26 - jupyter-nersc/old/jupyterhub_config.py | 947 ---------------------- jupyter-nersc/old/packages2.txt | 9 - jupyter-nersc/old/packages3.txt | 7 - jupyter-nersc/old/supervisord.conf | 16 - 12 files changed, 1176 deletions(-) delete mode 100644 jupyter-nersc/old/Dockerfile delete mode 100644 jupyter-nersc/old/build.sh delete mode 100755 jupyter-nersc/old/docker-entrypoint.sh delete mode 100644 jupyter-nersc/old/etc/ldap.conf delete mode 100644 jupyter-nersc/old/etc/nslcd.conf delete mode 100644 jupyter-nersc/old/etc/nsswitch.conf delete mode 100644 jupyter-nersc/old/etc/pam.d/jupyterhub delete mode 100644 jupyter-nersc/old/etc/pam.d/password-auth delete mode 100644 jupyter-nersc/old/jupyterhub_config.py delete mode 100644 jupyter-nersc/old/packages2.txt delete mode 100644 jupyter-nersc/old/packages3.txt delete mode 100644 jupyter-nersc/old/supervisord.conf diff --git a/jupyter-nersc/old/Dockerfile b/jupyter-nersc/old/Dockerfile deleted file mode 100644 index 76ef084..0000000 --- a/jupyter-nersc/old/Dockerfile +++ /dev/null @@ -1,84 +0,0 @@ -FROM registry.spin.nersc.gov/das/jupyterhub-base.gaffer:latest -LABEL maintainer="Rollin Thomas " - -# Additional Ubuntu packages - -RUN \ - apt-get update && \ - apt-get install --yes \ - csh \ - dvipng \ - ldap-utils \ - libnss-ldapd \ - libpam-ldap \ - nscd \ - rsyslog \ - supervisor \ - texlive-xetex \ - zsh - -# Python 3 Anaconda and additional packages - -ADD packages3.txt /tmp/packages3.txt -RUN \ - /opt/anaconda3/bin/conda update --yes conda && \ - /opt/anaconda3/bin/conda install --yes anaconda && \ - /opt/anaconda3/bin/conda install --file /tmp/packages3.txt && \ - /opt/anaconda3/bin/ipython kernel install && \ - /opt/anaconda3/bin/conda clean --yes --all - -# Python 2 Anaconda and additional packages - -ADD packages2.txt /tmp/packages2.txt -RUN \ - curl -s -o /tmp/miniconda2.sh https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh && \ - bash /tmp/miniconda2.sh -f -b -p /opt/anaconda2 && \ - rm /tmp/miniconda2.sh && \ - /opt/anaconda2/bin/conda update --yes conda && \ - /opt/anaconda2/bin/conda install --yes anaconda && \ - /opt/anaconda2/bin/conda install --file /tmp/packages2.txt && \ - /opt/anaconda2/bin/ipython kernel install && \ - /opt/anaconda2/bin/conda clean --yes --all - -# PAM, LDAP - -COPY etc/ /etc/ - -# GPFS - -RUN \ - mkdir /global && \ - ln -sf /global/u1 /global/homes && \ - ln -sf /global/project /project && \ - ln -s /global/common/datatran /usr/common && \ - echo "datatran" > /etc/clustername - -# Jupyterhub/lab features - -RUN \ - conda install --yes jupyterlab=0.33.8 && \ - pip install --no-cache-dir ipympl && \ - jupyter nbextension enable --sys-prefix --py widgetsnbextension && \ - jupyter labextension install \ - @jupyterlab/hub-extension \ - @jupyter-widgets/jupyterlab-manager \ - jupyter-matplotlib \ - @jupyterlab/toc \ - jupyterlab_bokeh - -# Not sure we need this - -RUN \ - /opt/anaconda2/bin/jupyter nbextension enable --sys-prefix --py widgetsnbextension - -# Config and entrypoint script - -ADD docker-entrypoint.sh jupyterhub_config.py /srv/ - -WORKDIR /srv - -# TEMPORARILY HERE: Set up supervisord - -COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf - -CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"] diff --git a/jupyter-nersc/old/build.sh b/jupyter-nersc/old/build.sh deleted file mode 100644 index daf9890..0000000 --- a/jupyter-nersc/old/build.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -branch=$(git symbolic-ref --short HEAD) - -docker build \ - --build-arg branch=$branch \ - --no-cache \ - --tag registry.spin.nersc.gov/das/jupyterhub-jupyter-spin.$branch:latest . diff --git a/jupyter-nersc/old/docker-entrypoint.sh b/jupyter-nersc/old/docker-entrypoint.sh deleted file mode 100755 index 97c0439..0000000 --- a/jupyter-nersc/old/docker-entrypoint.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -service rsyslog restart -service nslcd restart - -ip addr - -exec "$@" diff --git a/jupyter-nersc/old/etc/ldap.conf b/jupyter-nersc/old/etc/ldap.conf deleted file mode 100644 index 39af084..0000000 --- a/jupyter-nersc/old/etc/ldap.conf +++ /dev/null @@ -1,12 +0,0 @@ -# -# LDAP Defaults -# - -# See ldap.conf(5) for details -# This file should be world readable but not world writable. - - -URI ldaps://ldapcrt.nersc.gov -BASE ou=datatran,ou=Host,o=ldapsvc,dc=nersc,dc=gov -TLS_CACERTDIR /etc/openldap/cacerts -TLS_REQCERT never diff --git a/jupyter-nersc/old/etc/nslcd.conf b/jupyter-nersc/old/etc/nslcd.conf deleted file mode 100644 index 4d695ce..0000000 --- a/jupyter-nersc/old/etc/nslcd.conf +++ /dev/null @@ -1,25 +0,0 @@ -# $Id: nslcd.conf.tmpl 175 2012-10-24 23:13:43Z nsgsrc $ - -tls_reqcert never -uid nslcd -gid nslcd -uri ldaps://ldapcrt.nersc.gov/ -base ou=datatran,ou=host,o=ldapsvc,dc=nersc,dc=gov -ssl on - -# Fix regex to allow for two-letter names -# See: https://bugzilla.redhat.com/show_bug.cgi?id=706860 -validnames /^[a-z0-9._@$()][a-z0-9._@$() \~-]*[a-z0-9._@$()~-]$/i - -tls_cacertdir /etc/openldap/cacerts - -# Don't try to resolve any users in the local user database -nss_initgroups_ignoreusers ALLLOCAL - -# Set idle_timelimit to 4 minutes to avoid "can't contact LDAP server" log -# messages -idle_timelimit 240 - -filter passwd (&(objectClass=posixAccount)(authorizedService=ssh)) - -log syslog debug diff --git a/jupyter-nersc/old/etc/nsswitch.conf b/jupyter-nersc/old/etc/nsswitch.conf deleted file mode 100644 index 3d77354..0000000 --- a/jupyter-nersc/old/etc/nsswitch.conf +++ /dev/null @@ -1,21 +0,0 @@ -passwd: files ldap -shadow: files ldap -group: files ldap - -hosts: files dns - -bootparams: nisplus [NOTFOUND=return] files - -ethers: files -netmasks: files -networks: files -protocols: files -rpc: files -services: files - -netgroup: nisplus - -publickey: nisplus - -automount: files nisplus -aliases: files nisplus diff --git a/jupyter-nersc/old/etc/pam.d/jupyterhub b/jupyter-nersc/old/etc/pam.d/jupyterhub deleted file mode 100644 index a8a86f4..0000000 --- a/jupyter-nersc/old/etc/pam.d/jupyterhub +++ /dev/null @@ -1,13 +0,0 @@ -#%PAM-1.0 -auth required pam_sepermit.so -auth include password-auth -account required pam_nologin.so -account include password-auth -password include password-auth -# pam_selinux.so close should be the first session rule -session required pam_selinux.so close -session optional pam_loginuid.so -# pam_selinux.so open should only be followed by sessions to be executed in the user context -session required pam_selinux.so open env_params -session optional pam_keyinit.so force revoke -session include password-auth diff --git a/jupyter-nersc/old/etc/pam.d/password-auth b/jupyter-nersc/old/etc/pam.d/password-auth deleted file mode 100644 index 87cf4e7..0000000 --- a/jupyter-nersc/old/etc/pam.d/password-auth +++ /dev/null @@ -1,26 +0,0 @@ -#%PAM-1.0 - -# $Id: password-auth.tmpl 668 2013-11-01 20:00:51Z nsgsrc $ - -auth required pam_env.so -auth sufficient pam_unix.so nullok try_first_pass -auth requisite pam_succeed_if.so uid >= 400 quiet -auth sufficient pam_ldap.so use_first_pass -auth required pam_deny.so - -account required pam_access.so -account required pam_unix.so broken_shadow -account sufficient pam_localuser.so -account sufficient pam_succeed_if.so uid < 400 quiet -account [default=bad success=ok user_unknown=ignore] pam_ldap.so -account required pam_permit.so - -password requisite pam_cracklib.so try_first_pass retry=3 type= -password sufficient pam_unix.so sha512 shadow nullok try_first_pass use_authtok -password sufficient pam_ldap.so use_authtok -password required pam_deny.so - -session optional pam_keyinit.so revoke -session required pam_limits.so -session required pam_unix.so -session optional pam_ldap.so diff --git a/jupyter-nersc/old/jupyterhub_config.py b/jupyter-nersc/old/jupyterhub_config.py deleted file mode 100644 index 3de7f25..0000000 --- a/jupyter-nersc/old/jupyterhub_config.py +++ /dev/null @@ -1,947 +0,0 @@ -# Configuration file for jupyterhub. - -import os - -def comma_split(string): - """Handle env variables that may be None, empty string, or have spaces""" - try: - stripped = string.strip() - except AttributeError: - return list() - if stripped: - return [s.strip() for s in stripped.split(",")] - else: - return list() - -#------------------------------------------------------------------------------ -# Application(SingletonConfigurable) configuration -#------------------------------------------------------------------------------ - -## This is an application. - -## The date format used by logging formatters for %(asctime)s -#c.Application.log_datefmt = '%Y-%m-%d %H:%M:%S' - -## The Logging format template -#c.Application.log_format = '[%(name)s]%(highlevel)s %(message)s' - -## Set the log level by value or name. -#c.Application.log_level = 30 - -#------------------------------------------------------------------------------ -# JupyterHub(Application) configuration -#------------------------------------------------------------------------------ - -## An Application for starting a Multi-User Jupyter Notebook server. - -## Maximum number of concurrent servers that can be active at a time. -# -# Setting this can limit the total resources your users can consume. -# -# An active server is any server that's not fully stopped. It is considered -# active from the time it has been requested until the time that it has -# completely stopped. -# -# If this many user servers are active, users will not be able to launch new -# servers until a server is shutdown. Spawn requests will be rejected with a 429 -# error asking them to try again. -# -# If set to 0, no limit is enforced. -#c.JupyterHub.active_server_limit = 0 - -## Duration (in seconds) to determine the number of active users. -#c.JupyterHub.active_user_window = 1800 - -## Grant admin users permission to access single-user servers. -# -# Users should be properly informed if this is enabled. -#c.JupyterHub.admin_access = False - -## DEPRECATED since version 0.7.2, use Authenticator.admin_users instead. -#c.JupyterHub.admin_users = set() - -## Allow named single-user servers per user -#c.JupyterHub.allow_named_servers = False - -## Answer yes to any questions (e.g. confirm overwrite) -#c.JupyterHub.answer_yes = False - -## PENDING DEPRECATION: consider using service_tokens -# -# Dict of token:username to be loaded into the database. -# -# Allows ahead-of-time generation of API tokens for use by externally managed -# services, which authenticate as JupyterHub users. -# -# Consider using service_tokens for general services that talk to the JupyterHub -# API. -#c.JupyterHub.api_tokens = {} - -## Class for authenticating users. -# -# This should be a class with the following form: -# -# - constructor takes one kwarg: `config`, the IPython config object. -# -# with an authenticate method that: -# -# - is a coroutine (asyncio or tornado) -# - returns username on success, None on failure -# - takes two arguments: (handler, data), -# where `handler` is the calling web.RequestHandler, -# and `data` is the POST form data from the login page. -#c.JupyterHub.authenticator_class = 'jupyterhub.auth.PAMAuthenticator' - -## The base URL of the entire application. -# -# Add this to the beginning of all JupyterHub URLs. Use base_url to run -# JupyterHub within an existing website. -# -# .. deprecated: 0.9 -# Use JupyterHub.bind_url -#c.JupyterHub.base_url = '/' - -## The public facing URL of the whole JupyterHub application. -# -# This is the address on which the proxy will bind. Sets protocol, ip, base_url -#c.JupyterHub.bind_url = 'http://:8000' - -## Whether to shutdown the proxy when the Hub shuts down. -# -# Disable if you want to be able to teardown the Hub while leaving the proxy -# running. -# -# Only valid if the proxy was starting by the Hub process. -# -# If both this and cleanup_servers are False, sending SIGINT to the Hub will -# only shutdown the Hub, leaving everything else running. -# -# The Hub should be able to resume from database state. -#c.JupyterHub.cleanup_proxy = True - -## Whether to shutdown single-user servers when the Hub shuts down. -# -# Disable if you want to be able to teardown the Hub while leaving the single- -# user servers running. -# -# If both this and cleanup_proxy are False, sending SIGINT to the Hub will only -# shutdown the Hub, leaving everything else running. -# -# The Hub should be able to resume from database state. -#c.JupyterHub.cleanup_servers = True - -## Maximum number of concurrent users that can be spawning at a time. -# -# Spawning lots of servers at the same time can cause performance problems for -# the Hub or the underlying spawning system. Set this limit to prevent bursts of -# logins from attempting to spawn too many servers at the same time. -# -# This does not limit the number of total running servers. See -# active_server_limit for that. -# -# If more than this many users attempt to spawn at a time, their requests will -# be rejected with a 429 error asking them to try again. Users will have to wait -# for some of the spawning services to finish starting before they can start -# their own. -# -# If set to 0, no limit is enforced. -#c.JupyterHub.concurrent_spawn_limit = 100 - -## The config file to load -#c.JupyterHub.config_file = 'jupyterhub_config.py' - -## DEPRECATED: does nothing -#c.JupyterHub.confirm_no_ssl = False - -## Number of days for a login cookie to be valid. Default is two weeks. -#c.JupyterHub.cookie_max_age_days = 14 - -## The cookie secret to use to encrypt cookies. -# -# Loaded from the JPY_COOKIE_SECRET env variable by default. -# -# Should be exactly 256 bits (32 bytes). -#c.JupyterHub.cookie_secret = b'' - -## File in which to store the cookie secret. -#c.JupyterHub.cookie_secret_file = 'jupyterhub_cookie_secret' - -## The location of jupyterhub data files (e.g. /usr/local/share/jupyterhub) -#c.JupyterHub.data_files_path = '/opt/anaconda3/share/jupyterhub' - -## Include any kwargs to pass to the database connection. See -# sqlalchemy.create_engine for details. -#c.JupyterHub.db_kwargs = {} - -## url for the database. e.g. `sqlite:///jupyterhub.sqlite` -#c.JupyterHub.db_url = 'sqlite:///jupyterhub.sqlite' - -## log all database transactions. This has A LOT of output -#c.JupyterHub.debug_db = False - -## DEPRECATED since version 0.8: Use ConfigurableHTTPProxy.debug -#c.JupyterHub.debug_proxy = False - -## The default URL for users when they arrive (e.g. when user directs to "/") -# -# By default, redirects users to their own server. -#c.JupyterHub.default_url = '' - -## Register extra tornado Handlers for jupyterhub. -# -# Should be of the form ``("", Handler)`` -# -# The Hub prefix will be added, so `/my-page` will be served at `/hub/my-page`. -#c.JupyterHub.extra_handlers = [] - -## DEPRECATED: use output redirection instead, e.g. -# -# jupyterhub &>> /var/log/jupyterhub.log -#c.JupyterHub.extra_log_file = '' - -## Extra log handlers to set on JupyterHub logger -#c.JupyterHub.extra_log_handlers = [] - -## Generate default config file -#c.JupyterHub.generate_config = False - -## The URL on which the Hub will listen. This is a private URL for internal -# communication. Typically set in combination with hub_connect_url. If a unix -# socket, hub_connect_url **must** also be set. -# -# For example: -# -# "http://127.0.0.1:8081" -# "unix+http://%2Fsrv%2Fjupyterhub%2Fjupyterhub.sock" -# -# .. versionadded:: 0.9 -#c.JupyterHub.hub_bind_url = '' - -## The ip or hostname for proxies and spawners to use for connecting to the Hub. -# -# Use when the bind address (`hub_ip`) is 0.0.0.0 or otherwise different from -# the connect address. -# -# Default: when `hub_ip` is 0.0.0.0, use `socket.gethostname()`, otherwise use -# `hub_ip`. -# -# Note: Some spawners or proxy implementations might not support hostnames. -# Check your spawner or proxy documentation to see if they have extra -# requirements. -# -# .. versionadded:: 0.8 -#c.JupyterHub.hub_connect_ip = '' - -## DEPRECATED -# -# Use hub_connect_url -# -# .. versionadded:: 0.8 -# -# .. deprecated:: 0.9 -# Use hub_connect_url -#c.JupyterHub.hub_connect_port = 0 - -## The URL for connecting to the Hub. Spawners, services, and the proxy will use -# this URL to talk to the Hub. -# -# Only needs to be specified if the default hub URL is not connectable (e.g. -# using a unix+http:// bind url). -# -# .. seealso:: -# JupyterHub.hub_connect_ip -# JupyterHub.hub_bind_url -# -# .. versionadded:: 0.9 -#c.JupyterHub.hub_connect_url = '' - -## The ip address for the Hub process to *bind* to. -# -# By default, the hub listens on localhost only. This address must be accessible -# from the proxy and user servers. You may need to set this to a public ip or '' -# for all interfaces if the proxy or user servers are in containers or on a -# different host. -# -# See `hub_connect_ip` for cases where the bind and connect address should -# differ, or `hub_bind_url` for setting the full bind URL. -#c.JupyterHub.hub_ip = '127.0.0.1' - -## The internal port for the Hub process. -# -# This is the internal port of the hub itself. It should never be accessed -# directly. See JupyterHub.port for the public port to use when accessing -# jupyterhub. It is rare that this port should be set except in cases of port -# conflict. -# -# See also `hub_ip` for the ip and `hub_bind_url` for setting the full bind URL. -#c.JupyterHub.hub_port = 8081 - -## The public facing ip of the whole JupyterHub application (specifically -# referred to as the proxy). -# -# This is the address on which the proxy will listen. The default is to listen -# on all interfaces. This is the only address through which JupyterHub should be -# accessed by users. -# -# .. deprecated: 0.9 -# Use JupyterHub.bind_url -#c.JupyterHub.ip = '' -c.JupyterHub.ip = '0.0.0.0' - -## Supply extra arguments that will be passed to Jinja environment. -#c.JupyterHub.jinja_environment_options = {} - -## Interval (in seconds) at which to update last-activity timestamps. -#c.JupyterHub.last_activity_interval = 300 - -## Dict of 'group': ['usernames'] to load at startup. -# -# This strictly *adds* groups and users to groups. -# -# Loading one set of groups, then starting JupyterHub again with a different set -# will not remove users or groups from previous launches. That must be done -# through the API. -#c.JupyterHub.load_groups = {} - -## Specify path to a logo image to override the Jupyter logo in the banner. -#c.JupyterHub.logo_file = '' - -## File to write PID Useful for daemonizing JupyterHub. -#c.JupyterHub.pid_file = '' - -## The public facing port of the proxy. -# -# This is the port on which the proxy will listen. This is the only port through -# which JupyterHub should be accessed by users. -# -# .. deprecated: 0.9 -# Use JupyterHub.bind_url -#c.JupyterHub.port = 8000 - -## DEPRECATED since version 0.8 : Use ConfigurableHTTPProxy.api_url -#c.JupyterHub.proxy_api_ip = '' -c.JupyterHub.proxy_api_ip = '127.0.0.1' - -## DEPRECATED since version 0.8 : Use ConfigurableHTTPProxy.api_url -#c.JupyterHub.proxy_api_port = 0 - -## DEPRECATED since version 0.8: Use ConfigurableHTTPProxy.auth_token -#c.JupyterHub.proxy_auth_token = '' - -## Interval (in seconds) at which to check if the proxy is running. -#c.JupyterHub.proxy_check_interval = 30 - -## Select the Proxy API implementation. -#c.JupyterHub.proxy_class = 'jupyterhub.proxy.ConfigurableHTTPProxy' - -## DEPRECATED since version 0.8. Use ConfigurableHTTPProxy.command -#c.JupyterHub.proxy_cmd = [] - -## Redirect user to server (if running), instead of control panel. -#c.JupyterHub.redirect_to_server = True - -## Purge and reset the database. -#c.JupyterHub.reset_db = False - -## Interval (in seconds) at which to check connectivity of services with web -# endpoints. -#c.JupyterHub.service_check_interval = 60 - -## Dict of token:servicename to be loaded into the database. -# -# Allows ahead-of-time generation of API tokens for use by externally managed -# services. -#c.JupyterHub.service_tokens = {} - -## List of service specification dictionaries. -# -# A service -# -# For instance:: -# -# services = [ -# { -# 'name': 'cull_idle', -# 'command': ['/path/to/cull_idle_servers.py'], -# }, -# { -# 'name': 'formgrader', -# 'url': 'http://127.0.0.1:1234', -# 'api_token': 'super-secret', -# 'environment': -# } -# ] -#c.JupyterHub.services = [] -c.JupyterHub.services = [ - { - 'name': 'cull-idle', - 'admin': True, - 'command': 'cull_idle_servers.py --timeout=43200'.split(), - } -] - -## The class to use for spawning single-user servers. -# -# Should be a subclass of Spawner. -#c.JupyterHub.spawner_class = 'jupyterhub.spawner.LocalProcessSpawner' - -## Path to SSL certificate file for the public facing interface of the proxy -# -# When setting this, you should also set ssl_key -#c.JupyterHub.ssl_cert = '' - -## Path to SSL key file for the public facing interface of the proxy -# -# When setting this, you should also set ssl_cert -#c.JupyterHub.ssl_key = '' - -## Host to send statsd metrics to. An empty string (the default) disables sending -# metrics. -#c.JupyterHub.statsd_host = '' - -## Port on which to send statsd metrics about the hub -#c.JupyterHub.statsd_port = 8125 - -## Prefix to use for all metrics sent by jupyterhub to statsd -#c.JupyterHub.statsd_prefix = 'jupyterhub' - -## Run single-user servers on subdomains of this host. -# -# This should be the full `https://hub.domain.tld[:port]`. -# -# Provides additional cross-site protections for javascript served by single- -# user servers. -# -# Requires `.hub.domain.tld` to resolve to the same host as -# `hub.domain.tld`. -# -# In general, this is most easily achieved with wildcard DNS. -# -# When using SSL (i.e. always) this also requires a wildcard SSL certificate. -#c.JupyterHub.subdomain_host = '' - -## Paths to search for jinja templates, before using the default templates. -#c.JupyterHub.template_paths = [] - -## Extra variables to be passed into jinja templates -#c.JupyterHub.template_vars = {} - -## Extra settings overrides to pass to the tornado application. -#c.JupyterHub.tornado_settings = {} - -## Trust user-provided tokens (via JupyterHub.service_tokens) to have good -# entropy. -# -# If you are not inserting additional tokens via configuration file, this flag -# has no effect. -# -# In JupyterHub 0.8, internally generated tokens do not pass through additional -# hashing because the hashing is costly and does not increase the entropy of -# already-good UUIDs. -# -# User-provided tokens, on the other hand, are not trusted to have good entropy -# by default, and are passed through many rounds of hashing to stretch the -# entropy of the key (i.e. user-provided tokens are treated as passwords instead -# of random keys). These keys are more costly to check. -# -# If your inserted tokens are generated by a good-quality mechanism, e.g. -# `openssl rand -hex 32`, then you can set this flag to True to reduce the cost -# of checking authentication tokens. -#c.JupyterHub.trust_user_provided_tokens = False - -## Upgrade the database automatically on start. -# -# Only safe if database is regularly backed up. Only SQLite databases will be -# backed up to a local file automatically. -#c.JupyterHub.upgrade_db = False - -#------------------------------------------------------------------------------ -# Spawner(LoggingConfigurable) configuration -#------------------------------------------------------------------------------ - -## Base class for spawning single-user notebook servers. -# -# Subclass this, and override the following methods: -# -# - load_state - get_state - start - stop - poll -# -# As JupyterHub supports multiple users, an instance of the Spawner subclass is -# created for each user. If there are 20 JupyterHub users, there will be 20 -# instances of the subclass. - -## Extra arguments to be passed to the single-user server. -# -# Some spawners allow shell-style expansion here, allowing you to use -# environment variables here. Most, including the default, do not. Consult the -# documentation for your spawner to verify! -#c.Spawner.args = [] - -## The command used for starting the single-user server. -# -# Provide either a string or a list containing the path to the startup script -# command. Extra arguments, other than this path, should be provided via `args`. -# -# This is usually set if you want to start the single-user server in a different -# python environment (with virtualenv/conda) than JupyterHub itself. -# -# Some spawners allow shell-style expansion here, allowing you to use -# environment variables. Most, including the default, do not. Consult the -# documentation for your spawner to verify! -#c.Spawner.cmd = ['jupyterhub-singleuser'] -c.Spawner.cmd = ['/opt/anaconda3/bin/jupyter-labhub'] - -## Maximum number of consecutive failures to allow before shutting down -# JupyterHub. -# -# This helps JupyterHub recover from a certain class of problem preventing -# launch in contexts where the Hub is automatically restarted (e.g. systemd, -# docker, kubernetes). -# -# A limit of 0 means no limit and consecutive failures will not be tracked. -#c.Spawner.consecutive_failure_limit = 0 - -## Minimum number of cpu-cores a single-user notebook server is guaranteed to -# have available. -# -# If this value is set to 0.5, allows use of 50% of one CPU. If this value is -# set to 2, allows use of up to 2 CPUs. -# -# **This is a configuration setting. Your spawner must implement support for the -# limit to work.** The default spawner, `LocalProcessSpawner`, does **not** -# implement this support. A custom spawner **must** add support for this setting -# for it to be enforced. -#c.Spawner.cpu_guarantee = None - -## Maximum number of cpu-cores a single-user notebook server is allowed to use. -# -# If this value is set to 0.5, allows use of 50% of one CPU. If this value is -# set to 2, allows use of up to 2 CPUs. -# -# The single-user notebook server will never be scheduled by the kernel to use -# more cpu-cores than this. There is no guarantee that it can access this many -# cpu-cores. -# -# **This is a configuration setting. Your spawner must implement support for the -# limit to work.** The default spawner, `LocalProcessSpawner`, does **not** -# implement this support. A custom spawner **must** add support for this setting -# for it to be enforced. -#c.Spawner.cpu_limit = None - -## Enable debug-logging of the single-user server -#c.Spawner.debug = False - -## The URL the single-user server should start in. -# -# `{username}` will be expanded to the user's username -# -# Example uses: -# -# - You can set `notebook_dir` to `/` and `default_url` to `/tree/home/{username}` to allow people to -# navigate the whole filesystem from their notebook server, but still start in their home directory. -# - Start with `/notebooks` instead of `/tree` if `default_url` points to a notebook instead of a directory. -# - You can set this to `/lab` to have JupyterLab start by default, rather than Jupyter Notebook. -#c.Spawner.default_url = '' -c.Spawner.default_url = '/lab/tree/global/homes/{username[0]}/{username}' - -## Disable per-user configuration of single-user servers. -# -# When starting the user's single-user server, any config file found in the -# user's $HOME directory will be ignored. -# -# Note: a user could circumvent this if the user modifies their Python -# environment, such as when they have their own conda environments / virtualenvs -# / containers. -#c.Spawner.disable_user_config = False - -## Whitelist of environment variables for the single-user server to inherit from -# the JupyterHub process. -# -# This whitelist is used to ensure that sensitive information in the JupyterHub -# process's environment (such as `CONFIGPROXY_AUTH_TOKEN`) is not passed to the -# single-user server's process. -#c.Spawner.env_keep = ['PATH', 'PYTHONPATH', 'CONDA_ROOT', 'CONDA_DEFAULT_ENV', 'VIRTUAL_ENV', 'LANG', 'LC_ALL'] - -## Extra environment variables to set for the single-user server's process. -# -# Environment variables that end up in the single-user server's process come from 3 sources: -# - This `environment` configurable -# - The JupyterHub process' environment variables that are whitelisted in `env_keep` -# - Variables to establish contact between the single-user notebook and the hub (such as JUPYTERHUB_API_TOKEN) -# -# The `environment` configurable should be set by JupyterHub administrators to -# add installation specific environment variables. It is a dict where the key is -# the name of the environment variable, and the value can be a string or a -# callable. If it is a callable, it will be called with one parameter (the -# spawner instance), and should return a string fairly quickly (no blocking -# operations please!). -# -# Note that the spawner class' interface is not guaranteed to be exactly same -# across upgrades, so if you are using the callable take care to verify it -# continues to work after upgrades! -#c.Spawner.environment = {} -c.Spawner.environment = {"OMP_NUM_THREADS" : "2"} - -## Timeout (in seconds) before giving up on a spawned HTTP server -# -# Once a server has successfully been spawned, this is the amount of time we -# wait before assuming that the server is unable to accept connections. -#c.Spawner.http_timeout = 30 - -## The IP address (or hostname) the single-user server should listen on. -# -# The JupyterHub proxy implementation should be able to send packets to this -# interface. -#c.Spawner.ip = '' -c.Spawner.ip = '127.0.0.1' - -## Minimum number of bytes a single-user notebook server is guaranteed to have -# available. -# -# Allows the following suffixes: -# - K -> Kilobytes -# - M -> Megabytes -# - G -> Gigabytes -# - T -> Terabytes -# -# **This is a configuration setting. Your spawner must implement support for the -# limit to work.** The default spawner, `LocalProcessSpawner`, does **not** -# implement this support. A custom spawner **must** add support for this setting -# for it to be enforced. -#c.Spawner.mem_guarantee = None - -## Maximum number of bytes a single-user notebook server is allowed to use. -# -# Allows the following suffixes: -# - K -> Kilobytes -# - M -> Megabytes -# - G -> Gigabytes -# - T -> Terabytes -# -# If the single user server tries to allocate more memory than this, it will -# fail. There is no guarantee that the single-user notebook server will be able -# to allocate this much memory - only that it can not allocate more than this. -# -# **This is a configuration setting. Your spawner must implement support for the -# limit to work.** The default spawner, `LocalProcessSpawner`, does **not** -# implement this support. A custom spawner **must** add support for this setting -# for it to be enforced. -#c.Spawner.mem_limit = None - -## Path to the notebook directory for the single-user server. -# -# The user sees a file listing of this directory when the notebook interface is -# started. The current interface does not easily allow browsing beyond the -# subdirectories in this directory's tree. -# -# `~` will be expanded to the home directory of the user, and {username} will be -# replaced with the name of the user. -# -# Note that this does *not* prevent users from accessing files outside of this -# path! They can do so with many other means. -#c.Spawner.notebook_dir = '' -c.Spawner.notebook_dir = '/' - -## An HTML form for options a user can specify on launching their server. -# -# The surrounding `` element and the submit button are already provided. -# -# For example: -# -# .. code:: html -# -# Set your key: -# -#
-# Choose a letter: -# -# -# The data from this form submission will be passed on to your spawner in -# `self.user_options` -# -# Instead of a form snippet string, this could also be a callable that takes as -# one parameter the current spawner instance and returns a string. The callable -# will be called asynchronously if it returns a future, rather than a str. Note -# that the interface of the spawner class is not deemed stable across versions, -# so using this functionality might cause your JupyterHub upgrades to break. -#c.Spawner.options_form = traitlets.Undefined - -## Interval (in seconds) on which to poll the spawner for single-user server's -# status. -# -# At every poll interval, each spawner's `.poll` method is called, which checks -# if the single-user server is still running. If it isn't running, then -# JupyterHub modifies its own state accordingly and removes appropriate routes -# from the configurable proxy. -#c.Spawner.poll_interval = 30 - -## The port for single-user servers to listen on. -# -# Defaults to `0`, which uses a randomly allocated port number each time. -# -# If set to a non-zero value, all Spawners will use the same port, which only -# makes sense if each server is on a different address, e.g. in containers. -# -# New in version 0.7. -#c.Spawner.port = 0 - -## An optional hook function that you can implement to do work after the spawner -# stops. -# -# This can be set independent of any concrete spawner implementation. -#c.Spawner.post_stop_hook = None - -## An optional hook function that you can implement to do some bootstrapping work -# before the spawner starts. For example, create a directory for your user or -# load initial content. -# -# This can be set independent of any concrete spawner implementation. -# -# Example:: -# -# from subprocess import check_call -# def my_hook(spawner): -# username = spawner.user.name -# check_call(['./examples/bootstrap-script/bootstrap.sh', username]) -# -# c.Spawner.pre_spawn_hook = my_hook -#c.Spawner.pre_spawn_hook = None - -## Timeout (in seconds) before giving up on starting of single-user server. -# -# This is the timeout for start to return, not the timeout for the server to -# respond. Callers of spawner.start will assume that startup has failed if it -# takes longer than this. start should return when the server process is started -# and its location is known. -#c.Spawner.start_timeout = 60 - -#------------------------------------------------------------------------------ -# LocalProcessSpawner(Spawner) configuration -#------------------------------------------------------------------------------ - -## A Spawner that uses `subprocess.Popen` to start single-user servers as local -# processes. -# -# Requires local UNIX users matching the authenticated users to exist. Does not -# work on Windows. -# -# This is the default spawner for JupyterHub. -# -# Note: This spawner does not implement CPU / memory guarantees and limits. - -## Seconds to wait for single-user server process to halt after SIGINT. -# -# If the process has not exited cleanly after this many seconds, a SIGTERM is -# sent. -#c.LocalProcessSpawner.interrupt_timeout = 10 - -## Seconds to wait for process to halt after SIGKILL before giving up. -# -# If the process does not exit cleanly after this many seconds of SIGKILL, it -# becomes a zombie process. The hub process will log a warning and then give up. -#c.LocalProcessSpawner.kill_timeout = 5 - -## Extra keyword arguments to pass to Popen -# -# when spawning single-user servers. -# -# For example:: -# -# popen_kwargs = dict(shell=True) -#c.LocalProcessSpawner.popen_kwargs = {} - -## Specify a shell command to launch. -# -# The single-user command will be appended to this list, so it sould end with -# `-c` (for bash) or equivalent. -# -# For example:: -# -# c.LocalProcessSpawner.shell_cmd = ['bash', '-l', '-c'] -# -# to launch with a bash login shell, which would set up the user's own complete -# environment. -# -# .. warning:: -# -# Using shell_cmd gives users control over PATH, etc., -# which could change what the jupyterhub-singleuser launch command does. -# Only use this for trusted users. -#c.LocalProcessSpawner.shell_cmd = [] - -## Seconds to wait for single-user server process to halt after SIGTERM. -# -# If the process does not exit cleanly after this many seconds of SIGTERM, a -# SIGKILL is sent. -#c.LocalProcessSpawner.term_timeout = 5 - -#------------------------------------------------------------------------------ -# Authenticator(LoggingConfigurable) configuration -#------------------------------------------------------------------------------ - -## Base class for implementing an authentication provider for JupyterHub - -## Set of users that will have admin rights on this JupyterHub. -# -# Admin users have extra privileges: -# - Use the admin panel to see list of users logged in -# - Add / remove users in some authenticators -# - Restart / halt the hub -# - Start / stop users' single-user servers -# - Can access each individual users' single-user server (if configured) -# -# Admin access should be treated the same way root access is. -# -# Defaults to an empty set, in which case no user has admin access. -#c.Authenticator.admin_users = set() -c.Authenticator.admin_users = set(comma_split(os.environ.get("ADMINS"))) - -## Automatically begin the login process -# -# rather than starting with a "Login with..." link at `/hub/login` -# -# To work, `.login_url()` must give a URL other than the default `/hub/login`, -# such as an oauth handler or another automatic login handler, registered with -# `.get_handlers()`. -# -# .. versionadded:: 0.8 -#c.Authenticator.auto_login = False - -## Blacklist of usernames that are not allowed to log in. -# -# Use this with supported authenticators to restrict which users can not log in. -# This is an additional blacklist that further restricts users, beyond whatever -# restrictions the authenticator has in place. -# -# If empty, does not perform any additional restriction. -# -# .. versionadded: 0.9 -#c.Authenticator.blacklist = set() - -## Enable persisting auth_state (if available). -# -# auth_state will be encrypted and stored in the Hub's database. This can -# include things like authentication tokens, etc. to be passed to Spawners as -# environment variables. -# -# Encrypting auth_state requires the cryptography package. -# -# Additionally, the JUPYTERHUB_CRYPT_KEY environment variable must contain one -# (or more, separated by ;) 32B encryption keys. These can be either base64 or -# hex-encoded. -# -# If encryption is unavailable, auth_state cannot be persisted. -# -# New in JupyterHub 0.8 -#c.Authenticator.enable_auth_state = False - -## Dictionary mapping authenticator usernames to JupyterHub users. -# -# Primarily used to normalize OAuth user names to local users. -#c.Authenticator.username_map = {} - -## Regular expression pattern that all valid usernames must match. -# -# If a username does not match the pattern specified here, authentication will -# not be attempted. -# -# If not set, allow any username. -#c.Authenticator.username_pattern = '' - -## Whitelist of usernames that are allowed to log in. -# -# Use this with supported authenticators to restrict which users can log in. -# This is an additional whitelist that further restricts users, beyond whatever -# restrictions the authenticator has in place. -# -# If empty, does not perform any additional restriction. -#c.Authenticator.whitelist = set() -c.Authenticator.whitelist = set(comma_split(os.environ.get("WHITELIST"))) - -#------------------------------------------------------------------------------ -# LocalAuthenticator(Authenticator) configuration -#------------------------------------------------------------------------------ - -## Base class for Authenticators that work with local Linux/UNIX users -# -# Checks for local users, and can attempt to create them if they exist. - -## The command to use for creating users as a list of strings -# -# For each element in the list, the string USERNAME will be replaced with the -# user's username. The username will also be appended as the final argument. -# -# For Linux, the default value is: -# -# ['adduser', '-q', '--gecos', '""', '--disabled-password'] -# -# To specify a custom home directory, set this to: -# -# ['adduser', '-q', '--gecos', '""', '--home', '/customhome/USERNAME', '-- -# disabled-password'] -# -# This will run the command: -# -# adduser -q --gecos "" --home /customhome/river --disabled-password river -# -# when the user 'river' is created. -#c.LocalAuthenticator.add_user_cmd = [] - -## If set to True, will attempt to create local system users if they do not exist -# already. -# -# Supports Linux and BSD variants only. -#c.LocalAuthenticator.create_system_users = False - -## Whitelist all users from this UNIX group. -# -# This makes the username whitelist ineffective. -#c.LocalAuthenticator.group_whitelist = set() - -#------------------------------------------------------------------------------ -# PAMAuthenticator(LocalAuthenticator) configuration -#------------------------------------------------------------------------------ - -## Authenticate local UNIX users with PAM - -## Whether to check the user's account status via PAM during authentication. -# -# The PAM account stack performs non-authentication based account management. -# It is typically used to restrict/permit access to a service and this step is -# needed to access the host's user access control. -# -# Disabling this can be dangerous as authenticated but unauthorized users may be -# granted access and, therefore, arbitrary execution on the system. -#c.PAMAuthenticator.check_account = True - -## The text encoding to use when communicating with PAM -#c.PAMAuthenticator.encoding = 'utf8' - -## Whether to open a new PAM session when spawners are started. -# -# This may trigger things like mounting shared filsystems, loading credentials, -# etc. depending on system configuration, but it does not always work. -# -# If any errors are encountered when opening/closing PAM sessions, this is -# automatically set to False. -#c.PAMAuthenticator.open_sessions = True - -## The name of the PAM service to use for authentication -#c.PAMAuthenticator.service = 'login' -c.PAMAuthenticator.service = 'jupyterhub' - -#------------------------------------------------------------------------------ -# CryptKeeper(SingletonConfigurable) configuration -#------------------------------------------------------------------------------ - -## Encapsulate encryption configuration -# -# Use via the encryption_config singleton below. - -## -#c.CryptKeeper.keys = [] - -## The number of threads to allocate for encryption -#c.CryptKeeper.n_threads = 2 diff --git a/jupyter-nersc/old/packages2.txt b/jupyter-nersc/old/packages2.txt deleted file mode 100644 index 6f668cf..0000000 --- a/jupyter-nersc/old/packages2.txt +++ /dev/null @@ -1,9 +0,0 @@ -basemap -beautiful-soup -biopython -netcdf4 -r-car -r-essentials -rpy2 -theano -ujson diff --git a/jupyter-nersc/old/packages3.txt b/jupyter-nersc/old/packages3.txt deleted file mode 100644 index 06193db..0000000 --- a/jupyter-nersc/old/packages3.txt +++ /dev/null @@ -1,7 +0,0 @@ -basemap -biopython -netcdf4 -r-car -r-essentials -rpy2 -ujson diff --git a/jupyter-nersc/old/supervisord.conf b/jupyter-nersc/old/supervisord.conf deleted file mode 100644 index c8eb242..0000000 --- a/jupyter-nersc/old/supervisord.conf +++ /dev/null @@ -1,16 +0,0 @@ -[supervisord] -nodaemon=true - -[program:jupyterhub] -directory=/srv -command=jupyterhub --debug -stdout_logfile=/dev/fd/1 -stdout_logfile_maxbytes=0 -redirect_stderr=true - -[program:nslcd] -directory=/srv -command=/usr/sbin/nslcd --nofork -stdout_logfile=/dev/fd/1 -stdout_logfile_maxbytes=0 -redirect_stderr=true