Merge pull request #73 from NERSC/19-10

19 10

.gitignore

@@ -1,3 +1,4 @@
 jupyterhub.sqlite
 jupyterhub_cookie_secret
 /jupyter-compose/config
+/jupyter-compose-ssl/config

jupyter-compose-ssl/.env

@@ -0,0 +1,5 @@
+# Name of JupyterHub container data volume
+SSL_VOLUME_HOST=jupyterhub-ssl
+
+# Data volume container mount point
+SSL_VOLUME_CONTAINER=/jupyterhub-ssl

jupyter-compose-ssl/README.md

@@ -0,0 +1,34 @@
+# Developing with docker-compose
+
+These images are designed for local dev/debug/testing using docker-compose.
+This requires docker-compose.
+
+## Preparation
+
+Install Docker Compose (Just google for it).
+
+Build the images
+
+    cd web-jupyterhub && bash build.sh && cd ..
+    cd app-notebooks && bash build.sh && cd ..
+
+You may want to pull the proxy image ahead of time
+
+    docker pull jupyterhub/configurable-http-proxy
+
+Generate a key file
+
+    mkdir config
+    ssh-keygen -t rsa -N '' -C ca@localhost -f config/newkey
+    ssh-keygen -s config/newkey -h -I localhost config/newkey.pub
+
+## Bring up the containers
+
+    docker-compose up -d
+
+## Cleaning up and upgrading
+
+In general a docker-compose up -d will refresh things.  Some times that isn't enough.  If all else fails, try...
+
+    docker-compose stop
+    docker-compose rm -f

jupyter-compose-ssl/app-notebooks/Dockerfile

@@ -0,0 +1,54 @@
+ARG branch=unknown
+
+FROM registry.spin.nersc.gov/das/jupyter-base-${branch}:latest
+LABEL maintainer="Rollin Thomas <rcthomas@lbl.gov>"
+WORKDIR /tmp
+
+RUN \
+    apt-get update          &&  \
+    apt-get --yes upgrade   &&  \
+    apt-get --yes install       \
+        openssh-server
+
+# Configure sshd
+
+RUN \
+    mkdir -p /var/run/sshd
+
+# Python 3 Anaconda and additional packages
+
+RUN \
+    conda update --yes conda    &&  \
+    conda install --yes             \
+        ipykernel                   \
+        ipywidgets                  \   
+        jupyterlab                  \
+        notebook                &&  \
+    ipython kernel install      &&  \
+    conda clean --yes --all
+
+ADD get_port.py /opt/anaconda3/bin
+
+# Typical extensions
+
+RUN \
+    jupyter nbextension enable --sys-prefix --py widgetsnbextension
+
+RUN \
+    jupyter labextension install @jupyterlab/hub-extension
+
+# Some dummy users
+
+RUN \
+    adduser -q --gecos "" --disabled-password torgo     && \
+    echo torgo:the-master-would-not-approve | chpasswd
+
+RUN \
+    adduser -q --gecos "" --disabled-password master    && \
+    echo master:you-have-failed-us-torgo | chpasswd
+
+WORKDIR /srv
+ADD docker-entrypoint.sh .
+RUN chmod +x docker-entrypoint.sh
+ENTRYPOINT [ "./docker-entrypoint.sh" ]
+CMD [ "/usr/sbin/sshd", "-p", "22", "-D" ]

jupyter-compose-ssl/app-notebooks/build.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+branch=$(git symbolic-ref --short HEAD)
+
+docker build                    \
+    --build-arg branch=$branch  \
+    "$@"                        \
+    --tag app-notebooks:latest .

jupyter-compose-ssl/app-notebooks/docker-entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+for u in $(ls /home/) ; do 
+  mkdir /home/$u/.ssh/
+  cat /config/newkey.pub > /home/$u/.ssh/authorized_keys
+  chmod 700 /home/$u/.ssh/
+  chown -R $u /home/$u/.ssh
+  chmod 600 /home/$u/.ssh/authorized_keys
+done
+
+exec "$@"

jupyter-compose-ssl/app-notebooks/get_port.py

@@ -0,0 +1,23 @@
+#!/opt/anaconda3/bin/python
+
+import socket
+
+def main():
+    print(f"{ip()} {port()}")
+
+def port():
+    s = socket.socket()
+    s.bind(('', 0))
+    port = s.getsockname()[1]
+    s.close()
+    return port
+
+def ip(address=("8.8.8.8", 80)):
+    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    s.connect(address)
+    ip = s.getsockname()[0]
+    s.close()
+    return ip
+
+if __name__ == "__main__":
+    main()

jupyter-compose-ssl/docker-compose.yml

@@ -0,0 +1,67 @@
+version: "3"
+services:
+  web:
+    container_name: web
+    depends_on:
+      - proxy
+    image: web-jupyterhub:latest
+    environment:
+      - CONFIGPROXY_AUTH_TOKEN=the-sign-pointed-this-way
+      - ANNOUNCEMENT_JUPYTERHUB_API_TOKEN=the-hands-of-fate-have-doomed-this-man
+      - INTERNAL_SSL_PATH=${SSL_VOLUME_CONTAINER}
+    volumes:
+      - ./config:/config
+      - "ssl:${SSL_VOLUME_CONTAINER}:rw"
+    ports:
+      - 8081:8081
+    restart: unless-stopped
+    user: root
+  app:
+    container_name: app
+    image: app-notebooks:latest
+    volumes:
+      - ./config:/config
+#     - "ssl:${SSL_VOLUME_CONTAINER}:ro"
+  announcement:
+    container_name: announcement
+    image: web-announcement:latest
+    environment:
+      - JUPYTERHUB_API_TOKEN=the-hands-of-fate-have-doomed-this-man
+      - JUPYTERHUB_API_URL=https://web:8081/hub/api
+      - JUPYTERHUB_SSL_KEYFILE=/jupyterhub-ssl/hub-internal/hub-internal.key
+      - JUPYTERHUB_SSL_CERTFILE=/jupyterhub-ssl/hub-internal/hub-internal.crt
+      - JUPYTERHUB_SSL_CLIENT_CA=/jupyterhub-ssl/hub-ca_trust.crt
+    restart: always
+    volumes:
+      - "ssl:${SSL_VOLUME_CONTAINER}:ro"
+  proxy:
+    command: >
+      configurable-http-proxy
+        --port 8000
+        --api-ip 0.0.0.0
+        --api-port 8001
+        --error-target https://web:8081/hub/error
+        --api-ssl-key ${SSL_VOLUME_CONTAINER}/proxy-api/proxy-api.key
+        --api-ssl-cert ${SSL_VOLUME_CONTAINER}/proxy-api/proxy-api.crt
+        --api-ssl-ca ${SSL_VOLUME_CONTAINER}/proxy-api-ca_trust.crt
+        --api-ssl-request-cert
+        --api-ssl-reject-unauthorized
+        --client-ssl-key ${SSL_VOLUME_CONTAINER}/proxy-client/proxy-client.key
+        --client-ssl-cert ${SSL_VOLUME_CONTAINER}/proxy-client/proxy-client.crt
+        --client-ssl-ca ${SSL_VOLUME_CONTAINER}/proxy-client-ca_trust.crt
+        --client-ssl-request-cert
+        --client-ssl-reject-unauthorized
+    container_name: proxy
+    image: jupyterhub/configurable-http-proxy:latest
+    environment:
+      - CONFIGPROXY_AUTH_TOKEN=the-sign-pointed-this-way
+    ports:
+      - 8000:8000
+    restart: always
+    user: root
+    volumes:
+      - "ssl:${SSL_VOLUME_CONTAINER}:ro"
+volumes:
+  ssl:
+    external:
+      name: ${SSL_VOLUME_HOST}

jupyter-compose-ssl/web-announcement/Dockerfile

@@ -0,0 +1,14 @@
+ARG branch=unknown
+
+FROM registry.spin.nersc.gov/das/jupyter-base-${branch}:latest
+LABEL maintainer="Rollin Thomas <rcthomas@lbl.gov>"
+
+RUN \
+    pip install git+https://github.com/rcthomas/jupyterhub-announcement.git@0.4.1
+
+WORKDIR /srv
+
+ADD docker-entrypoint.sh announcement_config.py ./
+RUN chmod +x docker-entrypoint.sh
+ENTRYPOINT ["./docker-entrypoint.sh"]
+CMD ["python", "-m", "jupyterhub_announcement"]

jupyter-compose-ssl/web-announcement/announcement_config.py

@@ -0,0 +1,72 @@
+# Configuration file for application.
+
+#------------------------------------------------------------------------------
+# Application(SingletonConfigurable) configuration
+#------------------------------------------------------------------------------
+
+## This is an application.
+
+## The date format used by logging formatters for %(asctime)s
+#c.Application.log_datefmt = '%Y-%m-%d %H:%M:%S'
+
+## The Logging format template
+#c.Application.log_format = '[%(name)s]%(highlevel)s %(message)s'
+
+## Set the log level by value or name.
+#c.Application.log_level = 30
+c.Application.log_level = 0
+
+#------------------------------------------------------------------------------
+# AnnouncementService(Application) configuration
+#------------------------------------------------------------------------------
+
+## This is an application.
+
+## Config file to load
+#c.AnnouncementService.config_file = 'announcement_config.py'
+
+## Fixed message to show at the top of the page.
+#  
+#  A good use for this parameter would be a link to a more general live system
+#  status page or MOTD.
+#c.AnnouncementService.fixed_message = ''
+
+## Generate default config file
+#c.AnnouncementService.generate_config = False
+
+## Logo path, can be used to override JupyterHub one
+#c.AnnouncementService.logo_file = ''
+
+## Port this service will listen on
+#c.AnnouncementService.port = 8888
+
+## Announcement service prefix
+#c.AnnouncementService.service_prefix = '/services/announcement/'
+
+## Search paths for jinja templates, coming before default ones
+#c.AnnouncementService.template_paths = []
+
+#------------------------------------------------------------------------------
+# AnnouncementQueue(LoggingConfigurable) configuration
+#------------------------------------------------------------------------------
+
+## File path where announcements persist as JSON.
+#  
+#  For a persistent announcement queue, this parameter must be set to a non-empty
+#  value and correspond to a read+write-accessible path. The announcement queue
+#  is stored as a list of JSON objects. If this parameter is set to a non-empty
+#  value:
+#  
+#  * The persistence file is used to initialize the announcement queue
+#    at start-up. This is the only time the persistence file is read.
+#  * If the persistence file does not exist at start-up, it is
+#    created when an announcement is added to the queue.
+#  * The persistence file is over-written with the contents of the
+#    announcement queue each time a new announcement is added.
+#  
+#  If this parameter is set to an empty value (the default) then the queue is
+#  just empty at initialization and the queue is ephemeral; announcements will
+#  not be persisted on updates to the queue.
+#c.AnnouncementQueue.persist_path = ''
+c.AnnouncementQueue.persist_path = 'announcements.json'
+

jupyter-compose-ssl/web-announcement/build.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+branch=$(git symbolic-ref --short HEAD)
+
+docker build                    \
+    --build-arg branch=$branch  \
+    "$@"                        \
+    --tag web-announcement:latest .

jupyter-compose-ssl/web-announcement/docker-entrypoint.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# file_env VAR [DEFAULT]
+# ----------------------
+# Treat the value of VAR_FILE as the path to a secrets file and initialize VAR
+# with the contents of that file.  From postgres docker-entrypoint.sh.
+
+file_env() {
+    local var="$1"
+    local fileVar="${var}_FILE"
+    local def="${2:-}"
+    if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+        echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
+        exit 1
+    fi
+    local val="$def"
+    if [ "${!var:-}" ]; then
+        val="${!var}"
+    elif [ "${!fileVar:-}" ]; then
+        val="$(< "${!fileVar}")"
+    fi
+    export "$var"="$val"
+    unset "$fileVar"
+}
+
+file_env 'JUPYTERHUB_API_TOKEN'
+
+exec "$@"

jupyter-compose-ssl/web-jupyterhub/Dockerfile

@@ -0,0 +1,30 @@
+ARG branch=unknown
+
+FROM registry.spin.nersc.gov/das/jupyter-base-${branch}:latest
+LABEL maintainer="Rollin Thomas <rcthomas@lbl.gov>"
+
+# JupyterHub components
+
+RUN \
+    pip install git+https://github.com/kellyrowland/sshspawner.git@ssl
+#   pip install git+https://github.com/NERSC/sshspawner.git@clean-up
+
+# Some dummy users
+
+RUN \
+    adduser -q --gecos "" --disabled-password torgo     && \
+    echo torgo:the-master-would-not-approve | chpasswd
+
+RUN \
+    adduser -q --gecos "" --disabled-password master    && \
+    echo master:you-have-failed-us-torgo | chpasswd
+
+WORKDIR /srv
+ADD docker-entrypoint.sh .
+ADD jupyterhub_config.py .
+ADD templates templates
+RUN chmod +x docker-entrypoint.sh
+RUN echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
+
+CMD ["jupyterhub", "--debug"]
+ENTRYPOINT ["./docker-entrypoint.sh"]

jupyter-compose-ssl/web-jupyterhub/build.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+branch=$(git symbolic-ref --short HEAD)
+
+docker build                    \
+    --build-arg branch=$branch  \
+    "$@"                        \
+    --tag web-jupyterhub:latest .

jupyter-compose-ssl/web-jupyterhub/docker-entrypoint.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+for u in $(ls /home/) ; do
+  cp /config/newkey /tmp/$u.key
+  chmod 600 /tmp/$u.key
+  cp /config/newkey-cert.pub /tmp/$u.key-cert.pub
+  chmod 600 /tmp/$u.key-cert.pub
+done
+
+exec "$@"