From f9b7094b1eb1ad64b16435c51a698e44be11d4c5 Mon Sep 17 00:00:00 2001 From: "C.A.P. Linssen" Date: Wed, 16 Oct 2024 11:48:45 +0200 Subject: [PATCH] add notes about Jupyter-JSC --- vm_infra/create_custom_kernel_sandra.sh | 164 ++++++++++++++++++ ...nfrastructure_ocns_2024_jsc_jupyterhub.rst | 116 +++++++++++++ 2 files changed, 280 insertions(+) create mode 100644 vm_infra/create_custom_kernel_sandra.sh create mode 100644 vm_infra/notes_cloud_infrastructure_ocns_2024_jsc_jupyterhub.rst diff --git a/vm_infra/create_custom_kernel_sandra.sh b/vm_infra/create_custom_kernel_sandra.sh new file mode 100644 index 0000000..a3edb70 --- /dev/null +++ b/vm_infra/create_custom_kernel_sandra.sh @@ -0,0 +1,164 @@ +# INPUT NEEDED: +KERNEL_NAME=ocns_2024_nest_nestml_kernel + +export KERNEL_NAME=$(echo "${KERNEL_NAME}" | awk '{print tolower($0)}') +echo ${KERNEL_NAME} # double check + +# JUPYTER SEARCH PATH (for kernels-directory) +echo "jupyter search paths for kernels-directories" +if [ -z $JUPYTER_PATH ]; then + echo "$HOME/.local/share/jupyter" +else + tr ':' '\n' <<< "$JUPYTER_PATH" +fi + +# INPUT NEEDED: +export KERNEL_TYPE=project # private, project or other +export KERNEL_SPECS_PREFIX=${HOME}/.local + +################### +# project kernel +if [ "${KERNEL_TYPE}" == "project" ]; then + export KERNEL_SPECS_PREFIX=${PROJECT}/.local + echo "project kernel" +# private kernel +elif [ "${KERNEL_TYPE}" == "private" ]; then + export KERNEL_SPECS_PREFIX=${HOME}/.local + echo "private kernel" +else + if [ ! -d "$KERNEL_SPECS_PREFIX" ]; then + echo "ERROR: please create directory $KERNEL_SPECS_PREFIX" + fi + echo "other kernel" +fi +export KERNEL_SPECS_DIR=${KERNEL_SPECS_PREFIX}/share/jupyter/kernels + +# check if kernel name is unique +if [ -d "${KERNEL_SPECS_DIR}/${KERNEL_NAME}" ]; then + echo "ERROR: Kernel already exists in ${KERNEL_SPECS_DIR}/${KERNEL_NAME}" + echo " Rename kernel name or remove directory." +fi + +echo ${KERNEL_SPECS_DIR}/${KERNEL_NAME} # double check + +# INPUT NEEDED: +export KERNEL_VENVS_DIR=${PROJECT}/${USER}/jupyter/kernels + +################### +mkdir -p ${KERNEL_VENVS_DIR} +if [ "${KERNEL_TYPE}" != "private" ] && [ "${KERNEL_TYPE}" != "other" ]; then + echo "Please check the permissions and ensure your project partners have read/execute permissions:" + namei -l ${KERNEL_VENVS_DIR} +fi + +echo ${KERNEL_VENVS_DIR} # double check +ls -lt ${KERNEL_VENVS_DIR} + +module -q purge +module -q use $OTHERSTAGES +# XXX: was /2022 originally! +module -q load Stages/2024 +module -q load GCCcore/.12.3.0 +module -q load GCC +module -q load ParaStationMPI +module -q load CMake +# XXX: was without a version originally +module -q load Python +module -q load SciPy-Stack +module -q load GSL +module -q load mpi4py +if [ -d "${KERNEL_VENVS_DIR}/${KERNEL_NAME}" ]; then + echo "ERROR: Directory for virtual environment already ${KERNEL_VENVS_DIR}/${KERNEL_NAME}" + echo " Rename kernel name or remove directory." +else + python -m venv --system-site-packages ${KERNEL_VENVS_DIR}/${KERNEL_NAME} + source ${KERNEL_VENVS_DIR}/${KERNEL_NAME}/bin/activate + export PYTHONPATH=/p/project1/training2422/linssen1/site-packages:${VIRTUAL_ENV}/lib/python3.11/site-packages:${PYTHONPATH} + echo "Virtual environment initialized to: " + echo ${VIRTUAL_ENV} # double check +fi + +which pip +if [ -z "${VIRTUAL_ENV}" ]; then + echo "ERROR: Virtual environment not successfully initialized." +else + pip install --prefix ${VIRTUAL_ENV} --ignore-installed ipykernel + ls ${VIRTUAL_ENV}/lib/python3.11/site-packages/ # double check +fi + +# --prefix /p/project1/training2422/linssen1/site-packages +pip install --prefix ${VIRTUAL_ENV} scikit-learn +pip install --prefix ${VIRTUAL_ENV} tqdm +pip install --prefix ${VIRTUAL_ENV} git+https://github.com/nest/ode-toolbox +pip install --prefix ${VIRTUAL_ENV} PyNN +# parameters package is needed for the sequence learning tutorial! +pip install --prefix ${VIRTUAL_ENV} parameters jupytext +# pip install --user neatdend + +pip install --prefix ${VIRTUAL_ENV} git+https://github.com/clinssen/nestml@sequence_learning_and_compartmental_vectorization + +pip install --prefix ${VIRTUAL_ENV} git+https://github.com/WillemWybo/NEAT-2@enh/nestml-channel-generation +pip install --prefix ${VIRTUAL_ENV} /p/project1/training2422/linssen1/NEATmodels-master-2024-07-15.zip + +#cd nestml +#python setup.py install + +echo '#!/bin/bash'" + +# Load basic Python module +module -q purge +module -q use $OTHERSTAGES +module -q load Stages/2024 +module -q load GCCcore/.12.3.0 +module -q load GCC +module -q load ParaStationMPI +module -q load CMake +module -q load Python +module -q load SciPy-Stack +module -q load GSL +module -q load mpi4py +export PYTHONPATH=$PYTHONPATH:/p/project1/training2422/linssen1/site-packages:/p/project1/training2422/linssen1/nest/nest-simulator-install/lib64/python3.11/site-packages +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/p/project1/training2422/linssen1/nest/nest-simulator-install/lib64/nest + +# Load extra modules you need for your kernel (as you did in step 1.2) +#module load + +# Activate your Python virtual environment +source ${KERNEL_VENVS_DIR}/${KERNEL_NAME}/bin/activate + +# Ensure python packages installed in the virtual environment are always prefered +export PYTHONPATH=/p/project1/training2422/linssen1/site-packages:${VIRTUAL_ENV}/lib/python3.11/site-packages:"'${PYTHONPATH}'" + +exec python -m ipykernel "'$@' > ${VIRTUAL_ENV}/kernel.sh +chmod +x ${VIRTUAL_ENV}/kernel.sh + +cat ${VIRTUAL_ENV}/kernel.sh # double check + +python -m ipykernel install --name=${KERNEL_NAME} --prefix ${VIRTUAL_ENV} +export VIRTUAL_ENV_KERNELS=${VIRTUAL_ENV}/share/jupyter/kernels + +mv ${VIRTUAL_ENV_KERNELS}/${KERNEL_NAME}/kernel.json ${VIRTUAL_ENV_KERNELS}/${KERNEL_NAME}/kernel.json.orig + +echo '{ + "argv": [ + "'${KERNEL_VENVS_DIR}/${KERNEL_NAME}/kernel.sh'", + "-m", + "ipykernel_launcher", + "-f", + "{connection_file}" + ], + "display_name": "'${KERNEL_NAME}'", + "language": "python" +}' > ${VIRTUAL_ENV_KERNELS}/${KERNEL_NAME}/kernel.json + +cat ${VIRTUAL_ENV_KERNELS}/${KERNEL_NAME}/kernel.json # double check + +mkdir -p ${KERNEL_SPECS_DIR} +cd ${KERNEL_SPECS_DIR} +echo ${VIRTUAL_ENV_KERNELS}/${KERNEL_NAME} +ln -s ${VIRTUAL_ENV_KERNELS}/${KERNEL_NAME} . + +echo -e "\n\nThe new kernel '${KERNEL_NAME}' was added to your kernels in '${KERNEL_SPECS_DIR}/'\n" +ls ${KERNEL_SPECS_DIR} # double check + +deactivate diff --git a/vm_infra/notes_cloud_infrastructure_ocns_2024_jsc_jupyterhub.rst b/vm_infra/notes_cloud_infrastructure_ocns_2024_jsc_jupyterhub.rst new file mode 100644 index 0000000..0c68c78 --- /dev/null +++ b/vm_infra/notes_cloud_infrastructure_ocns_2024_jsc_jupyterhub.rst @@ -0,0 +1,116 @@ +- APPLY FOR RESOURCES + + https://dispatch.fz-juelich.de:8817/kurs_login/back=/ABBRUCH + + Since we have 40 participants, you could ask for ~ 60,000 core-hours in total on the CPU. + + ie. 128 cores per compute node on JUSUF X 12 hours X 40 nodes ~= 60,000 + + - JUSUF Compute nodes(HPC accounts) --> no internet access but more powerful + - JSC Cloud --> On the cloud, there is no need to go through the JuDoor. There is a possibility to have a list of usernames and passwords in advance. Compute resources quite limited (RAM/CPU). + + +- JOIN TRAINING PROJECT + + https://judoor.fz-juelich.de/login + + enter the training2422 project. Make sure it shows up under "JUSUF_CPU". + + +- LAUNCH THE SERVER + + https://jupyter.jsc.fz-juelich.de + (log into own account) + + new -> + JupyterLab 4.2 + JUSUF + linssen1 + use the training2422 project + + +- CREATE CUSTOM KERNEL + + First, compile NEST. + + Use the same modules as in the custom kernel build script. Also do a + + ml Boost/1.82.0 + + Clone and build: + + mkdir /p/project1/training2422/linssen1/nest + cd /p/project1/training2422/linssen1/nest + git clone https://github.com/nest/nest-simulator + mkdir /p/project1/training2422/linssen1/nest/nest-simulator-build + cd /p/project1/training2422/linssen1/nest/nest-simulator-build + cmake -DCMAKE_INSTALL_PREFIX:PATH=../nest-simulator-install ../nest-simulator + make -j32 install + + See https://github.com/FZJ-JSC/jupyter-jsc-notebooks/tree/documentation/03-HowTos for Sandra's script creating a custom kernel. + + delete the old kernel: + + rm -rv /p/project1/training2422/linssen1/jupyter/kernels/ocns_2024_nest_nestml_kernel + rm -rv /p/project1/training2422/.local/share/jupyter/kernels/ocns_2024_nest_nestml_kernel + + > I have used this one in the past: https://github.com/FZJ-JSC/jupyter-jsc-notebooks/blob/documentation/03-HowTos/Create_JupyterKernel_general.ipynb + + > The example that I implemented, which might be a bit outdated, is here: https://gitlab.jsc.fz-juelich.de/metaopt/freiburg2023/-/blob/main/create_kernel.sh + + > The user just needs to do ./create_kernel.sh from the command line the first time from their home directory, then all is configured. Just a quick note on this script, I made a shared installation of nest in a specific path, I specify this in the script for the kernel in lines 106 and 107 where I say: + + export PYTHONPATH=$PYTHONPATH:/p/project/training2222/nest-simulator/build/install/lib64/python3.9/site-packages:/p/project/training2222/L2L/:/p/project/training2222/sdict/:/p/project/training2222/JUBE/:/p/project/training2222/JUBE/bin/ + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/p/project/training2222/JUBE/bin/ + + you can do a similar thing for nest and nestml. + + + $ echo $PROJECT + /p/project1/training2422 + + Edit the script and give the kernel a name and set type to project. Add custom pip install commands, etc. Then run: + + $ ./create_custom_kernel_sandra.sh + + In my case it printed something like: + + jupyter search paths for kernels-directories + /p/project1/training2422/.local/share/jupyter + project kernel + /p/project1/training2422/.local/share/jupyter/kernels/ocns_2024_nest_nestml_kernel + + ... + + The new kernel 'ocns_2024_nest_nestml_kernel' was added to your kernels in '/p/project1/training2422/.local/share/jupyter/kernels/' + + +- POST KERNEL SETUP STEPS + + Run the following code in a new notebook under your own account name: + + import neat + import neatmodels + import nest + neatmodels.load_or_install_neuron_bbp_channels() + neatmodels.load_or_install_nest_bbp_channels() + + + +- DISTRIBUTE CUSTOM KERNEL + + You could place the kernel in $PROJECT/.local (in the same way as it would be in $HOME/.local) and makes it readable to every project member so that everyone will see your kernel right away + + Check for a list of kernels: + + ls -l $PROJECT/.local/share/jupyter/kernels + + +- INVITE STUDENTS + + for the instructions sheet: https://clinssen.github.io/OCNS-2024-NEST-workshop/ + + for the invite link, see JuDoor + + e.g. of the form: https://judoor.fz-juelich.de/projects/join/training2422 +