add notes about Jupyter-JSC

vm_infra/create_custom_kernel_sandra.sh

@@ -0,0 +1,164 @@
+# INPUT NEEDED:
+KERNEL_NAME=ocns_2024_nest_nestml_kernel
+
+export KERNEL_NAME=$(echo "${KERNEL_NAME}" | awk '{print tolower($0)}')
+echo ${KERNEL_NAME} # double check
+
+# JUPYTER SEARCH PATH (for kernels-directory)
+echo "jupyter search paths for kernels-directories"
+if [ -z $JUPYTER_PATH ]; then
+  echo "$HOME/.local/share/jupyter"
+else
+  tr ':' '\n' <<< "$JUPYTER_PATH"
+fi
+
+# INPUT NEEDED:
+export KERNEL_TYPE=project # private, project or other
+export KERNEL_SPECS_PREFIX=${HOME}/.local
+
+###################
+# project kernel
+if [ "${KERNEL_TYPE}" == "project" ]; then
+  export KERNEL_SPECS_PREFIX=${PROJECT}/.local
+  echo "project kernel"
+# private kernel
+elif [ "${KERNEL_TYPE}" == "private" ]; then
+  export KERNEL_SPECS_PREFIX=${HOME}/.local
+  echo "private kernel"
+else
+  if [ ! -d "$KERNEL_SPECS_PREFIX" ]; then
+    echo "ERROR: please create directory $KERNEL_SPECS_PREFIX"
+  fi
+  echo "other kernel"
+fi
+export KERNEL_SPECS_DIR=${KERNEL_SPECS_PREFIX}/share/jupyter/kernels
+
+# check if kernel name is unique
+if [ -d "${KERNEL_SPECS_DIR}/${KERNEL_NAME}" ]; then
+  echo "ERROR: Kernel already exists in ${KERNEL_SPECS_DIR}/${KERNEL_NAME}"
+  echo "       Rename kernel name or remove directory."
+fi
+
+echo ${KERNEL_SPECS_DIR}/${KERNEL_NAME} # double check
+
+# INPUT NEEDED:
+export KERNEL_VENVS_DIR=${PROJECT}/${USER}/jupyter/kernels
+
+###################
+mkdir -p ${KERNEL_VENVS_DIR}
+if [ "${KERNEL_TYPE}" != "private" ] && [ "${KERNEL_TYPE}" != "other" ]; then
+  echo "Please check the permissions and ensure your project partners have read/execute permissions:"
+  namei -l ${KERNEL_VENVS_DIR}
+fi
+
+echo ${KERNEL_VENVS_DIR} # double check
+ls -lt ${KERNEL_VENVS_DIR}
+
+module -q purge
+module -q use $OTHERSTAGES
+# XXX: was /2022 originally!
+module -q load Stages/2024
+module -q load GCCcore/.12.3.0
+module -q load GCC
+module -q load ParaStationMPI
+module -q load CMake
+# XXX: was without a version originally
+module -q load Python
+module -q load SciPy-Stack
+module -q load GSL
+module -q load mpi4py
+if [ -d "${KERNEL_VENVS_DIR}/${KERNEL_NAME}" ]; then
+  echo "ERROR: Directory for virtual environment already ${KERNEL_VENVS_DIR}/${KERNEL_NAME}"
+  echo "       Rename kernel name or remove directory."
+else
+  python -m venv --system-site-packages ${KERNEL_VENVS_DIR}/${KERNEL_NAME}
+  source ${KERNEL_VENVS_DIR}/${KERNEL_NAME}/bin/activate
+  export PYTHONPATH=/p/project1/training2422/linssen1/site-packages:${VIRTUAL_ENV}/lib/python3.11/site-packages:${PYTHONPATH}
+  echo "Virtual environment initialized to: "
+  echo ${VIRTUAL_ENV} # double check
+fi
+
+which pip
+if [ -z "${VIRTUAL_ENV}" ]; then
+  echo "ERROR: Virtual environment not successfully initialized."
+else
+  pip install --prefix ${VIRTUAL_ENV} --ignore-installed ipykernel
+  ls ${VIRTUAL_ENV}/lib/python3.11/site-packages/ # double check
+fi
+
+# --prefix /p/project1/training2422/linssen1/site-packages
+pip install --prefix ${VIRTUAL_ENV} scikit-learn
+pip install --prefix ${VIRTUAL_ENV} tqdm
+pip install --prefix ${VIRTUAL_ENV} git+https://github.com/nest/ode-toolbox
+pip install --prefix ${VIRTUAL_ENV} PyNN
+# parameters package is needed for the sequence learning tutorial!
+pip install --prefix ${VIRTUAL_ENV} parameters jupytext
+# pip install --user neatdend
+
+pip install --prefix ${VIRTUAL_ENV} git+https://github.com/clinssen/nestml@sequence_learning_and_compartmental_vectorization
+
+pip install --prefix ${VIRTUAL_ENV} git+https://github.com/WillemWybo/NEAT-2@enh/nestml-channel-generation
+pip install --prefix ${VIRTUAL_ENV} /p/project1/training2422/linssen1/NEATmodels-master-2024-07-15.zip
+
+#cd nestml
+#python setup.py install
+
+echo '#!/bin/bash'"
+
+# Load basic Python module
+module -q purge
+module -q use $OTHERSTAGES        
+module -q load Stages/2024
+module -q load GCCcore/.12.3.0
+module -q load GCC
+module -q load ParaStationMPI
+module -q load CMake
+module -q load Python
+module -q load SciPy-Stack
+module -q load GSL
+module -q load mpi4py
+export PYTHONPATH=$PYTHONPATH:/p/project1/training2422/linssen1/site-packages:/p/project1/training2422/linssen1/nest/nest-simulator-install/lib64/python3.11/site-packages
+export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/p/project1/training2422/linssen1/nest/nest-simulator-install/lib64/nest
+
+# Load extra modules you need for your kernel (as you did in step 1.2)
+#module load <module you need>
+    
+# Activate your Python virtual environment
+source ${KERNEL_VENVS_DIR}/${KERNEL_NAME}/bin/activate
+    
+# Ensure python packages installed in the virtual environment are always prefered
+export PYTHONPATH=/p/project1/training2422/linssen1/site-packages:${VIRTUAL_ENV}/lib/python3.11/site-packages:"'${PYTHONPATH}'"
+
+exec python -m ipykernel "'$@' > ${VIRTUAL_ENV}/kernel.sh
+chmod +x ${VIRTUAL_ENV}/kernel.sh
+
+cat ${VIRTUAL_ENV}/kernel.sh # double check
+
+python -m ipykernel install --name=${KERNEL_NAME} --prefix ${VIRTUAL_ENV}
+export VIRTUAL_ENV_KERNELS=${VIRTUAL_ENV}/share/jupyter/kernels
+
+mv ${VIRTUAL_ENV_KERNELS}/${KERNEL_NAME}/kernel.json ${VIRTUAL_ENV_KERNELS}/${KERNEL_NAME}/kernel.json.orig
+
+echo '{
+  "argv": [
+    "'${KERNEL_VENVS_DIR}/${KERNEL_NAME}/kernel.sh'",
+    "-m",
+    "ipykernel_launcher",
+    "-f",
+    "{connection_file}"
+  ],
+  "display_name": "'${KERNEL_NAME}'",
+  "language": "python"
+}' > ${VIRTUAL_ENV_KERNELS}/${KERNEL_NAME}/kernel.json
+
+cat ${VIRTUAL_ENV_KERNELS}/${KERNEL_NAME}/kernel.json # double check
+
+mkdir -p ${KERNEL_SPECS_DIR}
+cd ${KERNEL_SPECS_DIR}
+echo ${VIRTUAL_ENV_KERNELS}/${KERNEL_NAME}
+ln -s ${VIRTUAL_ENV_KERNELS}/${KERNEL_NAME} .
+
+echo -e "\n\nThe new kernel '${KERNEL_NAME}' was added to your kernels in '${KERNEL_SPECS_DIR}/'\n"
+ls ${KERNEL_SPECS_DIR} # double check
+
+deactivate

vm_infra/notes_cloud_infrastructure_ocns_2024_jsc_jupyterhub.rst

@@ -0,0 +1,116 @@
+- APPLY FOR RESOURCES
+
+	https://dispatch.fz-juelich.de:8817/kurs_login/back=/ABBRUCH
+
+	Since we have 40 participants, you could ask for ~ 60,000 core-hours in total on the CPU.
+
+	ie. 128 cores per compute node on JUSUF X 12 hours X 40 nodes ~= 60,000
+
+	- JUSUF Compute nodes(HPC accounts) --> no internet access but more powerful
+	- JSC Cloud --> On the cloud, there is no need to go through the JuDoor. There is a possibility to have a list of usernames and passwords in advance. Compute resources quite limited (RAM/CPU).
+
+
+- JOIN TRAINING PROJECT
+
+	https://judoor.fz-juelich.de/login
+
+	enter the training2422 project. Make sure it shows up under "JUSUF_CPU".
+
+
+- LAUNCH THE SERVER
+
+	https://jupyter.jsc.fz-juelich.de
+	(log into own account)
+
+	new ->
+		JupyterLab 4.2
+		JUSUF
+		linssen1
+		use the training2422 project
+
+
+- CREATE CUSTOM KERNEL
+
+	First, compile NEST.
+
+	Use the same modules as in the custom kernel build script. Also do a 
+
+		ml Boost/1.82.0
+
+	Clone and build:
+
+		mkdir /p/project1/training2422/linssen1/nest
+		cd /p/project1/training2422/linssen1/nest
+		git clone https://github.com/nest/nest-simulator
+		mkdir /p/project1/training2422/linssen1/nest/nest-simulator-build
+		cd /p/project1/training2422/linssen1/nest/nest-simulator-build
+		cmake -DCMAKE_INSTALL_PREFIX:PATH=../nest-simulator-install ../nest-simulator
+		make -j32 install
+
+	See https://github.com/FZJ-JSC/jupyter-jsc-notebooks/tree/documentation/03-HowTos for Sandra's script creating a custom kernel.
+
+	delete the old kernel:
+
+		rm -rv /p/project1/training2422/linssen1/jupyter/kernels/ocns_2024_nest_nestml_kernel
+		rm -rv /p/project1/training2422/.local/share/jupyter/kernels/ocns_2024_nest_nestml_kernel
+
+	> I have used this one in the past: https://github.com/FZJ-JSC/jupyter-jsc-notebooks/blob/documentation/03-HowTos/Create_JupyterKernel_general.ipynb
+
+	> The example that I implemented, which might be a bit outdated, is here: https://gitlab.jsc.fz-juelich.de/metaopt/freiburg2023/-/blob/main/create_kernel.sh
+
+	> The user just needs to do ./create_kernel.sh from the command line the first time from their home directory, then all is configured. Just a quick note on this script, I made a shared installation of nest in a specific path, I specify this in the script for the kernel in lines 106 and 107 where I say:
+
+		export PYTHONPATH=$PYTHONPATH:/p/project/training2222/nest-simulator/build/install/lib64/python3.9/site-packages:/p/project/training2222/L2L/:/p/project/training2222/sdict/:/p/project/training2222/JUBE/:/p/project/training2222/JUBE/bin/
+		export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/p/project/training2222/JUBE/bin/
+
+	you can do a similar thing for nest and nestml.
+
+
+	$ echo $PROJECT
+	/p/project1/training2422
+
+	Edit the script and give the kernel a name and set type to project. Add custom pip install commands, etc. Then run:
+
+	$ ./create_custom_kernel_sandra.sh
+
+	In my case it printed something like:
+
+	jupyter search paths for kernels-directories
+	/p/project1/training2422/.local/share/jupyter
+	project kernel
+	/p/project1/training2422/.local/share/jupyter/kernels/ocns_2024_nest_nestml_kernel
+
+	...
+
+	The new kernel 'ocns_2024_nest_nestml_kernel' was added to your kernels in '/p/project1/training2422/.local/share/jupyter/kernels/'
+
+
+- POST KERNEL SETUP STEPS
+
+	Run the following code in a new notebook under your own account name:
+
+		import neat
+		import neatmodels
+		import nest
+		neatmodels.load_or_install_neuron_bbp_channels()
+		neatmodels.load_or_install_nest_bbp_channels()
+
+
+
+- DISTRIBUTE CUSTOM KERNEL
+
+	You could place the kernel in $PROJECT/.local  (in the same way as it would be in $HOME/.local) and makes it readable to every project member so that everyone will see your kernel right away
+
+	Check for a list of kernels:
+
+		ls -l $PROJECT/.local/share/jupyter/kernels
+
+
+- INVITE STUDENTS
+
+	for the instructions sheet: https://clinssen.github.io/OCNS-2024-NEST-workshop/
+
+	for the invite link, see JuDoor
+
+	e.g. of the form: https://judoor.fz-juelich.de/projects/join/training2422
+