From 83fc94066d68a2511ede1cddf3f86d241561941e Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Wed, 10 Apr 2024 16:07:13 +0200 Subject: [PATCH] Update and fix gzdev jobs in Jenkins (#1128) * Modernize DSL distributions * Update and fix the docker-in-docker support until the point of being able to run rocker * Remove old docker install code --------- Signed-off-by: Jose Luis Rivero --- .../docker/lib/_install_nvidia_docker.sh | 22 +------- .../lib/docker_generate_dockerfile.bash | 12 +++++ .../docker/lib/gzdev-base-linux.bash | 50 +++++++++---------- jenkins-scripts/dsl/gzdev.dsl | 5 +- 4 files changed, 42 insertions(+), 47 deletions(-) diff --git a/jenkins-scripts/docker/lib/_install_nvidia_docker.sh b/jenkins-scripts/docker/lib/_install_nvidia_docker.sh index 8e766acbf..517665ceb 100644 --- a/jenkins-scripts/docker/lib/_install_nvidia_docker.sh +++ b/jenkins-scripts/docker/lib/_install_nvidia_docker.sh @@ -1,22 +1,7 @@ -INSTALL_NVIDIA_DOCKER1=""" -echo '# BEGIN SECTION: install docker (in docker)' -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - -sudo add-apt-repository \"deb [arch=amd64] https://download.docker.com/linux/ubuntu \$(lsb_release -cs) stable\" -sudo apt-get update -sudo apt-get install -y docker-ce -echo '# END SECTION' - -echo '# BEGIN SECTION: install nvidia-docker1 (in docker)' -sudo apt-get install -y wget nvidia-340 nvidia-modprobe -wget -P /tmp https://github.com/NVIDIA/nvidia-docker/releases/download/v1.0.1/nvidia-docker_1.0.1-1_amd64.deb -sudo dpkg -i /tmp/nvidia-docker*.deb && rm /tmp/nvidia-docker*.deb -echo '# END SECTION' -""" - INSTALL_NVIDIA_DOCKER2=""" echo '# BEGIN SECTION: install docker (in docker)' curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - -sudo add-apt-repository \"deb [arch=amd64] https://download.docker.com/linux/ubuntu \$(lsb_release -cs) stable\" +sudo add-apt-repository -y \"deb [arch=amd64] https://download.docker.com/linux/ubuntu \$(lsb_release -cs) stable\" sudo apt-get update # sudo apt-get install -y docker-ce echo '# END SECTION' @@ -29,11 +14,8 @@ curl -s -L https://nvidia.github.io/nvidia-docker/\$distribution/nvidia-docker.l sudo tee /etc/apt/sources.list.d/nvidia-docker.list sudo apt-get update sudo apt-get install -y nvidia-docker2 -# systemctl daemon-reload -# systemctl restart docker +sudo docker info echo '# END SECTION' """ -DOCKER2_CMD="sudo docker run --privileged -e DISPLAY=unix:0 -v /sys:/sys:ro -v /var/run/docker.sock:/var/run/docker.sock -v /tmp/.X11-unix:/tmp/.X11-unix:rw --runtime=nvidia -e DOCKER_FIX= -v /dev/log:/dev/log:ro -v /run/log:/run/log:ro -v /sys/fs/cgroup:/sys/fs/cgroup:ro --device /dev/snd --tty --rm" - DEPENDENCY_PKGS="${DEPENDENCY_PKGS} curl software-properties-common" diff --git a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash index e7f66c419..b78f8f432 100644 --- a/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash +++ b/jenkins-scripts/docker/lib/docker_generate_dockerfile.bash @@ -413,6 +413,13 @@ DELIM_DISPLAY fi fi +if ${USE_DOCKER_IN_DOCKER}; then +cat >> Dockerfile << DELIM_WORKAROUND_DOCKER_IN_DOCKER_HOOK +# Avoid ERROR: invoke-rc.d: policy-rc.d denied execution of start. +RUN sed -i "s/^exit 101$/exit 0/" /usr/sbin/policy-rc.d +DELIM_WORKAROUND_DOCKER_IN_DOCKER_HOOK +fi + if [ `expr length "${DOCKER_POSTINSTALL_HOOK}"` -gt 1 ]; then cat >> Dockerfile << DELIM_WORKAROUND_POST_HOOK RUN ${DOCKER_POSTINSTALL_HOOK} @@ -468,11 +475,16 @@ RUN adduser --uid \$USERID --gid \$GID --gecos "Developer" --disabled-password \ RUN adduser \$USER sudo RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers RUN chown -R \$USER:\$USER /home/\$USER +# Needed if USE_DOCKER_IN_DOCKER is active. Harmless to be here +RUN groupadd docker +RUN gpasswd -a \$USER docker +RUN newgrp docker # permit access to USER variable inside docker ENV USER \$USER USER \$USER # Must use sudo where necessary from this point on + DELIM_DOCKER_USER if [[ -n ${SOFTWARE_DIR} ]]; then diff --git a/jenkins-scripts/docker/lib/gzdev-base-linux.bash b/jenkins-scripts/docker/lib/gzdev-base-linux.bash index 9acbe12f8..7deb198a9 100644 --- a/jenkins-scripts/docker/lib/gzdev-base-linux.bash +++ b/jenkins-scripts/docker/lib/gzdev-base-linux.bash @@ -13,49 +13,49 @@ set -ex export MAKE_JOBS=${MAKE_JOBS} export DISPLAY=${DISPLAY} -${INSTALL_NVIDIA_DOCKER1} +${INSTALL_NVIDIA_DOCKER2} echo '# BEGIN SECTION: install pip requirements' cd ${WORKSPACE}/gzdev -pip3 install -r requirements.txt +# pip3 install -r requirements.txt echo '# END SECTION' -echo '# BEGIN SECTION: run gzdev for gazebo8 with nvidia' -cd ${WORKSPACE}/gzdev -./gzdev.py spawn --gzv=8 --nvidia -echo '# END SECTION' +echo '# BEGIN SECTION: smoke tests for ign-docker-env' +sudo pip3 install git+https://github.com/adlarkin/ign-rocker.git -echo '# BEGIN SECTION: Disply log file.' -cat ./gz8.log -echo '# END SECTION' +# TODO: rocker can not play well docker-in-docker installations +# out of the box. Needs more work. + +# xvfb :1 -ac -noreset -core -screen 0 1280x1024x24 & +# export display=:1.0 +# export mesa_gl_version_override=3.3 + +# test_timeout=300 +# test_start=\`date +%s\` +# sudo bash -c "timeout --preserve-status \$test_timeout ./gzdev.py ign-docker-env dome --linux-distro ubuntu:bionic" +# test_end=\`date +%s\` +# diff=\$(expr \$test_end - \$test_start) -echo '# BEGIN SECTION: check that gazebo is running' -gazebo_detection=false -seconds_waiting=0 -cat gz8.log | grep "Connected to gazebo master" && gazebo_detection=true -while (! \$gazebo_detection); do - sleep 1 - docker top gz8 | grep gazebo && gazebo_detection=true - docker top gz8 | grep gzserver && gazebo_detection=true - seconds_waiting=\$((seconds_waiting+1)) - [ \$seconds_waiting -gt 30 ] && break -done -# clean up gazebo instances -docker rm -f gz8 || true -killall -9 gazebo gzserver gzclient || true -! \${gazebo_detection} && exit 1 +# if [ \$diff -lt \$test_timeout ]; then +# echo 'the test took less than \$test_timeout. something bad happened' +# exit 1 +# fi echo '# END SECTION' DELIM export USE_DOCKER_IN_DOCKER=true export OSRF_REPOS_TO_USE="stable" +export USE_ROS_REPO=true +export ROS2=true export DEPENDENCY_PKGS="python3-pip \ bash \ apt-transport-https \ ca-certificates \ curl \ software-properties-common \ - psmisc" # killall + python3-rocker \ + psmisc \ + xvfb" . "${SCRIPT_DIR}/lib/docker_generate_dockerfile.bash" . "${SCRIPT_DIR}/lib/docker_run.bash" diff --git a/jenkins-scripts/dsl/gzdev.dsl b/jenkins-scripts/dsl/gzdev.dsl index 1137eac9b..ed954c4a1 100644 --- a/jenkins-scripts/dsl/gzdev.dsl +++ b/jenkins-scripts/dsl/gzdev.dsl @@ -1,7 +1,7 @@ import _configs_.* import javaposse.jobdsl.dsl.Job -def supported_distros = [ 'bionic' ] +def supported_distros = [ 'jammy' ] def supported_arches = [ 'amd64' ] @@ -17,6 +17,8 @@ supported_distros.each { distro -> gzdev_ci_job.with { + + label Globals.nontest_label("large-memory") scm { git { @@ -54,7 +56,6 @@ supported_distros.each { distro -> gzdev_any_job.with { - // use only the most powerful nodes label Globals.nontest_label("large-memory") steps {