From 3a61a54fd250fc61d20e18f6e58297150d938054 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 28 Feb 2025 00:25:47 +0100 Subject: [PATCH] Add GPU requirements to Windows jobs: adapt Linux job tags (#1258) * Add GPU requirement to Windows jobs * Add a check to avoid non platform tags * Define docker + reliable when needed for linux jobs --------- Signed-off-by: Jose Luis Rivero --- jenkins-scripts/dsl/debian.dsl | 2 +- jenkins-scripts/dsl/dsl_checks.bash | 15 +++++++++++++++ jenkins-scripts/dsl/extra.dsl | 4 ++-- jenkins-scripts/dsl/gazebo_libs.dsl | 14 +++++++------- jenkins-scripts/dsl/gz-collections.yaml | 14 ++++++++++++++ jenkins-scripts/dsl/gzdev.dsl | 4 ++-- jenkins-scripts/dsl/ignition_collection.dsl | 8 ++++---- jenkins-scripts/dsl/ros_gz_bridge.dsl | 4 ++-- 8 files changed, 47 insertions(+), 18 deletions(-) diff --git a/jenkins-scripts/dsl/debian.dsl b/jenkins-scripts/dsl/debian.dsl index 771ea89b6..b1b22c9c0 100644 --- a/jenkins-scripts/dsl/debian.dsl +++ b/jenkins-scripts/dsl/debian.dsl @@ -8,7 +8,7 @@ OSRFLinuxBase.create(ratt_pkg_job) ratt_pkg_job.with { // use only the most powerful nodes - label Globals.nontest_label("large-memory") + label Globals.nontest_label("docker && large-memory") parameters { diff --git a/jenkins-scripts/dsl/dsl_checks.bash b/jenkins-scripts/dsl/dsl_checks.bash index 37ba05d44..64d61fe40 100755 --- a/jenkins-scripts/dsl/dsl_checks.bash +++ b/jenkins-scripts/dsl/dsl_checks.bash @@ -101,3 +101,18 @@ if [[ -n ${avoid_infinite_build_archive} ]]; then echo "${avoid_infinite_build_archive}" exit 1 fi + +check_tag_without_platforms() +{ + tag=${1} + echo $(awk -v tag="$tag" 'FNR==1{filename=FILENAME} // && $0 ~ tag && !/win/ && !/docker/ {print filename ": " $0}' *.xml) +} + +for tag in gpu-reliable large-memory; do + no_tag_without_platforms=$(check_tag_without_platforms ${tag}) + if [[ -n ${no_tag_without_platforms} ]]; then + echo "Found jobs with the ${tag}" + echo "${no_tag_without_platforms}" + exit 1 + fi +done diff --git a/jenkins-scripts/dsl/extra.dsl b/jenkins-scripts/dsl/extra.dsl index 3731dea48..97adf8a5a 100644 --- a/jenkins-scripts/dsl/extra.dsl +++ b/jenkins-scripts/dsl/extra.dsl @@ -23,7 +23,7 @@ release_repo_debbuilds.each { software -> build_pkg_job.with { // use only the most powerful nodes - label Globals.nontest_label("large-memory") + label Globals.nontest_label("docker && large-memory") steps { shell("""\ @@ -43,7 +43,7 @@ gbp_repo_debbuilds.each { software -> build_pkg_job.with { // use only the most powerful nodes - label Globals.nontest_label("large-memory") + label Globals.nontest_label("docker && large-memory") parameters { diff --git a/jenkins-scripts/dsl/gazebo_libs.dsl b/jenkins-scripts/dsl/gazebo_libs.dsl index 89520d09e..d70251720 100644 --- a/jenkins-scripts/dsl/gazebo_libs.dsl +++ b/jenkins-scripts/dsl/gazebo_libs.dsl @@ -29,12 +29,12 @@ String get_windows_distro_sortname(ci_config) + ci_config.system.version.split('_').collect { it[0] }.join('') } -void generate_label_by_requirements(job, lib_name, requirements) +void generate_label_by_requirements(job, lib_name, requirements, base_label) { if (requirements.nvidia_gpu.contains(lib_name) && requirements.large_memory.contains(lib_name)) { - println ("ERROR: more than one label is generated by requirements") + println ("ERROR: more than one label is not supported by requirements") exit(1) } @@ -46,7 +46,7 @@ void generate_label_by_requirements(job, lib_name, requirements) job.with { - label Globals.nontest_label(label) + label Globals.nontest_label("${base_label} && ${label}") if (requirements.nvidia_gpu.contains(lib_name)) { // unstable build if missing valid gpu display @@ -127,7 +127,7 @@ void generate_ci_job(gz_ci_job, lib_name, branch, ci_config, "gazebosim/${lib_name}", branch, ws_checkout_dir) - generate_label_by_requirements(gz_ci_job, lib_name, ci_config.requirements) + generate_label_by_requirements(gz_ci_job, lib_name, ci_config.requirements, 'docker') gz_ci_job.with { steps { @@ -217,7 +217,7 @@ void generate_win_ci_job(gz_win_ci_job, lib_name, branch, ci_config) "gazebosim/${lib_name}", branch, ws_checkout_dir) - + generate_label_by_requirements(gz_win_ci_job, lib_name, ci_config.requirements, 'win') add_win_devel_bat_call(gz_win_ci_job, lib_name, ws_checkout_dir, @@ -446,7 +446,7 @@ branch_index.each { lib_name, distro_configs -> is_testing_enabled(lib_name, ci_config), ENABLE_CPPCHECK, branch_names) - generate_label_by_requirements(gz_ci_any_job, lib_name, ci_config.requirements) + generate_label_by_requirements(gz_ci_any_job, lib_name, ci_config.requirements, 'docker') gz_ci_any_job.with { steps @@ -527,7 +527,7 @@ branch_index.each { lib_name, distro_configs -> GenericAnyJobGitHub.create(abi_job, "gazebosim/${lib_name}", branch_names) - generate_label_by_requirements(abi_job, lib_name, ci_config.requirements) + generate_label_by_requirements(abi_job, lib_name, ci_config.requirements, 'docker') abi_job.with { steps { diff --git a/jenkins-scripts/dsl/gz-collections.yaml b/jenkins-scripts/dsl/gz-collections.yaml index da0e7d473..01725e33e 100644 --- a/jenkins-scripts/dsl/gz-collections.yaml +++ b/jenkins-scripts/dsl/gz-collections.yaml @@ -595,6 +595,13 @@ ci_configs: version: legacy arch: amd64 requirements: + large_memory: + - gz-physics + nvidia_gpu: + - gz-sim + - gz-gui + - gz-rendering + - gz-sensors exclude: all: - gz-fortress @@ -624,6 +631,13 @@ ci_configs: version: legacy_ogre23 arch: amd64 requirements: + large_memory: + - gz-physics + nvidia_gpu: + - gz-sim + - gz-gui + - gz-rendering + - gz-sensors exclude: all: - gz-harmonic diff --git a/jenkins-scripts/dsl/gzdev.dsl b/jenkins-scripts/dsl/gzdev.dsl index ed954c4a1..b2cd6ed12 100644 --- a/jenkins-scripts/dsl/gzdev.dsl +++ b/jenkins-scripts/dsl/gzdev.dsl @@ -18,7 +18,7 @@ supported_distros.each { distro -> gzdev_ci_job.with { - label Globals.nontest_label("large-memory") + label Globals.nontest_label("docker && large-memory") scm { git { @@ -56,7 +56,7 @@ supported_distros.each { distro -> gzdev_any_job.with { - label Globals.nontest_label("large-memory") + label Globals.nontest_label("docker && large-memory") steps { shell("""\ diff --git a/jenkins-scripts/dsl/ignition_collection.dsl b/jenkins-scripts/dsl/ignition_collection.dsl index c5f42c408..357de5a1e 100644 --- a/jenkins-scripts/dsl/ignition_collection.dsl +++ b/jenkins-scripts/dsl/ignition_collection.dsl @@ -40,7 +40,7 @@ void generate_install_job(prefix, gz_collection_name, distro, arch) def dev_package = "${prefix}-${gz_collection_name}" def job_name = 'gz_launch-install-test-job.bash' - label Globals.nontest_label("gpu-reliable") + label Globals.nontest_label("docker && gpu-reliable") steps { shell("""\ @@ -98,12 +98,12 @@ gz_collections_yaml.collections.each { collection -> stringParam("ARCH", arch, "Architecture to build packages for") labelParam('JENKINS_NODE_TAG') { description('Jenkins node or group to run the build on') - defaultValue('gpu-reliable') + defaultValue('docker && gpu-reliable') } } // Designed to be run manually. No triggers. - label Globals.nontest_label("gpu-reliable") + label Globals.nontest_label("docker && gpu-reliable") steps { systemGroovyCommand("""\ @@ -127,7 +127,7 @@ gz_collections_yaml.collections.each { collection -> if [[ ${gz_collection_name} == 'citadel' || ${gz_collection_name} == 'fortress' ]]; then export GZ_SIM_RUNTIME_TEST_USE_IGN=true fi - if [[ \${JENKINS_NODE_TAG} == 'gpu-reliable' ]]; then + if [[ \${JENKINS_NODE_TAG} == '*gpu-reliable*' ]]; then export ENABLE_GZ_SIM_RUNTIME_TEST=true fi /bin/bash -x ./scripts/jenkins-scripts/docker/gz_launch-install-test-job.bash diff --git a/jenkins-scripts/dsl/ros_gz_bridge.dsl b/jenkins-scripts/dsl/ros_gz_bridge.dsl index f590d7ff9..a196f2a1d 100644 --- a/jenkins-scripts/dsl/ros_gz_bridge.dsl +++ b/jenkins-scripts/dsl/ros_gz_bridge.dsl @@ -155,12 +155,12 @@ void generate_install_test_job(Job install_test_job) stringParam("OSRF_REPOS_TO_USE", 'stable', "Repositories to add to the testing install") labelParam('JENKINS_NODE_TAG') { description('Jenkins node or group to run the build on') - defaultValue('gpu-reliable') + defaultValue('docker && gpu-reliable') } } // Designed to be run manually. No triggers. - label Globals.nontest_label("gpu-reliable") + label Globals.nontest_label("docker && gpu-reliable") steps { systemGroovyCommand("""\