Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add GPU requirements to Windows jobs: adapt Linux job tags #1258

Merged
merged 9 commits into from
Feb 27, 2025
2 changes: 1 addition & 1 deletion jenkins-scripts/dsl/debian.dsl
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ OSRFLinuxBase.create(ratt_pkg_job)
ratt_pkg_job.with
{
// use only the most powerful nodes
label Globals.nontest_label("large-memory")
label Globals.nontest_label("docker && large-memory")

parameters
{
Expand Down
15 changes: 15 additions & 0 deletions jenkins-scripts/dsl/dsl_checks.bash
Original file line number Diff line number Diff line change
Expand Up @@ -101,3 +101,18 @@ if [[ -n ${avoid_infinite_build_archive} ]]; then
echo "${avoid_infinite_build_archive}"
exit 1
fi

check_tag_without_platforms()
{
tag=${1}
echo $(awk -v tag="$tag" 'FNR==1{filename=FILENAME} /<assignedNode>/ && $0 ~ tag && !/win/ && !/docker/ {print filename ": " $0}' *.xml)
}

for tag in gpu-reliable large-memory; do
no_tag_without_platforms=$(check_tag_without_platforms ${tag})
if [[ -n ${no_tag_without_platforms} ]]; then
echo "Found jobs with the ${tag}"
echo "${no_tag_without_platforms}"
exit 1
fi
done
4 changes: 2 additions & 2 deletions jenkins-scripts/dsl/extra.dsl
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ release_repo_debbuilds.each { software ->
build_pkg_job.with
{
// use only the most powerful nodes
label Globals.nontest_label("large-memory")
label Globals.nontest_label("docker && large-memory")

steps {
shell("""\
Expand All @@ -43,7 +43,7 @@ gbp_repo_debbuilds.each { software ->
build_pkg_job.with
{
// use only the most powerful nodes
label Globals.nontest_label("large-memory")
label Globals.nontest_label("docker && large-memory")

parameters
{
Expand Down
14 changes: 7 additions & 7 deletions jenkins-scripts/dsl/gazebo_libs.dsl
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,12 @@ String get_windows_distro_sortname(ci_config)
+ ci_config.system.version.split('_').collect { it[0] }.join('')
}

void generate_label_by_requirements(job, lib_name, requirements)
void generate_label_by_requirements(job, lib_name, requirements, base_label)
{

if (requirements.nvidia_gpu.contains(lib_name) &&
requirements.large_memory.contains(lib_name)) {
println ("ERROR: more than one label is generated by requirements")
println ("ERROR: more than one label is not supported by requirements")
exit(1)
}

Expand All @@ -46,7 +46,7 @@ void generate_label_by_requirements(job, lib_name, requirements)

job.with
{
label Globals.nontest_label(label)
label Globals.nontest_label("${base_label} && ${label}")

if (requirements.nvidia_gpu.contains(lib_name)) {
// unstable build if missing valid gpu display
Expand Down Expand Up @@ -127,7 +127,7 @@ void generate_ci_job(gz_ci_job, lib_name, branch, ci_config,
"gazebosim/${lib_name}",
branch,
ws_checkout_dir)
generate_label_by_requirements(gz_ci_job, lib_name, ci_config.requirements)
generate_label_by_requirements(gz_ci_job, lib_name, ci_config.requirements, 'docker')
gz_ci_job.with
{
steps {
Expand Down Expand Up @@ -217,7 +217,7 @@ void generate_win_ci_job(gz_win_ci_job, lib_name, branch, ci_config)
"gazebosim/${lib_name}",
branch,
ws_checkout_dir)

generate_label_by_requirements(gz_win_ci_job, lib_name, ci_config.requirements, 'win')
add_win_devel_bat_call(gz_win_ci_job,
lib_name,
ws_checkout_dir,
Expand Down Expand Up @@ -446,7 +446,7 @@ branch_index.each { lib_name, distro_configs ->
is_testing_enabled(lib_name, ci_config),
ENABLE_CPPCHECK,
branch_names)
generate_label_by_requirements(gz_ci_any_job, lib_name, ci_config.requirements)
generate_label_by_requirements(gz_ci_any_job, lib_name, ci_config.requirements, 'docker')
gz_ci_any_job.with
{
steps
Expand Down Expand Up @@ -527,7 +527,7 @@ branch_index.each { lib_name, distro_configs ->
GenericAnyJobGitHub.create(abi_job,
"gazebosim/${lib_name}",
branch_names)
generate_label_by_requirements(abi_job, lib_name, ci_config.requirements)
generate_label_by_requirements(abi_job, lib_name, ci_config.requirements, 'docker')
abi_job.with
{
steps {
Expand Down
14 changes: 14 additions & 0 deletions jenkins-scripts/dsl/gz-collections.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -595,6 +595,13 @@ ci_configs:
version: legacy
arch: amd64
requirements:
large_memory:
- gz-physics
nvidia_gpu:
- gz-sim
- gz-gui
- gz-rendering
- gz-sensors
exclude:
all:
- gz-fortress
Expand Down Expand Up @@ -624,6 +631,13 @@ ci_configs:
version: legacy_ogre23
arch: amd64
requirements:
large_memory:
- gz-physics
nvidia_gpu:
- gz-sim
- gz-gui
- gz-rendering
- gz-sensors
exclude:
all:
- gz-harmonic
Expand Down
4 changes: 2 additions & 2 deletions jenkins-scripts/dsl/gzdev.dsl
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ supported_distros.each { distro ->
gzdev_ci_job.with
{

label Globals.nontest_label("large-memory")
label Globals.nontest_label("docker && large-memory")

scm {
git {
Expand Down Expand Up @@ -56,7 +56,7 @@ supported_distros.each { distro ->

gzdev_any_job.with
{
label Globals.nontest_label("large-memory")
label Globals.nontest_label("docker && large-memory")

steps {
shell("""\
Expand Down
8 changes: 4 additions & 4 deletions jenkins-scripts/dsl/ignition_collection.dsl
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ void generate_install_job(prefix, gz_collection_name, distro, arch)
def dev_package = "${prefix}-${gz_collection_name}"
def job_name = 'gz_launch-install-test-job.bash'

label Globals.nontest_label("gpu-reliable")
label Globals.nontest_label("docker && gpu-reliable")

steps {
shell("""\
Expand Down Expand Up @@ -98,12 +98,12 @@ gz_collections_yaml.collections.each { collection ->
stringParam("ARCH", arch, "Architecture to build packages for")
labelParam('JENKINS_NODE_TAG') {
description('Jenkins node or group to run the build on')
defaultValue('gpu-reliable')
defaultValue('docker && gpu-reliable')
}
}

// Designed to be run manually. No triggers.
label Globals.nontest_label("gpu-reliable")
label Globals.nontest_label("docker && gpu-reliable")

steps {
systemGroovyCommand("""\
Expand All @@ -127,7 +127,7 @@ gz_collections_yaml.collections.each { collection ->
if [[ ${gz_collection_name} == 'citadel' || ${gz_collection_name} == 'fortress' ]]; then
export GZ_SIM_RUNTIME_TEST_USE_IGN=true
fi
if [[ \${JENKINS_NODE_TAG} == 'gpu-reliable' ]]; then
if [[ \${JENKINS_NODE_TAG} == '*gpu-reliable*' ]]; then
export ENABLE_GZ_SIM_RUNTIME_TEST=true
fi
/bin/bash -x ./scripts/jenkins-scripts/docker/gz_launch-install-test-job.bash
Expand Down
4 changes: 2 additions & 2 deletions jenkins-scripts/dsl/ros_gz_bridge.dsl
Original file line number Diff line number Diff line change
Expand Up @@ -155,12 +155,12 @@ void generate_install_test_job(Job install_test_job)
stringParam("OSRF_REPOS_TO_USE", 'stable', "Repositories to add to the testing install")
labelParam('JENKINS_NODE_TAG') {
description('Jenkins node or group to run the build on')
defaultValue('gpu-reliable')
defaultValue('docker && gpu-reliable')
}
}

// Designed to be run manually. No triggers.
label Globals.nontest_label("gpu-reliable")
label Globals.nontest_label("docker && gpu-reliable")

steps {
systemGroovyCommand("""\
Expand Down