Create Conda CI test env in one step (#4935)

Issue: rapidsai/build-planning#22 Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) - Mike Sarahan (https://github.com/msarahan) Approvers: - Chuck Hastings (https://github.com/ChuckHastings) - James Lamb (https://github.com/jameslamb) URL: #4935
rapidsai · Feb 20, 2025 · 2bcf76e · 2bcf76e
1 parent b777261
commit 2bcf76e
Show file tree

Hide file tree

Showing 6 changed files with 98 additions and 72 deletions.
diff --git a/ci/build_docs.sh b/ci/build_docs.sh
@@ -1,8 +1,20 @@
 #!/bin/bash
-# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+# Copyright (c) 2023-2025, NVIDIA CORPORATION.
 
 set -euo pipefail
 
+if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then
+  CONDA_CUDA_VERSION="11.8"
+  DGL_CHANNEL="dglteam/label/th23_cu118"
+else
+  CONDA_CUDA_VERSION="12.1"
+  DGL_CHANNEL="dglteam/label/th23_cu121"
+fi
+
+rapids-logger "Downloading artifacts from previous jobs"
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
+PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
+
 rapids-logger "Create test conda environment"
 . /opt/conda/etc/profile.d/conda.sh
 
@@ -13,43 +25,19 @@ export RAPIDS_VERSION_NUMBER="$RAPIDS_VERSION_MAJOR_MINOR"
 rapids-dependency-file-generator \
   --output conda \
   --file-key docs \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \
+  --prepend-channel "${CPP_CHANNEL}" \
+  --prepend-channel "${PYTHON_CHANNEL}" \
+  --prepend-channel conda-forge \
+  --prepend-channel nvidia \
+  --prepend-channel "${DGL_CHANNEL}" \
+  | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n docs
 conda activate docs
 
 rapids-print-env
 
-rapids-logger "Downloading artifacts from previous jobs"
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
-PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
-
-if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then
-  CONDA_CUDA_VERSION="11.8"
-  DGL_CHANNEL="dglteam/label/th23_cu118"
-else
-  CONDA_CUDA_VERSION="12.1"
-  DGL_CHANNEL="dglteam/label/th23_cu121"
-fi
-
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${PYTHON_CHANNEL}" \
-  --channel conda-forge \
-  --channel nvidia \
-  --channel "${DGL_CHANNEL}" \
-  "libcugraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
-  "pylibcugraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
-  "cugraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
-  "cugraph-pyg=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
-  "cugraph-dgl=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
-  "cugraph-service-server=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
-  "cugraph-service-client=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
-  "libcugraph_etl=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
-  "pylibwholegraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
-  'pytorch>=2.3' \
-  "cuda-version=${CONDA_CUDA_VERSION}"
-
 export RAPIDS_DOCS_DIR="$(mktemp -d)"
 
 rapids-logger "Build CPP docs"

diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh
@@ -51,6 +51,8 @@ NEXT_UCXX_SHORT_TAG_PEP440=$(python -c "from packaging.version import Version; p
 DEPENDENCIES=(
   cudf
   cugraph
+  cugraph-dgl
+  cugraph-pyg
   cugraph-service-server
   cugraph-service-client
   cuxfilter

diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh
@@ -8,13 +8,15 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../
 
 . /opt/conda/etc/profile.d/conda.sh
 
-RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)"
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 
 rapids-logger "Generate C++ testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
   --file-key test_cpp \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" \
+  --prepend-channel "${CPP_CHANNEL}" \
+  | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test
 
@@ -23,19 +25,11 @@ set +u
 conda activate test
 set -u
 
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
-
 RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/
 mkdir -p "${RAPIDS_TESTS_DIR}"
 
 rapids-print-env
 
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  "libcugraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
-  "libcugraph_etl=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
-  "libcugraph-tests=${RAPIDS_VERSION_MAJOR_MINOR}.*"
-
 rapids-logger "Check GPU usage"
 nvidia-smi
 

diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh
@@ -1,17 +1,22 @@
 #!/bin/bash
-# Copyright (c) 2020-2024, NVIDIA CORPORATION.
+# Copyright (c) 2020-2025, NVIDIA CORPORATION.
 
 set -Eeuo pipefail
 
 . /opt/conda/etc/profile.d/conda.sh
 
-RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)"
+rapids-logger "Downloading artifacts from previous jobs"
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
+PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
 
 rapids-logger "Generate notebook testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
   --file-key test_notebooks \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \
+  --prepend-channel "${CPP_CHANNEL}" \
+  --prepend-channel "${PYTHON_CHANNEL}" \
+  | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test
 
@@ -22,17 +27,6 @@ set -u
 
 rapids-print-env
 
-rapids-logger "Downloading artifacts from previous jobs"
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
-PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
-
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${PYTHON_CHANNEL}" \
-  "libcugraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
-  "pylibcugraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
-  "cugraph=${RAPIDS_VERSION_MAJOR_MINOR}.*"
-
 NBTEST="$(realpath "$(dirname "$0")/utils/nbtest.sh")"
 NOTEBOOK_LIST="$(realpath "$(dirname "$0")/notebook_list.py")"
 EXITCODE=0

diff --git a/ci/test_python.sh b/ci/test_python.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2022-2024, NVIDIA CORPORATION.
+# Copyright (c) 2022-2025, NVIDIA CORPORATION.
 
 set -euo pipefail
 
@@ -8,13 +8,18 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../
 
 . /opt/conda/etc/profile.d/conda.sh
 
-RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)"
+rapids-logger "Downloading artifacts from previous jobs"
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
+PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
 
 rapids-logger "Generate Python testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
   --file-key test_python \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \
+  --prepend-channel "${CPP_CHANNEL}" \
+  --prepend-channel "${PYTHON_CHANNEL}" \
+  | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test
 
@@ -23,25 +28,12 @@ set +u
 conda activate test
 set -u
 
-rapids-logger "Downloading artifacts from previous jobs"
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
-PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
-
 RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}
 RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${PWD}/coverage-results"}
 mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}"
 
 rapids-print-env
 
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${PYTHON_CHANNEL}" \
-  "libcugraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
-  "pylibcugraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
-  "cugraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
-  "cugraph-service-server=${RAPIDS_VERSION_MAJOR_MINOR}.*" \
-  "cugraph-service-client=${RAPIDS_VERSION_MAJOR_MINOR}.*"
-
 rapids-logger "Check GPU usage"
 nvidia-smi
 

diff --git a/dependencies.yaml b/dependencies.yaml
@@ -45,11 +45,24 @@ files:
       - cuda_version
       - docs
       - py_version
+      - depends_on_libcugraph
+      - depends_on_pylibcugraph
+      - depends_on_cugraph
+      - depends_on_cugraph_pyg
+      - depends_on_cugraph_dgl
+      - depends_on_cugraph_service_server
+      - depends_on_cugraph_service_client
+      - depends_on_libcugraph_etl
+      - depends_on_pylibwholegraph
+      - depends_on_pytorch
   test_cpp:
     output: none
     includes:
       - cuda_version
       - test_cpp
+      - depends_on_libcugraph
+      - depends_on_libcugraph_etl
+      - depends_on_libcugraph_tests
   test_notebooks:
     output: none
     includes:
@@ -59,6 +72,9 @@ files:
       - test_notebook
       - test_python_common
       - test_python_cugraph
+      - depends_on_libcugraph
+      - depends_on_pylibcugraph
+      - depends_on_cugraph
   test_python:
     output: none
     includes:
@@ -70,6 +86,11 @@ files:
       - test_python_common
       - test_python_cugraph
       - test_python_pylibcugraph
+      - depends_on_libcugraph
+      - depends_on_pylibcugraph
+      - depends_on_cugraph
+      - depends_on_cugraph_service_server
+      - depends_on_cugraph_service_client
   py_build_cugraph:
     output: pyproject
     pyproject_dir: python/cugraph
@@ -517,6 +538,18 @@ dependencies:
               - cugraph-cu11==25.4.*,>=0.0.0a0
           - {matrix: null, packages: [*cugraph_unsuffixed]}
 
+  depends_on_cugraph_dgl:
+    common:
+      - output_types: conda
+        packages:
+          - cugraph-dgl==25.4.*,>=0.0.0a0
+
+  depends_on_cugraph_pyg:
+    common:
+      - output_types: conda
+        packages:
+          - cugraph-pyg==25.4.*,>=0.0.0a0
+
   depends_on_cugraph_service_client:
     common:
       - output_types: [conda, requirements, pyproject]
@@ -528,6 +561,29 @@ dependencies:
           - --extra-index-url=https://pypi.nvidia.com
           - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
 
+  depends_on_cugraph_service_server:
+    common:
+      - output_types: [conda, requirements, pyproject]
+        packages:
+          - cugraph-service-server==25.4.*,>=0.0.0a0
+      - output_types: requirements
+        packages:
+          # pip recognizes the index as a global option for the requirements.txt file
+          - --extra-index-url=https://pypi.nvidia.com
+          - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
+
+  depends_on_libcugraph_etl:
+    common:
+      - output_types: conda
+        packages:
+          - libcugraph_etl==25.4.*,>=0.0.0a0
+
+  depends_on_libcugraph_tests:
+    common:
+      - output_types: conda
+        packages:
+          - libcugraph-tests==25.4.*,>=0.0.0a0
+
   depends_on_pytorch:
     common:
       - output_types: [conda]