consolidate more conda solves in CI (#18014)

Contributes to rapidsai/build-planning#22 Follow-up to #17995 * ensures all of the conda solves in CI are consolidated * ensures the locally-downloaded files are considered in conda solves # Authors: - James Lamb (https://github.com/jameslamb) - Matthew Murray (https://github.com/Matt711) Approvers: - Kyle Edwards (https://github.com/KyleFromNVIDIA) URL: #18014
rapidsai · Feb 15, 2025 · e497a1e · e497a1e
1 parent 4e37806
commit e497a1e
Show file tree

Hide file tree

Showing 7 changed files with 42 additions and 52 deletions.
diff --git a/ci/build_docs.sh b/ci/build_docs.sh
@@ -13,28 +13,22 @@ rapids-logger "Create test conda environment"
 
 ENV_YAML_DIR="$(mktemp -d)"
 
+rapids-logger "Downloading artifacts from previous jobs"
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
+PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
+
 rapids-dependency-file-generator \
   --output conda \
   --file-key docs \
+  --prepend-channel "${CPP_CHANNEL}" \
+  --prepend-channel "${PYTHON_CHANNEL}" \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${ENV_YAML_DIR}/env.yaml"
 
 rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n docs
 conda activate docs
 
 rapids-print-env
 
-rapids-logger "Downloading artifacts from previous jobs"
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
-PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
-
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${PYTHON_CHANNEL}" \
-  "libcudf=${RAPIDS_VERSION}" \
-  "pylibcudf=${RAPIDS_VERSION}" \
-  "cudf=${RAPIDS_VERSION}" \
-  "dask-cudf=${RAPIDS_VERSION}"
-
 RAPIDS_DOCS_DIR="$(mktemp -d)"
 export RAPIDS_DOCS_DIR
 

diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh
@@ -51,6 +51,9 @@ DEPENDENCIES=(
   dask-cudf
   kvikio
   libcudf
+  libcudf-example
+  libcudf_kafka
+  libcudf-tests
   libkvikio
   librmm
   pylibcudf

diff --git a/ci/test_cpp_common.sh b/ci/test_cpp_common.sh
@@ -1,11 +1,12 @@
 #!/bin/bash
-# Copyright (c) 2022-2024, NVIDIA CORPORATION.
+# Copyright (c) 2022-2025, NVIDIA CORPORATION.
 
 set -euo pipefail
 
 . /opt/conda/etc/profile.d/conda.sh
 
-RAPIDS_VERSION="$(rapids-version)"
+rapids-logger "Downloading artifacts from previous jobs"
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 
 rapids-logger "Generate C++ testing dependencies"
 
@@ -14,6 +15,7 @@ ENV_YAML_DIR="$(mktemp -d)"
 rapids-dependency-file-generator \
   --output conda \
   --file-key test_cpp \
+  --prepend-channel "${CPP_CHANNEL}" \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml"
 
 rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n test
@@ -23,20 +25,11 @@ set +u
 conda activate test
 set -u
 
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
-
 RESULTS_DIR=${RAPIDS_TESTS_DIR:-"$(mktemp -d)"}
 RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${RESULTS_DIR}/test-results"}/
 mkdir -p "${RAPIDS_TESTS_DIR}"
 
 rapids-print-env
 
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  "libcudf=${RAPIDS_VERSION}" \
-  "libcudf_kafka=${RAPIDS_VERSION}" \
-  "libcudf-tests=${RAPIDS_VERSION}" \
-  "libcudf-example=${RAPIDS_VERSION}"
-
 rapids-logger "Check GPU usage"
 nvidia-smi
diff --git a/ci/test_java.sh b/ci/test_java.sh
@@ -1,11 +1,12 @@
 #!/bin/bash
-# Copyright (c) 2022-2024, NVIDIA CORPORATION.
+# Copyright (c) 2022-2025, NVIDIA CORPORATION.
 
 set -euo pipefail
 
 . /opt/conda/etc/profile.d/conda.sh
 
-RAPIDS_VERSION="$(rapids-version)"
+rapids-logger "Downloading artifacts from previous jobs"
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 
 rapids-logger "Generate Java testing dependencies"
 
@@ -14,6 +15,7 @@ ENV_YAML_DIR="$(mktemp -d)"
 rapids-dependency-file-generator \
   --output conda \
   --file-key test_java \
+  --prepend-channel "${CPP_CHANNEL}" \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml"
 
 rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n test
@@ -27,13 +29,6 @@ set -u
 
 rapids-print-env
 
-rapids-logger "Downloading artifacts from previous jobs"
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
-
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  "libcudf=${RAPIDS_VERSION}"
-
 rapids-logger "Check GPU usage"
 nvidia-smi
 

diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh
@@ -5,7 +5,9 @@ set -euo pipefail
 
 . /opt/conda/etc/profile.d/conda.sh
 
-RAPIDS_VERSION="$(rapids-version)"
+rapids-logger "Downloading artifacts from previous jobs"
+CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
+PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
 
 rapids-logger "Generate notebook testing dependencies"
 
@@ -14,6 +16,8 @@ ENV_YAML_DIR="$(mktemp -d)"
 rapids-dependency-file-generator \
   --output conda \
   --file-key test_notebooks \
+  --prepend-channel "${CPP_CHANNEL}" \
+  --prepend-channel "${PYTHON_CHANNEL}" \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${ENV_YAML_DIR}/env.yaml"
 
 rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n test
@@ -25,16 +29,6 @@ set -u
 
 rapids-print-env
 
-rapids-logger "Downloading artifacts from previous jobs"
-CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
-PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
-
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${PYTHON_CHANNEL}" \
-  "cudf=${RAPIDS_VERSION}" \
-  "libcudf=${RAPIDS_VERSION}"
-
 NBTEST="$(realpath "$(dirname "$0")/utils/nbtest.sh")"
 pushd notebooks
 

diff --git a/ci/test_python_common.sh b/ci/test_python_common.sh
@@ -7,8 +7,6 @@ set -euo pipefail
 
 . /opt/conda/etc/profile.d/conda.sh
 
-RAPIDS_VERSION="$(rapids-version)"
-
 rapids-logger "Downloading artifacts from previous jobs"
 CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
@@ -20,6 +18,8 @@ FILE_KEY=$1
 rapids-dependency-file-generator \
   --output conda \
   --file-key "${FILE_KEY}" \
+  --prepend-channel "${CPP_CHANNEL}" \
+  --prepend-channel "${PYTHON_CHANNEL}" \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};dependencies=${RAPIDS_DEPENDENCIES}" \
     | tee "${ENV_YAML_DIR}/env.yaml"
 
@@ -36,10 +36,3 @@ RAPIDS_COVERAGE_DIR=${RAPIDS_COVERAGE_DIR:-"${RESULTS_DIR}/coverage-results"}
 mkdir -p "${RAPIDS_TESTS_DIR}" "${RAPIDS_COVERAGE_DIR}"
 
 rapids-print-env
-
-rapids-mamba-retry install \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${PYTHON_CHANNEL}" \
-  "cudf=${RAPIDS_VERSION}" \
-  "pylibcudf=${RAPIDS_VERSION}" \
-  "libcudf=${RAPIDS_VERSION}"
diff --git a/dependencies.yaml b/dependencies.yaml
@@ -55,7 +55,9 @@ files:
     output: none
     includes:
       - cuda_version
+      - depends_on_libcudf
       - test_cpp
+      - test_cpp_cudf
   test_python_cudf_pandas:
     output: none
     includes:
@@ -98,11 +100,14 @@ files:
       - build_all
       - cuda
       - cuda_version
+      - depends_on_libcudf
       - test_java
   test_notebooks:
     output: none
     includes:
       - cuda_version
+      - depends_on_cudf
+      - depends_on_libcudf
       - notebooks
       - py_version
   checks:
@@ -125,6 +130,10 @@ files:
     includes:
       - cuda
       - cuda_version
+      - depends_on_cudf
+      - depends_on_dask_cudf
+      - depends_on_pylibcudf
+      - depends_on_libcudf
       - docs
       - py_version
   py_build_cudf:
@@ -835,6 +844,15 @@ dependencies:
               - cuda-sanitizer-api=11.8.86
           - matrix:  # Fallback for CUDA 11 or no matrix
             packages:
+  # packages we want in the 'test_cpp' group in 'files', for CI, but which
+  # shouldn't be added to 'all' for building a development environment
+  test_cpp_cudf:
+    common:
+      - output_types: conda
+        packages:
+          - libcudf-example==25.4.*,>=0.0.0a0
+          - libcudf_kafka==25.4.*,>=0.0.0a0
+          - libcudf-tests==25.4.*,>=0.0.0a0
   test_java:
     common:
       - output_types: conda