From c2ef5c696d4d71ed0323bd347cdc0cc235ce9dba Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Wed, 5 Feb 2025 09:06:00 -0500 Subject: [PATCH 01/51] Add rattler-build recipe for libcudf --- ci/build_cpp.sh | 19 +- conda/recipes/libcudf/build.sh | 9 - conda/recipes/libcudf/install_libcudf.sh | 4 - .../libcudf/install_libcudf_example.sh | 5 - .../recipes/libcudf/install_libcudf_kafka.sh | 4 - .../recipes/libcudf/install_libcudf_tests.sh | 5 - conda/recipes/libcudf/meta.yaml | 220 ----------------- conda/recipes/libcudf/recipe.yaml | 233 ++++++++++++++++++ python/libcudf/pyproject.toml | 2 +- 9 files changed, 251 insertions(+), 250 deletions(-) delete mode 100644 conda/recipes/libcudf/build.sh delete mode 100644 conda/recipes/libcudf/install_libcudf.sh delete mode 100644 conda/recipes/libcudf/install_libcudf_example.sh delete mode 100644 conda/recipes/libcudf/install_libcudf_kafka.sh delete mode 100644 conda/recipes/libcudf/install_libcudf_tests.sh delete mode 100644 conda/recipes/libcudf/meta.yaml create mode 100644 conda/recipes/libcudf/recipe.yaml diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 0c324d01cdf..f7a7d41b342 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -18,9 +18,24 @@ rapids-logger "Begin cpp build" sccache --zero-stats # With boa installed conda build forward to boa -RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry build \ - conda/recipes/libcudf +RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) +export RAPIDS_PACKAGE_VERSION + +source rapids-rattler-channel-string + +# --no-build-id allows for caching with `sccache` +# more info is available at +# https://rattler.build/latest/tips_and_tricks/#using-sccache-or-ccache-with-rattler-build +rattler-build build --recipe conda/recipes/libcudf \ + --experimental \ + --no-build-id \ + --channel-priority disabled \ + --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ + "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats +# remove build_cache directory +rm -rf "$RAPIDS_CONDA_BLD_OUTPUT_DIR"/build_cache + rapids-upload-conda-to-s3 cpp diff --git a/conda/recipes/libcudf/build.sh b/conda/recipes/libcudf/build.sh deleted file mode 100644 index a3a0415575b..00000000000 --- a/conda/recipes/libcudf/build.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018-2024, NVIDIA CORPORATION. - -export cudf_ROOT="$(realpath ./cpp/build)" - -./build.sh -n -v \ - libcudf libcudf_kafka benchmarks tests \ - --build_metrics --incl_cache_stats --allgpuarch \ - --cmake-args=\"-DCMAKE_INSTALL_LIBDIR=lib -DCUDF_ENABLE_ARROW_S3=ON\" diff --git a/conda/recipes/libcudf/install_libcudf.sh b/conda/recipes/libcudf/install_libcudf.sh deleted file mode 100644 index 173f8cfa90f..00000000000 --- a/conda/recipes/libcudf/install_libcudf.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018-2022, NVIDIA CORPORATION. - -cmake --install cpp/build diff --git a/conda/recipes/libcudf/install_libcudf_example.sh b/conda/recipes/libcudf/install_libcudf_example.sh deleted file mode 100644 index 1a52dec99e3..00000000000 --- a/conda/recipes/libcudf/install_libcudf_example.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018-2024, NVIDIA CORPORATION. - -# build and install libcudf examples -./cpp/examples/build.sh --install diff --git a/conda/recipes/libcudf/install_libcudf_kafka.sh b/conda/recipes/libcudf/install_libcudf_kafka.sh deleted file mode 100644 index 9eae2510027..00000000000 --- a/conda/recipes/libcudf/install_libcudf_kafka.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018-2022, NVIDIA CORPORATION. - -cmake --install cpp/libcudf_kafka/build diff --git a/conda/recipes/libcudf/install_libcudf_tests.sh b/conda/recipes/libcudf/install_libcudf_tests.sh deleted file mode 100644 index 069462eec9d..00000000000 --- a/conda/recipes/libcudf/install_libcudf_tests.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018-2022, NVIDIA CORPORATION. - -cmake --install cpp/build --component testing -cmake --install cpp/libcudf_kafka/build --component testing diff --git a/conda/recipes/libcudf/meta.yaml b/conda/recipes/libcudf/meta.yaml deleted file mode 100644 index f7bd7280f0f..00000000000 --- a/conda/recipes/libcudf/meta.yaml +++ /dev/null @@ -1,220 +0,0 @@ -# Copyright (c) 2018-2025, NVIDIA CORPORATION. - -{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} -{% set cuda_major = cuda_version.split('.')[0] %} -{% set date_string = environ['RAPIDS_DATE_STRING'] %} - -package: - name: libcudf-split - -source: - path: ../../.. - -build: - script_env: - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN - - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CUDA_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - - CMAKE_GENERATOR - - PARALLEL_LEVEL - - RAPIDS_ARTIFACTS_DIR - - SCCACHE_BUCKET - - SCCACHE_IDLE_TIMEOUT - - SCCACHE_REGION - - SCCACHE_S3_KEY_PREFIX=libcudf-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=libcudf-linux64 # [linux64] - - SCCACHE_S3_USE_SSL - - SCCACHE_S3_NO_CREDENTIALS - -requirements: - build: - - cmake {{ cmake_version }} - - {{ compiler('c') }} - - {{ compiler('cxx') }} - {% if cuda_major == "11" %} - - {{ compiler('cuda') }} ={{ cuda_version }} - {% else %} - - {{ compiler('cuda') }} - {% endif %} - - cuda-version ={{ cuda_version }} - - ninja - - {{ stdlib("c") }} - host: - - librmm ={{ minor_version }} - - libkvikio ={{ minor_version }} - {% if cuda_major == "11" %} - - cudatoolkit - - libcufile {{ cuda11_libcufile_host_version }} # [linux64] - - libcufile-dev {{ cuda11_libcufile_host_version }} # [linux64] - - libcurand {{ cuda11_libcurand_host_version }} - - libcurand-dev {{ cuda11_libcurand_host_version }} - - cuda-nvrtc ={{ cuda_version }} - - cuda-nvrtc-dev ={{ cuda_version }} - - cuda-nvtx ={{ cuda_version }} - {% else %} - - cuda-nvrtc-dev - - cuda-nvtx-dev - - libcufile-dev # [linux64] - - libcurand-dev - {% endif %} - - cuda-version ={{ cuda_version }} - - nvcomp {{ nvcomp_version }} - - dlpack {{ dlpack_version }} - - librdkafka {{ librdkafka_version }} - - flatbuffers {{ flatbuffers_version }} - - rapids-logger =0.1 - - zlib {{ zlib_version }} - -outputs: - - name: libcudf - version: {{ version }} - script: install_libcudf.sh - build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - run_exports: - - {{ pin_subpackage("libcudf", max_pin="x.x") }} - ignore_run_exports_from: - - {{ compiler('cuda') }} - requirements: - build: - - cmake {{ cmake_version }} - host: - - cuda-version ={{ cuda_version }} - run: - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - {% if cuda_major == "11" %} - - cudatoolkit - - libcufile {{ cuda11_libcufile_run_version }} # [linux64] - {% else %} - - cuda-nvrtc - - libcufile # [linux64] - {% endif %} - - nvcomp {{ nvcomp_version }} - - librmm ={{ minor_version }} - - libkvikio ={{ minor_version }} - - dlpack {{ dlpack_version }} - - rapids-logger =0.1 - test: - commands: - - test -f $PREFIX/lib/libcudf.so - - test -f $PREFIX/include/cudf/column/column.hpp - about: - home: https://rapids.ai/ - license: Apache-2.0 - license_family: APACHE - license_file: LICENSE - summary: libcudf library - - name: libcudf_kafka - version: {{ version }} - script: install_libcudf_kafka.sh - build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - ignore_run_exports_from: - - {{ compiler('cuda') }} - requirements: - build: - - cmake {{ cmake_version }} - host: - - librdkafka {{ librdkafka_version }} - - {{ pin_subpackage('libcudf', exact=True) }} - run: - - librdkafka {{ librdkafka_version }} - - {{ pin_subpackage('libcudf', exact=True) }} - test: - commands: - - test -f $PREFIX/lib/libcudf_kafka.so - about: - home: https://rapids.ai/ - license: Apache-2.0 - license_family: APACHE - license_file: LICENSE - summary: libcudf_kafka library - - name: libcudf-example - version: {{ version }} - script: install_libcudf_example.sh - build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - ignore_run_exports_from: - - {{ compiler('cuda') }} - {% if cuda_major != "11" %} - - cuda-nvtx-dev - {% endif %} - requirements: - build: - - cmake {{ cmake_version }} - - {{ compiler('c') }} - - {{ compiler('cxx') }} - {% if cuda_major == "11" %} - - {{ compiler('cuda') }} ={{ cuda_version }} - {% else %} - - {{ compiler('cuda') }} - {% endif %} - - cuda-version ={{ cuda_version }} - - ninja - - {{ stdlib("c") }} - host: - - {{ pin_subpackage('libcudf', exact=True) }} - {% if cuda_major == "11" %} - - cuda-nvtx ={{ cuda_version }} - {% else %} - - cuda-nvtx-dev - {% endif %} - - cuda-version ={{ cuda_version }} - run: - - {{ pin_subpackage('libcudf', exact=True) }} - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - {% if cuda_major != "11" %} - - cuda-nvtx - {% endif %} - about: - home: https://rapids.ai/ - license: Apache-2.0 - license_family: APACHE - license_file: LICENSE - summary: libcudf example executables - - name: libcudf-tests - version: {{ version }} - script: install_libcudf_tests.sh - build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - ignore_run_exports_from: - - {{ compiler('cuda') }} - {% if cuda_major != "11" %} - - libcurand-dev - {% endif %} - requirements: - build: - - cmake {{ cmake_version }} - host: - - {{ pin_subpackage('libcudf', exact=True) }} - - {{ pin_subpackage('libcudf_kafka', exact=True) }} - - cuda-version ={{ cuda_version }} - {% if cuda_major == "11" %} - - libcurand {{ cuda11_libcurand_run_version }} - {% else %} - - libcurand-dev - {% endif %} - run: - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - - {{ pin_subpackage('libcudf', exact=True) }} - - {{ pin_subpackage('libcudf_kafka', exact=True) }} - {% if cuda_major == "11" %} - - libcurand {{ cuda11_libcurand_run_version }} - {% else %} - - libcurand - {% endif %} - about: - home: https://rapids.ai/ - license: Apache-2.0 - license_family: APACHE - license_file: LICENSE - summary: libcudf test & benchmark executables diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml new file mode 100644 index 00000000000..6a3704614ee --- /dev/null +++ b/conda/recipes/libcudf/recipe.yaml @@ -0,0 +1,233 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/prefix-dev/recipe-format/main/schema.json +# Copyright (c) 2018-2025, NVIDIA CORPORATION. +schema_version: 1 + +context: + version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} + minor_version: ${{ (version | split('.'))[:2] | join('.') }} + cuda_version: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[:2] | join(".") }} + cuda_major: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[0] }} + date_string: ${{ env.get("RAPIDS_DATE_STRING") }} + head_rev: ${{ git.head_rev(".")[:8] }} + +recipe: + name: libcudf-split + +cache: + source: + path: ../../.. + + requirements: + build: + - ${{ compiler('c') }} + - ${{ compiler('cxx') }} + - ${{ compiler('cuda') }} =${{ cuda_version }} + - cuda-version =${{ cuda_version }} + - cmake ${{ cmake_version }} + - ninja + - ${{ stdlib("c") }} + host: + - librmm =${{ minor_version }} + - libkvikio =${{ minor_version }} + - if: cuda_major == "11" + then: + - cudatoolkit + - libcufile ${{ cuda11_libcublas_host_version }} + - libcufile-dev ${{ cuda11_libcublas_host_version }} + - libcurand ${{ cuda11_libcurand_host_version }} + - libcurand-dev ${{ cuda11_libcurand_host_version }} + - cuda-nvrtc =${{ cuda_version }} + - cuda-nvrtc-dev =${{ cuda_version }} + - cuda-nvtx =${{ cuda_version }} + else: + - cuda-nvrtc-dev + - cuda-nvtx-dev + - libcurand-dev + - if: linux64 + then: + - libcufile-dev + - cuda-version =${{ cuda_version }} + - nvcomp ${{ nvcomp_version }} + - dlpack ${{ dlpack_version }} + - librdkafka ${{ librdkafka_version }} + - flatbuffers =${{ flatbuffers_version }} + - rapids-logger =0.1 + - zlib ${{ zlib_version }} + + build: + script: + content: + - cudf_ROOT="$(realpath ./cpp/build)" + - export cudf_ROOT + - > + ./build.sh -n -v \ + libcudf libcudf_kafka benchmarks tests \ + --build_metrics --incl_cache_stats --allgpuarch \ + --cmake-args=\"-DCMAKE_INSTALL_LIBDIR=lib -DCUDF_ENABLE_ARROW_S3=ON\" + secrets: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + + env: + CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} + CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} + CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} + CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} + PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL") }} + RAPIDS_ARTIFACTS_DIR: ${{ env.get("RAPIDS_ARTIFACTS_DIR") }} + SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }} + SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }} + SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }} + SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} + SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} + SCCACHE_S3_KEY_PREFIX: libcudf-${{ env.get("RAPIDS_CONDA_ARCH") }} + + +outputs: + - package: + name: libcudf + version: ${{ version }} + build: + script: + - cmake --install cpp/build + string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} + requirements: + host: + - cuda-version =${{ cuda_version }} + run: + - ${{ pin_compatible('cuda-version', upper_bound='x', lower_bound='x') }} + - if: cuda_major == "11" + then: + - cudatoolkit + - if: linux64 + then: + - libcufile ${{ cuda11_libcufile_run_version }} + else: + - cuda-nvrtc + - if: linux64 + then: + - libcufile + - nvcomp ${{ nvcomp_version }} + - librmm =${{ minor_version }} + - libkvikio =${{ minor_version }} + - dlpack ${{ dlpack_version }} + - rapids-logger =0.1 + run_exports: + - ${{ pin_subpackage("libcudf", upper_bound="x.x") }} + ignore_run_exports: + by_name: + - cuda-version + tests: + - script: + - test -f $PREFIX/lib/libcudf.so + - test -f $PREFIX/include/cudf/column/column.hpp + about: + homepage: ${{ load_from_file("python/libcudf/pyproject.toml").project.urls.Homepage }} + license: ${{ load_from_file("python/libcudf/pyproject.toml").project.license.text }} + summary: ${{ load_from_file("python/libcudf/pyproject.toml").project.description }} + + - package: + name: libcudf_kafka + version: ${{ version }} + build: + script: + - cmake --install cpp/libcudf_kafka/build + string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} + requirements: + host: + - librdkafka ${{ librdkafka_version }} + - ${{ pin_subpackage('libcudf', exact=True) }} + run: + - librdkafka ${{ librdkafka_version }} + - ${{ pin_subpackage('libcudf', exact=True) }} + ignore_run_exports: + by_name: + - cuda-version + tests: + - script: + - test -f $PREFIX/lib/libcudf_kafka.so + about: + homepage: https://rapids.ai/ + license: Apache-2.0 + summary: libcudf_kafka library + + - package: + name: libcudf-example + version: ${{ version }} + build: + script: + - ./cpp/examples/build.sh --install + string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} + requirements: + build: + - cmake ${{ cmake_version }} + - ${{ compiler('c') }} + - ${{ compiler('cxx') }} + - ${{ compiler('cuda') }} =${{ cuda_version }} + - cuda-version =${{ cuda_version }} + - ninja + - ${{ stdlib("c") }} + host: + - ${{ pin_subpackage('libcudf', exact=True) }} + - if: cuda_major == "11" + then: + - cuda-nvtx =${{ cuda_version }} + else: + - cuda-nvtx-dev + - cuda-version =${{ cuda_version }} + run: + - ${{ pin_subpackage('libcudf', exact=True) }} + - ${{ pin_compatible('cuda-version', upper_bound='x', lower_bound='x') }} + - if: cuda_major != "11" + then: + - cuda-nvtx + ignore_run_exports: + from_package: + - if: cuda_major != "11" + then: + - cuda-nvtx-dev + by_name: + - cuda-version + about: + homepage: ${{ load_from_file("python/libcudf/pyproject.toml").project.urls.Homepage }} + license: ${{ load_from_file("python/libcudf/pyproject.toml").project.license.text }} + summary: libcudf example executables + + - package: + name: libcudf-tests + version: ${{ version }} + build: + script: + - cmake --install cpp/build --component testing + - cmake --install cpp/libcudf_kafka/build --component testing + string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} + requirements: + host: + - ${{ pin_subpackage('libcudf', exact=True) }} + - ${{ pin_subpackage('libcudf_kafka', exact=True) }} + - cuda-version =${{ cuda_version }} + - if: cuda_major == "11" + then: + - libcurand ${{ cuda11_libcurand_run_version }} + else: + run: + - ${{ pin_compatible('cuda-version', upper_bound='x', lower_bound='x') }} + - ${{ pin_subpackage('libcudf', exact=True) }} + - ${{ pin_subpackage('libcudf_kafka', exact=True) }} + - if: cuda_major == "11" + then: + - libcurand ${{ cuda11_libcurand_run_version }} + else: + - libcurand + ignore_run_exports: + from_package: + - if: cuda_major != "11" + then: + - libcurand-dev + by_name: + - cuda-version + about: + homepage: ${{ load_from_file("python/libcudf/pyproject.toml").project.urls.Homepage }} + license: ${{ load_from_file("python/libcudf/pyproject.toml").project.license.text }} + summary: libcudf test & benchmark executables diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index 01fe6097936..784a0c49894 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -27,7 +27,7 @@ readme = { file = "README.md", content-type = "text/markdown" } authors = [ { name = "NVIDIA Corporation" }, ] -license = { text = "Apache 2.0" } +license = { text = "Apache-2.0" } requires-python = ">=3.10" classifiers = [ "Intended Audience :: Developers", From 41bf994198fa1c390ec15e31be007accee85196f Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Thu, 13 Feb 2025 12:36:09 -0500 Subject: [PATCH 02/51] Add rattler-build recipe for pylibcudf --- conda/recipes/pylibcudf/build.sh | 4 -- conda/recipes/pylibcudf/meta.yaml | 100 -------------------------- conda/recipes/pylibcudf/recipe.yaml | 105 ++++++++++++++++++++++++++++ python/pylibcudf/pyproject.toml | 2 +- 4 files changed, 106 insertions(+), 105 deletions(-) delete mode 100644 conda/recipes/pylibcudf/build.sh delete mode 100644 conda/recipes/pylibcudf/meta.yaml create mode 100644 conda/recipes/pylibcudf/recipe.yaml diff --git a/conda/recipes/pylibcudf/build.sh b/conda/recipes/pylibcudf/build.sh deleted file mode 100644 index 483346504db..00000000000 --- a/conda/recipes/pylibcudf/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) 2018-2024, NVIDIA CORPORATION. - -# This assumes the script is executed from the root of the repo directory -./build.sh pylibcudf diff --git a/conda/recipes/pylibcudf/meta.yaml b/conda/recipes/pylibcudf/meta.yaml deleted file mode 100644 index ae02cf8d4e5..00000000000 --- a/conda/recipes/pylibcudf/meta.yaml +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright (c) 2018-2025, NVIDIA CORPORATION. - -{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set py_version = environ['CONDA_PY'] %} -{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} -{% set cuda_major = cuda_version.split('.')[0] %} -{% set date_string = environ['RAPIDS_DATE_STRING'] %} - -package: - name: pylibcudf - version: {{ version }} - -source: - path: ../../.. - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - script_env: - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN - - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CUDA_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - - CMAKE_GENERATOR - - PARALLEL_LEVEL - - SCCACHE_BUCKET - - SCCACHE_IDLE_TIMEOUT - - SCCACHE_REGION - - SCCACHE_S3_KEY_PREFIX=pylibcudf-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=pylibcudf-linux64 # [linux64] - - SCCACHE_S3_USE_SSL - - SCCACHE_S3_NO_CREDENTIALS - ignore_run_exports_from: - - {{ compiler('cuda') }} - {% if cuda_major != "11" %} - - cuda-cudart-dev - - libcufile-dev # [linux64] - {% endif %} - -requirements: - build: - - cmake {{ cmake_version }} - - ninja - - {{ compiler('c') }} - - {{ compiler('cxx') }} - {% if cuda_major == "11" %} - - {{ compiler('cuda') }} ={{ cuda_version }} - {% else %} - - {{ compiler('cuda') }} - {% endif %} - - cuda-version ={{ cuda_version }} - - {{ stdlib("c") }} - host: - - python - - cython >=3.0.3 - - rapids-build-backend >=0.3.0,<0.4.0.dev0 - - scikit-build-core >=0.10.0 - - dlpack >=0.8,<1.0 - - libcudf ={{ version }} - - rmm ={{ minor_version }} - {% if cuda_major == "11" %} - - cudatoolkit - {% else %} - - cuda-cudart-dev - - cuda-nvrtc - - libcufile-dev # [linux64] - {% endif %} - - cuda-version ={{ cuda_version }} - run: - - python - - typing_extensions >=4.0.0 - - pandas >=2.0,<2.2.4dev0 - - numpy >=1.23,<2.1 - - pyarrow>=14.0.0,<20.0.0a0 - - libcudf ={{ version }} - - {{ pin_compatible('rmm', max_pin='x.x') }} - - fsspec >=0.6.0 - {% if cuda_major == "11" %} - - cuda-python >=11.8.5,<12.0a0 - {% else %} - - cuda-python >=12.6.2,<13.0a0 - {% endif %} - - nvtx >=0.2.1 - - packaging - -test: - requires: - - cuda-version ={{ cuda_version }} - imports: - - pylibcudf - -about: - home: https://rapids.ai/ - license: Apache-2.0 - license_family: APACHE - license_file: LICENSE - summary: pylibcudf library diff --git a/conda/recipes/pylibcudf/recipe.yaml b/conda/recipes/pylibcudf/recipe.yaml new file mode 100644 index 00000000000..ec3d9241444 --- /dev/null +++ b/conda/recipes/pylibcudf/recipe.yaml @@ -0,0 +1,105 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/prefix-dev/recipe-format/main/schema.json +# Copyright (c) 2018-2025, NVIDIA CORPORATION. +schema_version: 1 + +context: + version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} + minor_version: ${{ (version | split('.'))[:2] | join('.') }} + cuda_version: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[:2] | join(".") }} + cuda_major: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[0] }} + date_string: ${{ env.get("RAPIDS_DATE_STRING") }} + head_rev: ${{ git.head_rev(".")[:8] }} + +package: + name: pylibcudf + version: ${{ version }} + +source: + path: ../../.. + +build: + string: cuda${{ cuda_major }}_py${{ py_version }}_${{ date_string }}_${{ head_rev }} + script: + content: + - ./build.sh pylibcudf + secrets: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + + env: + CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} + CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} + CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} + CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} + SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }} + SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }} + SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }} + SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} + SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} + SCCACHE_S3_KEY_PREFIX: pylibcudf-${{ env.get("RAPIDS_CONDA_ARCH") }} + +requirements: + build: + - cmake ${{ cmake_version }} + - ninja + - ${{ compiler("c") }} + - ${{ compiler("cxx") }} + - ${{ compiler("cuda") }} + - cuda-version =${{ cuda_version }} + - ${{ stdlib("c") }} + host: + - python + - cython >=3.0.3 + - rapids-build-backend >=0.3.0,<0.4.0.dev0 + - scikit-build-core >=0.10.0 + - dlpack >=0.8,<1.0 + - libcudf =${{ version }} + - rmm =${{ minor_version }} + - if: cuda_major == "11" + then: + - cudatoolkit + else: + - cuda-cudart-dev + - cuda-nvrtc + - if: linux64 + then: + - libcufile-dev + - cuda-version =${{ cuda_version }} + run: + - python + - typing_extensions >=4.0.0 + - pandas >=2.0,<2.2.4dev0 + - numpy >=1.23,<3.0a0 + - pyarrow>=14.0.0,<20.0.0a0 + - libcudf =${{ version }} + - ${{ pin_compatible('rmm', upper_bound='x.x') }} + - fsspec >=0.6.0 + - if: cuda_major == "11" + then: + - cuda-python >=11.8.5,<12.0a0 + else: + - cuda-python >=12.6.2,<13.0a0 + - nvtx >=0.2.1 + - packaging + ignore_run_exports: + from_package: + - if: not (cuda_major == "11") + then: + - cuda-cudart-dev + - if: linux64 + then: + - libcufile-dev + by_name: + - cuda-version + +tests: + - python: + imports: + - pylibcudf + pip_check: false + +about: + homepage: ${{ load_from_file("python/pylibcudf/pyproject.toml").project.urls.Homepage }} + license: ${{ load_from_file("python/pylibcudf/pyproject.toml").project.license.text }} + summary: ${{ load_from_file("python/pylibcudf/pyproject.toml").project.description }} diff --git a/python/pylibcudf/pyproject.toml b/python/pylibcudf/pyproject.toml index e12d1ffdb39..8ea6f0e94a4 100644 --- a/python/pylibcudf/pyproject.toml +++ b/python/pylibcudf/pyproject.toml @@ -15,7 +15,7 @@ readme = { file = "README.md", content-type = "text/markdown" } authors = [ { name = "NVIDIA Corporation" }, ] -license = { text = "Apache 2.0" } +license = { text = "Apache-2.0" } requires-python = ">=3.10" dependencies = [ "cuda-python>=11.8.5,<12.0a0", From 67e73d434ad60d62132386003ab618bf7786ed39 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Thu, 13 Feb 2025 12:56:16 -0500 Subject: [PATCH 03/51] Add rattler-build recipe for dask-cudf --- conda/recipes/dask-cudf/build.sh | 4 -- conda/recipes/dask-cudf/meta.yaml | 62 -------------------------- conda/recipes/dask-cudf/recipe.yaml | 68 +++++++++++++++++++++++++++++ python/dask_cudf/pyproject.toml | 2 +- 4 files changed, 69 insertions(+), 67 deletions(-) delete mode 100644 conda/recipes/dask-cudf/build.sh delete mode 100644 conda/recipes/dask-cudf/meta.yaml create mode 100644 conda/recipes/dask-cudf/recipe.yaml diff --git a/conda/recipes/dask-cudf/build.sh b/conda/recipes/dask-cudf/build.sh deleted file mode 100644 index 473f52c28a0..00000000000 --- a/conda/recipes/dask-cudf/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) 2018-2019, NVIDIA CORPORATION. - -# This assumes the script is executed from the root of the repo directory -./build.sh dask_cudf diff --git a/conda/recipes/dask-cudf/meta.yaml b/conda/recipes/dask-cudf/meta.yaml deleted file mode 100644 index a476d5d53df..00000000000 --- a/conda/recipes/dask-cudf/meta.yaml +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright (c) 2018-2024, NVIDIA CORPORATION. - -{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set py_version = environ['CONDA_PY'] %} -{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} -{% set cuda_major = cuda_version.split('.')[0] %} -{% set date_string = environ['RAPIDS_DATE_STRING'] %} - -package: - name: dask-cudf - version: {{ version }} - -source: - path: ../../.. - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - script_env: - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN - - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CUDA_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - - CMAKE_GENERATOR - - PARALLEL_LEVEL - - SCCACHE_BUCKET - - SCCACHE_IDLE_TIMEOUT - - SCCACHE_REGION - - SCCACHE_S3_KEY_PREFIX=dask-cudf-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=dask-cudf-linux64 # [linux64] - - SCCACHE_S3_USE_SSL - - SCCACHE_S3_NO_CREDENTIALS - -requirements: - host: - - python - - rapids-build-backend >=0.3.0,<0.4.0.dev0 - - setuptools - - cuda-version ={{ cuda_version }} - run: - - python - - cudf ={{ version }} - - pynvml >=12.0.0,<13.0.0a0 - - rapids-dask-dependency ={{ minor_version }} - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - -test: - requires: - - cuda-version ={{ cuda_version }} - imports: - - dask_cudf - - -about: - home: https://rapids.ai/ - license: Apache-2.0 - license_family: APACHE - license_file: LICENSE - summary: dask-cudf library diff --git a/conda/recipes/dask-cudf/recipe.yaml b/conda/recipes/dask-cudf/recipe.yaml new file mode 100644 index 00000000000..e096eab101e --- /dev/null +++ b/conda/recipes/dask-cudf/recipe.yaml @@ -0,0 +1,68 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/prefix-dev/recipe-format/main/schema.json +# Copyright (c) 2018-2025, NVIDIA CORPORATION. +schema_version: 1 + +context: + version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} + minor_version: ${{ (version | split('.'))[:2] | join('.') }} + cuda_version: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[:2] | join(".") }} + cuda_major: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[0] }} + date_string: ${{ env.get("RAPIDS_DATE_STRING") }} + head_rev: ${{ git.head_rev(".")[:8] }} + +package: + name: dask-cudf + version: ${{ version }} + +source: + path: ../../.. + +build: + string: cuda${{ cuda_major }}_py${{ py_version }}_${{ date_string }}_${{ head_rev }} + script: + content: + - ./build.sh dask_cudf + secrets: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + + env: + CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} + CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} + CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} + CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} + SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }} + SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }} + SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }} + SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} + SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} + SCCACHE_S3_KEY_PREFIX: dask-cudf-${{ env.get("RAPIDS_CONDA_ARCH") }} + +requirements: + host: + - python + - rapids-build-backend >=0.3.0,<0.4.0.dev0 + - setuptools + - cuda-version =${{ cuda_version }} + run: + - python + - cudf =${{ version }} + - pynvml >=12.0.0,<13.0.0a0 + - rapids-dask-dependency =${{ minor_version }} + - ${{ pin_compatible('cuda-version', upper_bound='x', lower_bound='x') }} + ignore_run_exports: + by_name: + - cuda-version + + +tests: + - python: + imports: + - dask_cudf + pip_check: false + +about: + homepage: ${{ load_from_file("python/dask_cudf/pyproject.toml").project.urls.Homepage }} + license: ${{ load_from_file("python/dask_cudf/pyproject.toml").project.license.text }} + summary: ${{ load_from_file("python/dask_cudf/pyproject.toml").project.description }} diff --git a/python/dask_cudf/pyproject.toml b/python/dask_cudf/pyproject.toml index 83493d7f2a4..fd2bac3c0d2 100644 --- a/python/dask_cudf/pyproject.toml +++ b/python/dask_cudf/pyproject.toml @@ -16,7 +16,7 @@ readme = { file = "README.md", content-type = "text/markdown" } authors = [ { name = "NVIDIA Corporation" }, ] -license = { text = "Apache 2.0" } +license = { text = "Apache-2.0" } requires-python = ">=3.10" dependencies = [ "cudf==25.4.*,>=0.0.0a0", From b8a201b2f5bb33f262517c209b32860db83303c9 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Thu, 20 Feb 2025 10:40:30 -0500 Subject: [PATCH 04/51] Add rattler-build recipe for custreamz --- conda/recipes/custreamz/build.sh | 4 -- conda/recipes/custreamz/meta.yaml | 65 -------------------------- conda/recipes/custreamz/recipe.yaml | 72 +++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 69 deletions(-) delete mode 100644 conda/recipes/custreamz/build.sh delete mode 100644 conda/recipes/custreamz/meta.yaml create mode 100644 conda/recipes/custreamz/recipe.yaml diff --git a/conda/recipes/custreamz/build.sh b/conda/recipes/custreamz/build.sh deleted file mode 100644 index 88fccf90c69..00000000000 --- a/conda/recipes/custreamz/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) 2020-2022, NVIDIA CORPORATION. - -# This assumes the script is executed from the root of the repo directory -./build.sh -v custreamz diff --git a/conda/recipes/custreamz/meta.yaml b/conda/recipes/custreamz/meta.yaml deleted file mode 100644 index a031f05a73a..00000000000 --- a/conda/recipes/custreamz/meta.yaml +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright (c) 2018-2024, NVIDIA CORPORATION. - -{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set py_version = environ['CONDA_PY'] %} -{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} -{% set cuda_major = cuda_version.split('.')[0] %} -{% set date_string = environ['RAPIDS_DATE_STRING'] %} - -package: - name: custreamz - version: {{ version }} - -source: - path: ../../.. - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - script_env: - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN - - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CUDA_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - - CMAKE_GENERATOR - - PARALLEL_LEVEL - - SCCACHE_BUCKET - - SCCACHE_IDLE_TIMEOUT - - SCCACHE_REGION - - SCCACHE_S3_KEY_PREFIX=custreamz-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=custreamz-linux64 # [linux64] - - SCCACHE_S3_USE_SSL - - SCCACHE_S3_NO_CREDENTIALS - -requirements: - host: - - python - - rapids-build-backend >=0.3.0,<0.4.0.dev0 - - setuptools - - python-confluent-kafka >=2.5.0,<2.6.0a0 - - cudf_kafka ={{ version }} - - cuda-version ={{ cuda_version }} - run: - - python - - streamz - - cudf ={{ version }} - - cudf_kafka ={{ version }} - - rapids-dask-dependency ={{ minor_version }} - - python-confluent-kafka >=2.5.0,<2.6.0a0 - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - -test: - requires: - - cuda-version ={{ cuda_version }} - imports: - - custreamz - -about: - home: https://rapids.ai/ - license: Apache-2.0 - license_family: APACHE - license_file: LICENSE - summary: cuStreamz library diff --git a/conda/recipes/custreamz/recipe.yaml b/conda/recipes/custreamz/recipe.yaml new file mode 100644 index 00000000000..a12b4d41f87 --- /dev/null +++ b/conda/recipes/custreamz/recipe.yaml @@ -0,0 +1,72 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/prefix-dev/recipe-format/main/schema.json +# Copyright (c) 2018-2025, NVIDIA CORPORATION. +schema_version: 1 + +context: + version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} + minor_version: ${{ (version | split('.'))[:2] | join('.') }} + cuda_version: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[:2] | join(".") }} + cuda_major: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[0] }} + date_string: ${{ env.get("RAPIDS_DATE_STRING") }} + head_rev: ${{ git.head_rev(".")[:8] }} + + +package: + name: custreamz + version: ${{ version }} + +source: + path: ../../.. + +build: + string: cuda${{ cuda_major }}_py${{ py_version }}_${{ date_string }}_${{ head_rev }} + script: + content: + - ./build.sh -v custreamz + secrets: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + + env: + CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} + CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} + CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} + CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} + SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }} + SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }} + SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }} + SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} + SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} + SCCACHE_S3_KEY_PREFIX: pylibcudf-${{ env.get("RAPIDS_CONDA_ARCH") }} + +requirements: + host: + - python + - rapids-build-backend >=0.3.0,<0.4.0.dev0 + - setuptools + - python-confluent-kafka >=2.5.0,<2.6.0a0 + - cudf_kafka =${{ version }} + - cuda-version =${{ cuda_version }} + run: + - python + - streamz + - cudf =${{ version }} + - cudf_kafka =${{ version }} + - rapids-dask-dependency =${{ minor_version }} + - python-confluent-kafka >=2.5.0,<2.6.0a0 + - ${{ pin_compatible('cuda-version', upper_bound='x', lower_bound='x') }} + ignore_run_exports: + by_name: + - cuda-version + +tests: + - python: + imports: + - custreamz + pip_check: false + +about: + homepage: $${{ load_from_file("python/custreamz/pyproject.toml").project.urls.Homepage }} + license: $${{ load_from_file("python/custreamz/pyproject.toml").project.license.text }} + summary: $${{ load_from_file("python/custreamz/pyproject.toml").project.description }} From f0bd7b342b79955d0baa4348544e62559405156a Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Thu, 20 Feb 2025 10:59:20 -0500 Subject: [PATCH 05/51] Add rattler-build recipe for cudf_kafka --- conda/recipes/cudf_kafka/build.sh | 3 - conda/recipes/cudf_kafka/meta.yaml | 86 ---------------------------- conda/recipes/cudf_kafka/recipe.yaml | 85 +++++++++++++++++++++++++++ python/cudf_kafka/pyproject.toml | 2 +- 4 files changed, 86 insertions(+), 90 deletions(-) delete mode 100644 conda/recipes/cudf_kafka/build.sh delete mode 100644 conda/recipes/cudf_kafka/meta.yaml create mode 100644 conda/recipes/cudf_kafka/recipe.yaml diff --git a/conda/recipes/cudf_kafka/build.sh b/conda/recipes/cudf_kafka/build.sh deleted file mode 100644 index 9458349d101..00000000000 --- a/conda/recipes/cudf_kafka/build.sh +++ /dev/null @@ -1,3 +0,0 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. - -./build.sh -v cudf_kafka diff --git a/conda/recipes/cudf_kafka/meta.yaml b/conda/recipes/cudf_kafka/meta.yaml deleted file mode 100644 index a070c041d99..00000000000 --- a/conda/recipes/cudf_kafka/meta.yaml +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright (c) 2020-2025, NVIDIA CORPORATION. - -{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set py_version = environ['CONDA_PY'] %} -{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} -{% set cuda_major = cuda_version.split('.')[0] %} -{% set date_string = environ['RAPIDS_DATE_STRING'] %} - -package: - name: cudf_kafka - version: {{ version }} - -source: - path: ../../.. - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - script_env: - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN - - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CUDA_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - - CMAKE_GENERATOR - - PARALLEL_LEVEL - - SCCACHE_BUCKET - - SCCACHE_IDLE_TIMEOUT - - SCCACHE_REGION - - SCCACHE_S3_KEY_PREFIX=cudf-kafka-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=cudf-kafka-linux64 # [linux64] - - SCCACHE_S3_USE_SSL - - SCCACHE_S3_NO_CREDENTIALS - ignore_run_exports_from: - - {{ compiler('cuda') }} - {% if cuda_major != "11" %} - - cuda-cudart-dev - {% endif %} - -requirements: - build: - - cmake {{ cmake_version }} - - ninja - - {{ compiler('c') }} - - {{ compiler('cxx') }} - {% if cuda_major == "11" %} - - {{ compiler('cuda') }} ={{ cuda_version }} - {% else %} - - {{ compiler('cuda') }} - {% endif %} - - cuda-version ={{ cuda_version }} - - {{ stdlib("c") }} - host: - - python - - cython >=3.0.3 - - cuda-version ={{ cuda_version }} - - pylibcudf ={{ version }} - - libcudf_kafka ={{ version }} - - rapids-build-backend >=0.3.0,<0.4.0.dev0 - - scikit-build-core >=0.10.0 - {% if cuda_major != "11" %} - - cuda-cudart-dev - {% endif %} - run: - - python - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - - libcudf_kafka ={{ version }} - - pylibcudf ={{ version }} - {% if cuda_major != "11" %} - - cuda-cudart - {% endif %} - -test: - requires: - - cuda-version ={{ cuda_version }} - imports: - - cudf_kafka - -about: - home: https://rapids.ai/ - license: Apache-2.0 - license_family: APACHE - license_file: LICENSE - summary: libcudf_kafka library diff --git a/conda/recipes/cudf_kafka/recipe.yaml b/conda/recipes/cudf_kafka/recipe.yaml new file mode 100644 index 00000000000..c36a91ec0f3 --- /dev/null +++ b/conda/recipes/cudf_kafka/recipe.yaml @@ -0,0 +1,85 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/prefix-dev/recipe-format/main/schema.json +# Copyright (c) 2018-2025, NVIDIA CORPORATION. +schema_version: 1 + +context: + version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} + minor_version: ${{ (version | split('.'))[:2] | join('.') }} + cuda_version: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[:2] | join(".") }} + cuda_major: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[0] }} + date_string: ${{ env.get("RAPIDS_DATE_STRING") }} + head_rev: ${{ git.head_rev(".")[:8] }} + + +package: + name: cudf_kafka + version: ${{ version }} + +source: + path: ../../.. + +build: + string: cuda${{ cuda_major }}_py${{ py_version }}_${{ date_string }}_${{ head_rev }} + script: + content: + - ./build.sh -v cudf_kafka + secrets: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + env: + CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} + CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} + CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} + CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} + SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }} + SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }} + SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }} + SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} + SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} + SCCACHE_S3_KEY_PREFIX: cudf-kafka-${{ env.get("RAPIDS_CONDA_ARCH") }} + +requirements: + build: + - cmake ${{ cmake_version }} + - ninja + - ${{ compiler("c") }} + - ${{ compiler("cxx") }} + - ${{ compiler("cuda") }} + - cuda-version =${{ cuda_version }} + - ${{ stdlib("c") }} + host: + - python + - cython >=3.0.3 + - cuda-version =${{ cuda_version }} + - pylibcudf =${{ version }} + - libcudf_kafka =${{ version }} + - rapids-build-backend >=0.3.0,<0.4.0.dev0 + - scikit-build-core >=0.10.0 + - if: cuda_major != "11" + then: cuda-cudart-dev + run: + - python + - ${{ pin_compatible('cuda-version', upper_bound='x', lower_bound='x') }} + - libcudf_kafka =${{ version }} + - pylibcudf =${{ version }} + - if: cuda_major != "11" + then: cuda-cudart + ignore_run_exports: + from_package: + - if: cuda_major != "11" + then: cuda-cudart-dev + by_name: + - cuda-version + + +tests: + - python: + imports: + - cudf_kafka + pip_check: false + +about: + homepage: ${{ load_from_file("python/cudf_kafka/pyproject.toml").project.urls.Homepage }} + license: ${{ load_from_file("python/cudf_kafka/pyproject.toml").project.license.text }} + summary: ${{ load_from_file("python/cudf_kafka/pyproject.toml").project.description }} diff --git a/python/cudf_kafka/pyproject.toml b/python/cudf_kafka/pyproject.toml index 424010e632c..764c8c64a7e 100644 --- a/python/cudf_kafka/pyproject.toml +++ b/python/cudf_kafka/pyproject.toml @@ -15,7 +15,7 @@ readme = { file = "README.md", content-type = "text/markdown" } authors = [ { name = "NVIDIA Corporation" }, ] -license = { text = "Apache 2.0" } +license = { text = "Apache-2.0" } requires-python = ">=3.10" dependencies = [ "cudf==25.4.*,>=0.0.0a0", From 49c1554f039b207405a9ee270c5b89febf51f06c Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Thu, 20 Feb 2025 11:05:31 -0500 Subject: [PATCH 06/51] Add rattler-build recipe for cudf-polars --- conda/recipes/cudf-polars/build.sh | 4 -- conda/recipes/cudf-polars/meta.yaml | 61 ------------------------- conda/recipes/cudf-polars/recipe.yaml | 66 +++++++++++++++++++++++++++ python/cudf_polars/pyproject.toml | 2 +- 4 files changed, 67 insertions(+), 66 deletions(-) delete mode 100644 conda/recipes/cudf-polars/build.sh delete mode 100644 conda/recipes/cudf-polars/meta.yaml create mode 100644 conda/recipes/cudf-polars/recipe.yaml diff --git a/conda/recipes/cudf-polars/build.sh b/conda/recipes/cudf-polars/build.sh deleted file mode 100644 index 06e2f1bcb99..00000000000 --- a/conda/recipes/cudf-polars/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. - -# This assumes the script is executed from the root of the repo directory -./build.sh cudf_polars diff --git a/conda/recipes/cudf-polars/meta.yaml b/conda/recipes/cudf-polars/meta.yaml deleted file mode 100644 index 64a147d3c63..00000000000 --- a/conda/recipes/cudf-polars/meta.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright (c) 2024-2025, NVIDIA CORPORATION. - -{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set py_version = environ['CONDA_PY'] %} -{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} -{% set cuda_major = cuda_version.split('.')[0] %} -{% set date_string = environ['RAPIDS_DATE_STRING'] %} - -package: - name: cudf-polars - version: {{ version }} - -source: - path: ../../.. - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - script_env: - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN - - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CUDA_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - - CMAKE_GENERATOR - - PARALLEL_LEVEL - - SCCACHE_BUCKET - - SCCACHE_IDLE_TIMEOUT - - SCCACHE_REGION - - SCCACHE_S3_KEY_PREFIX=cudf-polars-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=cudf-polars-linux64 # [linux64] - - SCCACHE_S3_USE_SSL - - SCCACHE_S3_NO_CREDENTIALS - -requirements: - host: - - python - - rapids-build-backend >=0.3.0,<0.4.0.dev0 - - setuptools - - cuda-version ={{ cuda_version }} - run: - - python - - pylibcudf ={{ version }} - - polars >=1.20,<1.24 - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - -test: - requires: - - cuda-version ={{ cuda_version }} - imports: - - cudf_polars - - -about: - home: https://rapids.ai/ - license: Apache-2.0 - license_family: APACHE - license_file: LICENSE - summary: cudf-polars library diff --git a/conda/recipes/cudf-polars/recipe.yaml b/conda/recipes/cudf-polars/recipe.yaml new file mode 100644 index 00000000000..27cce268d4d --- /dev/null +++ b/conda/recipes/cudf-polars/recipe.yaml @@ -0,0 +1,66 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/prefix-dev/recipe-format/main/schema.json +# Copyright (c) 2018-2025, NVIDIA CORPORATION. +schema_version: 1 + +context: + version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} + minor_version: ${{ (version | split('.'))[:2] | join('.') }} + cuda_version: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[:2] | join(".") }} + cuda_major: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[0] }} + date_string: ${{ env.get("RAPIDS_DATE_STRING") }} + head_rev: ${{ git.head_rev(".")[:8] }} + +package: + name: cudf-polars + version: ${{ version }} + +source: + path: ../../.. + +build: + string: cuda${{ cuda_major }}_py${{ py_version }}_${{ date_string }}_${{ head_rev }} + script: + content: + - ./build.sh cudf_polars + secrets: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + env: + CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} + CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} + CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} + CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} + SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }} + SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }} + SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }} + SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} + SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} + SCCACHE_S3_KEY_PREFIX: cudf-polars-${{ env.get("RAPIDS_CONDA_ARCH") }} + +requirements: + host: + - python + - rapids-build-backend >=0.3.0,<0.4.0.dev0 + - setuptools + - cuda-version =${{ cuda_version }} + run: + - python + - pylibcudf =${{ version }} + - polars >=1.20,<1.23 + - ${{ pin_compatible('cuda-version', upper_bound='x', lower_bound='x') }} + ignore_run_exports: + by_name: + - cuda-version + +tests: + - python: + imports: + - cudf_polars + pip_check: false + + +about: + homepage: ${{ load_from_file("python/cudf_polars/pyproject.toml").project.urls.Homepage }} + license: ${{ load_from_file("python/cudf_polars/pyproject.toml").project.license.text }} + summary: ${{ load_from_file("python/cudf_polars/pyproject.toml").project.description }} diff --git a/python/cudf_polars/pyproject.toml b/python/cudf_polars/pyproject.toml index e9fc054efc2..fb44caaa0c0 100644 --- a/python/cudf_polars/pyproject.toml +++ b/python/cudf_polars/pyproject.toml @@ -16,7 +16,7 @@ readme = { file = "README.md", content-type = "text/markdown" } authors = [ { name = "NVIDIA Corporation" }, ] -license = { text = "Apache 2.0" } +license = { text = "Apache-2.0" } requires-python = ">=3.10" dependencies = [ "polars>=1.20,<1.24", From 8f8f2625d80b15068439866a10e1ae79fe5da3cc Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Thu, 20 Feb 2025 11:14:41 -0500 Subject: [PATCH 07/51] Add rattler-build recipe for cudf --- conda/recipes/cudf/build.sh | 4 -- conda/recipes/cudf/meta.yaml | 119 ------------------------------- conda/recipes/cudf/recipe.yaml | 124 +++++++++++++++++++++++++++++++++ python/cudf/pyproject.toml | 2 +- 4 files changed, 125 insertions(+), 124 deletions(-) delete mode 100644 conda/recipes/cudf/build.sh delete mode 100644 conda/recipes/cudf/meta.yaml create mode 100644 conda/recipes/cudf/recipe.yaml diff --git a/conda/recipes/cudf/build.sh b/conda/recipes/cudf/build.sh deleted file mode 100644 index 43d046402c7..00000000000 --- a/conda/recipes/cudf/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright (c) 2018-2022, NVIDIA CORPORATION. - -# This assumes the script is executed from the root of the repo directory -./build.sh cudf diff --git a/conda/recipes/cudf/meta.yaml b/conda/recipes/cudf/meta.yaml deleted file mode 100644 index 43060ef1c87..00000000000 --- a/conda/recipes/cudf/meta.yaml +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright (c) 2018-2025, NVIDIA CORPORATION. - -{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} -{% set py_version = environ['CONDA_PY'] %} -{% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} -{% set cuda_major = cuda_version.split('.')[0] %} -{% set date_string = environ['RAPIDS_DATE_STRING'] %} - -package: - name: cudf - version: {{ version }} - -source: - path: ../../.. - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: cuda{{ cuda_major }}_py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - script_env: - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN - - CMAKE_C_COMPILER_LAUNCHER - - CMAKE_CUDA_COMPILER_LAUNCHER - - CMAKE_CXX_COMPILER_LAUNCHER - - CMAKE_GENERATOR - - PARALLEL_LEVEL - - SCCACHE_BUCKET - - SCCACHE_IDLE_TIMEOUT - - SCCACHE_REGION - - SCCACHE_S3_KEY_PREFIX=cudf-aarch64 # [aarch64] - - SCCACHE_S3_KEY_PREFIX=cudf-linux64 # [linux64] - - SCCACHE_S3_USE_SSL - - SCCACHE_S3_NO_CREDENTIALS - ignore_run_exports_from: - - {{ compiler('cuda') }} - {% if cuda_major != "11" %} - - cuda-cudart-dev - - libcufile-dev # [linux64] - {% endif %} - -requirements: - build: - - cmake {{ cmake_version }} - - ninja - - {{ compiler('c') }} - - {{ compiler('cxx') }} - {% if cuda_major == "11" %} - - {{ compiler('cuda') }} ={{ cuda_version }} - {% else %} - - {{ compiler('cuda') }} - {% endif %} - - cuda-version ={{ cuda_version }} - - {{ stdlib("c") }} - host: - - python - - cython >=3.0.3 - - rapids-build-backend >=0.3.0,<0.4.0.dev0 - - scikit-build-core >=0.10.0 - - dlpack >=0.8,<1.0 - - libcudf ={{ version }} - - pylibcudf ={{ version }} - - rmm ={{ minor_version }} - {% if cuda_major == "11" %} - - cudatoolkit - {% else %} - - cuda-cudart-dev - - cuda-nvrtc - - libcufile-dev # [linux64] - {% endif %} - - cuda-version ={{ cuda_version }} - run: - - python - - typing_extensions >=4.0.0 - - pandas >=2.0,<2.2.4dev0 - - cupy >=12.0.0 - - numba-cuda >=0.4.0,<0.5.0a0 - - numba >=0.59.1,<0.62.0a0 - - numpy >=1.23,<2.1 - - pyarrow>=14.0.0,<20.0.0a0 - - libcudf ={{ version }} - - pylibcudf ={{ version }} - - {{ pin_compatible('rmm', max_pin='x.x') }} - - fsspec >=0.6.0 - {% if cuda_major == "11" %} - - cudatoolkit - - ptxcompiler >=0.7.0 - - cubinlinker # CUDA enhanced compatibility. - - cuda-python >=11.8.5,<12.0a0 - {% else %} - - cuda-cudart - - libcufile # [linux64] - # Needed by Numba for CUDA support - - cuda-nvcc-impl - # TODO: Add nvjitlink here - # xref: https://github.com/rapidsai/cudf/issues/12822 - - cuda-nvrtc - - cuda-python >=12.6.2,<13.0a0 - - pynvjitlink - {% endif %} - - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }} - - nvtx >=0.2.1 - - packaging - - cachetools - - rich - -test: - requires: - - cuda-version ={{ cuda_version }} - imports: - - cudf - -about: - home: https://rapids.ai/ - license: Apache-2.0 - license_family: APACHE - license_file: LICENSE - summary: cuDF GPU DataFrame core library diff --git a/conda/recipes/cudf/recipe.yaml b/conda/recipes/cudf/recipe.yaml new file mode 100644 index 00000000000..39ac07fbb38 --- /dev/null +++ b/conda/recipes/cudf/recipe.yaml @@ -0,0 +1,124 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/prefix-dev/recipe-format/main/schema.json +# Copyright (c) 2018-2025, NVIDIA CORPORATION. +schema_version: 1 + +context: + version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} + minor_version: ${{ (version | split('.'))[:2] | join('.') }} + cuda_version: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[:2] | join(".") }} + cuda_major: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[0] }} + date_string: ${{ env.get("RAPIDS_DATE_STRING") }} + head_rev: ${{ git.head_rev(".")[:8] }} + +package: + name: cudf + version: ${{ version }} + +source: + path: ../../.. + +build: + string: cuda${{ cuda_major }}_py${{ py_version }}_${{ date_string }}_${{ head_rev }} + script: + content: + - ./build.sh cudf + secrets: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + env: + CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} + CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} + CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} + CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} + SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }} + SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }} + SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }} + SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} + SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} + SCCACHE_S3_KEY_PREFIX: cudf-${{ env.get("RAPIDS_CONDA_ARCH") }} + +requirements: + build: + - cmake ${{ cmake_version }} + - ninja + - ${{ compiler("c") }} + - ${{ compiler("cxx") }} + - ${{ compiler("cuda") }} + - cuda-version =${{ cuda_version }} + - ${{ stdlib("c") }} + host: + - python + - cython >=3.0.3 + - rapids-build-backend >=0.3.0,<0.4.0.dev0 + - scikit-build-core >=0.10.0 + - dlpack >=0.8,<1.0 + - libcudf =${{ version }} + - pylibcudf =${{ version }} + - rmm =${{ minor_version }} + - if: cuda_major == "11" + then: + - cudatoolkit + else: + - cuda-cudart-dev + - cuda-nvrtc + - if: linux64 + then: + - libcufile-dev + - cuda-version =${{ cuda_version }} + run: + - python + - typing_extensions >=4.0.0 + - pandas >=2.0,<2.2.4dev0 + - cupy >=12.0.0 + - numba-cuda >=0.2.0,<0.3.0a0 + - numba >=0.59.1,<0.61.0a0 + - numpy >=1.23,<3.0a0 + - pyarrow>=14.0.0,<20.0.0a0 + - libcudf =${{ version }} + - pylibcudf =${{ version }} + - ${{ pin_compatible('rmm', upper_bound='x.x') }} + - fsspec >=0.6.0 + - if: cuda_major == "11" + then: + - cudatoolkit + - ptxcompiler >=0.7.0 + - cubinlinker # CUDA enhanced compatibility. + - cuda-python >=11.8.5,<12.0a0 + else: + - cuda-cudart + # Needed by Numba for CUDA support + - cuda-nvcc-impl + # TODO: Add nvjitlink here + # xref: https://github.com/rapidsai/cudf/issues/12822 + - cuda-nvrtc + - cuda-python >=12.6.2,<13.0a0 + - pynvjitlink + - if: linux64 + then: + - libcufile + - ${{ pin_compatible('cuda-version', upper_bound='x', lower_bound='x') }} + - nvtx >=0.2.1 + - packaging + - cachetools + - rich + + ignore_run_exports: + from_package: + - if: cuda_major != "11" + then: + - cuda-cudart-dev + - if: linux64 + then: libcufile-dev + + +tests: + - python: + imports: + - cudf + pip_check: false + +about: + homepage: ${{ load_from_file("python/cudf/pyproject.toml").project.urls.Homepage }} + license: ${{ load_from_file("python/cudf/pyproject.toml").project.license.text }} + summary: ${{ load_from_file("python/cudf/pyproject.toml").project.description }} diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 8b8abe90ac9..2ce5131ea8e 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -15,7 +15,7 @@ readme = { file = "README.md", content-type = "text/markdown" } authors = [ { name = "NVIDIA Corporation" }, ] -license = { text = "Apache 2.0" } +license = { text = "Apache-2.0" } requires-python = ">=3.10" dependencies = [ "cachetools", From e085f3b0265860f5899ce16c97104a7aec4cf993 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Thu, 20 Feb 2025 11:29:29 -0500 Subject: [PATCH 08/51] Update `build_python.sh` to use `rattler-build` --- ci/build_cpp.sh | 1 - ci/build_python.sh | 97 ++++++++++++++++++++++++++++++---------------- 2 files changed, 63 insertions(+), 35 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index f7a7d41b342..78a15bc8092 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -17,7 +17,6 @@ rapids-logger "Begin cpp build" sccache --zero-stats -# With boa installed conda build forward to boa RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) export RAPIDS_PACKAGE_VERSION diff --git a/ci/build_python.sh b/ci/build_python.sh index abbdc3f3a3b..2736691e3f7 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -3,8 +3,6 @@ set -euo pipefail -rapids-configure-conda-channels - source rapids-configure-sccache source rapids-date-string @@ -21,51 +19,82 @@ CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) sccache --zero-stats -# TODO: Remove `--no-test` flag once importing on a CPU +# TODO: Remove `--test skip` flag once importing on a CPU # node works correctly # With boa installed conda build forwards to the boa builder -RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry build \ - --no-test \ - --channel "${CPP_CHANNEL}" \ - conda/recipes/pylibcudf +RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) +export RAPIDS_PACKAGE_VERSION + +# populates `RATTLER_CHANNELS` array +source rapids-rattler-channel-string + +# --no-build-id allows for caching with `sccache` +# more info is available at +# https://rattler.build/latest/tips_and_tricks/#using-sccache-or-ccache-with-rattler-build +rattler-build build --recipe conda/recipes/pylibcudf \ + --experimental \ + --no-build-id \ + --channel-priority disabled \ + --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ + --test skip \ + -c "${CPP_CHANNEL}" \ + "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats sccache --zero-stats -RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry build \ - --no-test \ - --channel "${CPP_CHANNEL}" \ - --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - conda/recipes/cudf +rattler-build build --recipe conda/recipes/cudf \ + --experimental \ + --no-build-id \ + --channel-priority disabled \ + --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ + --test skip \ + -c "${CPP_CHANNEL}" \ + "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats sccache --zero-stats -RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry build \ - --no-test \ - --channel "${CPP_CHANNEL}" \ - --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - conda/recipes/dask-cudf - -RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry build \ - --no-test \ - --channel "${CPP_CHANNEL}" \ - --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - conda/recipes/cudf_kafka +rattler-build build --recipe conda/recipes/dask-cudf \ + --experimental \ + --no-build-id \ + --channel-priority disabled \ + --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ + --test skip \ + -c "${CPP_CHANNEL}" \ + "${RATTLER_CHANNELS[@]}" + +rattler-build build --recipe conda/recipes/cudf_kafka \ + --experimental \ + --no-build-id \ + --channel-priority disabled \ + --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ + --test skip \ + -c "${CPP_CHANNEL}" \ + "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats -RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry build \ - --no-test \ - --channel "${CPP_CHANNEL}" \ - --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - conda/recipes/custreamz - -RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry build \ - --no-test \ - --channel "${CPP_CHANNEL}" \ - --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - conda/recipes/cudf-polars +rattler-build build --recipe conda/recipes/custreamz \ + --experimental \ + --no-build-id \ + --channel-priority disabled \ + --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ + --test skip \ + -c "${CPP_CHANNEL}" \ + "${RATTLER_CHANNELS[@]}" + +rattler-build build --recipe conda/recipes/cudf-polars \ + --experimental \ + --no-build-id \ + --channel-priority disabled \ + --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ + --test skip \ + -c "${CPP_CHANNEL}" \ + "${RATTLER_CHANNELS[@]}" + +# remove build_cache directory +rm -rf "$RAPIDS_CONDA_BLD_OUTPUT_DIR"/build_cache rapids-upload-conda-to-s3 python From 273ebba9562a2672c9870400d8a51ac4219533fb Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Thu, 20 Feb 2025 11:59:11 -0500 Subject: [PATCH 09/51] Fix ignore_run_export condition --- ci/build_python.sh | 1 - conda/recipes/libcudf/recipe.yaml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/build_python.sh b/ci/build_python.sh index 2736691e3f7..16393f74107 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -21,7 +21,6 @@ sccache --zero-stats # TODO: Remove `--test skip` flag once importing on a CPU # node works correctly -# With boa installed conda build forwards to the boa builder RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) export RAPIDS_PACKAGE_VERSION diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index 6a3704614ee..467287274dd 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -186,7 +186,7 @@ outputs: from_package: - if: cuda_major != "11" then: - - cuda-nvtx-dev + - cuda-nvtx by_name: - cuda-version about: From 369277c9ae9fcc81738d3a8ef2151a6d3dc9a862 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Thu, 20 Feb 2025 17:19:41 -0500 Subject: [PATCH 10/51] disable non-conda workflows --- .github/workflows/pr.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 2c583598f54..fc7728964f0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -17,7 +17,7 @@ jobs: - changed-files - checks - conda-cpp-build - - cpp-linters + # - cpp-linters - conda-cpp-checks - conda-cpp-tests - conda-python-build @@ -41,7 +41,7 @@ jobs: - pandas-tests-diff - narwhals-tests - telemetry-setup - - third-party-integration-tests-cudf-pandas + # - third-party-integration-tests-cudf-pandas secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-25.04 if: always() @@ -135,14 +135,14 @@ jobs: with: build_type: pull-request node_type: "cpu16" - cpp-linters: - secrets: inherit - needs: checks - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.04 - with: - build_type: pull-request - run_script: "ci/cpp_linters.sh" - node_type: "cpu16" + # cpp-linters: + # secrets: inherit + # needs: checks + # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.04 + # with: + # build_type: pull-request + # run_script: "ci/cpp_linters.sh" + # node_type: "cpu16" conda-cpp-checks: needs: conda-cpp-build secrets: inherit From 806a0e4c43d32ef0b21c2b262d11290a767e83e2 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 21 Feb 2025 09:31:33 -0500 Subject: [PATCH 11/51] remove cuda_version pin from compiler --- conda/recipes/libcudf/recipe.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index 467287274dd..e84f05514c2 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -21,7 +21,7 @@ cache: build: - ${{ compiler('c') }} - ${{ compiler('cxx') }} - - ${{ compiler('cuda') }} =${{ cuda_version }} + - ${{ compiler('cuda') }} - cuda-version =${{ cuda_version }} - cmake ${{ cmake_version }} - ninja From 700d98e335317ceb174ffd8e7239ff5d92869628 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 21 Feb 2025 09:31:42 -0500 Subject: [PATCH 12/51] fix libcufile version typo --- conda/recipes/libcudf/recipe.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index e84f05514c2..62c94b42725 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -32,8 +32,8 @@ cache: - if: cuda_major == "11" then: - cudatoolkit - - libcufile ${{ cuda11_libcublas_host_version }} - - libcufile-dev ${{ cuda11_libcublas_host_version }} + - libcufile ${{ cuda11_libcufile_host_version }} + - libcufile-dev ${{ cuda11_libcufile_host_version }} - libcurand ${{ cuda11_libcurand_host_version }} - libcurand-dev ${{ cuda11_libcurand_host_version }} - cuda-nvrtc =${{ cuda_version }} From 635363665bbfa342635e7723acbe028c7fb8be95 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 21 Feb 2025 09:32:17 -0500 Subject: [PATCH 13/51] swap build and requirements ordering --- conda/recipes/libcudf/recipe.yaml | 58 +++++++++++++++---------------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index 62c94b42725..f09871853aa 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -17,6 +17,34 @@ cache: source: path: ../../.. + build: + script: + content: + - cudf_ROOT="$(realpath ./cpp/build)" + - export cudf_ROOT + - > + ./build.sh -n -v \ + libcudf libcudf_kafka benchmarks tests \ + --build_metrics --incl_cache_stats --allgpuarch \ + --cmake-args=\"-DCMAKE_INSTALL_LIBDIR=lib -DCUDF_ENABLE_ARROW_S3=ON\" + secrets: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + env: + CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} + CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} + CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} + CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} + PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL") }} + RAPIDS_ARTIFACTS_DIR: ${{ env.get("RAPIDS_ARTIFACTS_DIR") }} + SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }} + SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }} + SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }} + SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} + SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} + SCCACHE_S3_KEY_PREFIX: libcudf-${{ env.get("RAPIDS_CONDA_ARCH") }} + requirements: build: - ${{ compiler('c') }} @@ -54,36 +82,6 @@ cache: - rapids-logger =0.1 - zlib ${{ zlib_version }} - build: - script: - content: - - cudf_ROOT="$(realpath ./cpp/build)" - - export cudf_ROOT - - > - ./build.sh -n -v \ - libcudf libcudf_kafka benchmarks tests \ - --build_metrics --incl_cache_stats --allgpuarch \ - --cmake-args=\"-DCMAKE_INSTALL_LIBDIR=lib -DCUDF_ENABLE_ARROW_S3=ON\" - secrets: - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN - - env: - CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} - CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} - CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} - CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} - PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL") }} - RAPIDS_ARTIFACTS_DIR: ${{ env.get("RAPIDS_ARTIFACTS_DIR") }} - SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }} - SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }} - SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }} - SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} - SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} - SCCACHE_S3_KEY_PREFIX: libcudf-${{ env.get("RAPIDS_CONDA_ARCH") }} - - outputs: - package: name: libcudf From 526cd38357c061f1dec0e62c8b75b7b31e1de4dd Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 21 Feb 2025 09:33:57 -0500 Subject: [PATCH 14/51] add buildstringified py_version to all python recipes --- conda/recipes/cudf-polars/recipe.yaml | 1 + conda/recipes/cudf/recipe.yaml | 1 + conda/recipes/cudf_kafka/recipe.yaml | 1 + conda/recipes/dask-cudf/recipe.yaml | 1 + conda/recipes/pylibcudf/recipe.yaml | 1 + 5 files changed, 5 insertions(+) diff --git a/conda/recipes/cudf-polars/recipe.yaml b/conda/recipes/cudf-polars/recipe.yaml index 27cce268d4d..7aca4efa4d8 100644 --- a/conda/recipes/cudf-polars/recipe.yaml +++ b/conda/recipes/cudf-polars/recipe.yaml @@ -8,6 +8,7 @@ context: cuda_version: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[:2] | join(".") }} cuda_major: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[0] }} date_string: ${{ env.get("RAPIDS_DATE_STRING") }} + py_version: ${{ env.get("RAPIDS_PY_VERSION") | version_to_buildstring }} head_rev: ${{ git.head_rev(".")[:8] }} package: diff --git a/conda/recipes/cudf/recipe.yaml b/conda/recipes/cudf/recipe.yaml index 39ac07fbb38..4b2c8fd2f23 100644 --- a/conda/recipes/cudf/recipe.yaml +++ b/conda/recipes/cudf/recipe.yaml @@ -8,6 +8,7 @@ context: cuda_version: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[:2] | join(".") }} cuda_major: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[0] }} date_string: ${{ env.get("RAPIDS_DATE_STRING") }} + py_version: ${{ env.get("RAPIDS_PY_VERSION") | version_to_buildstring }} head_rev: ${{ git.head_rev(".")[:8] }} package: diff --git a/conda/recipes/cudf_kafka/recipe.yaml b/conda/recipes/cudf_kafka/recipe.yaml index c36a91ec0f3..6d2719cc620 100644 --- a/conda/recipes/cudf_kafka/recipe.yaml +++ b/conda/recipes/cudf_kafka/recipe.yaml @@ -8,6 +8,7 @@ context: cuda_version: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[:2] | join(".") }} cuda_major: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[0] }} date_string: ${{ env.get("RAPIDS_DATE_STRING") }} + py_version: ${{ env.get("RAPIDS_PY_VERSION") | version_to_buildstring }} head_rev: ${{ git.head_rev(".")[:8] }} diff --git a/conda/recipes/dask-cudf/recipe.yaml b/conda/recipes/dask-cudf/recipe.yaml index e096eab101e..78338ee3aa5 100644 --- a/conda/recipes/dask-cudf/recipe.yaml +++ b/conda/recipes/dask-cudf/recipe.yaml @@ -8,6 +8,7 @@ context: cuda_version: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[:2] | join(".") }} cuda_major: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[0] }} date_string: ${{ env.get("RAPIDS_DATE_STRING") }} + py_version: ${{ env.get("RAPIDS_PY_VERSION") | version_to_buildstring }} head_rev: ${{ git.head_rev(".")[:8] }} package: diff --git a/conda/recipes/pylibcudf/recipe.yaml b/conda/recipes/pylibcudf/recipe.yaml index ec3d9241444..f6bd76b2697 100644 --- a/conda/recipes/pylibcudf/recipe.yaml +++ b/conda/recipes/pylibcudf/recipe.yaml @@ -8,6 +8,7 @@ context: cuda_version: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[:2] | join(".") }} cuda_major: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[0] }} date_string: ${{ env.get("RAPIDS_DATE_STRING") }} + py_version: ${{ env.get("RAPIDS_PY_VERSION") | version_to_buildstring }} head_rev: ${{ git.head_rev(".")[:8] }} package: From f277e5de99cfedf9561ab4d9740072a0734d5c20 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 21 Feb 2025 09:35:04 -0500 Subject: [PATCH 15/51] use double-quotes everywhere --- conda/recipes/cudf-polars/recipe.yaml | 8 +++--- conda/recipes/cudf/recipe.yaml | 10 +++---- conda/recipes/cudf_kafka/recipe.yaml | 8 +++--- conda/recipes/custreamz/recipe.yaml | 8 +++--- conda/recipes/dask-cudf/recipe.yaml | 8 +++--- conda/recipes/libcudf/recipe.yaml | 40 +++++++++++++-------------- conda/recipes/pylibcudf/recipe.yaml | 8 +++--- 7 files changed, 45 insertions(+), 45 deletions(-) diff --git a/conda/recipes/cudf-polars/recipe.yaml b/conda/recipes/cudf-polars/recipe.yaml index 7aca4efa4d8..45cab112402 100644 --- a/conda/recipes/cudf-polars/recipe.yaml +++ b/conda/recipes/cudf-polars/recipe.yaml @@ -4,9 +4,9 @@ schema_version: 1 context: version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} - minor_version: ${{ (version | split('.'))[:2] | join('.') }} - cuda_version: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[:2] | join(".") }} - cuda_major: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[0] }} + minor_version: ${{ (version | split("."))[:2] | join(".") }} + cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} + cuda_major: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }} date_string: ${{ env.get("RAPIDS_DATE_STRING") }} py_version: ${{ env.get("RAPIDS_PY_VERSION") | version_to_buildstring }} head_rev: ${{ git.head_rev(".")[:8] }} @@ -49,7 +49,7 @@ requirements: - python - pylibcudf =${{ version }} - polars >=1.20,<1.23 - - ${{ pin_compatible('cuda-version', upper_bound='x', lower_bound='x') }} + - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} ignore_run_exports: by_name: - cuda-version diff --git a/conda/recipes/cudf/recipe.yaml b/conda/recipes/cudf/recipe.yaml index 4b2c8fd2f23..9c5d856b6b5 100644 --- a/conda/recipes/cudf/recipe.yaml +++ b/conda/recipes/cudf/recipe.yaml @@ -4,9 +4,9 @@ schema_version: 1 context: version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} - minor_version: ${{ (version | split('.'))[:2] | join('.') }} - cuda_version: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[:2] | join(".") }} - cuda_major: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[0] }} + minor_version: ${{ (version | split("."))[:2] | join(".") }} + cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} + cuda_major: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }} date_string: ${{ env.get("RAPIDS_DATE_STRING") }} py_version: ${{ env.get("RAPIDS_PY_VERSION") | version_to_buildstring }} head_rev: ${{ git.head_rev(".")[:8] }} @@ -78,7 +78,7 @@ requirements: - pyarrow>=14.0.0,<20.0.0a0 - libcudf =${{ version }} - pylibcudf =${{ version }} - - ${{ pin_compatible('rmm', upper_bound='x.x') }} + - ${{ pin_compatible("rmm", upper_bound="x.x") }} - fsspec >=0.6.0 - if: cuda_major == "11" then: @@ -98,7 +98,7 @@ requirements: - if: linux64 then: - libcufile - - ${{ pin_compatible('cuda-version', upper_bound='x', lower_bound='x') }} + - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - nvtx >=0.2.1 - packaging - cachetools diff --git a/conda/recipes/cudf_kafka/recipe.yaml b/conda/recipes/cudf_kafka/recipe.yaml index 6d2719cc620..bd86900fc25 100644 --- a/conda/recipes/cudf_kafka/recipe.yaml +++ b/conda/recipes/cudf_kafka/recipe.yaml @@ -4,9 +4,9 @@ schema_version: 1 context: version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} - minor_version: ${{ (version | split('.'))[:2] | join('.') }} - cuda_version: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[:2] | join(".") }} - cuda_major: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[0] }} + minor_version: ${{ (version | split("."))[:2] | join(".") }} + cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} + cuda_major: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }} date_string: ${{ env.get("RAPIDS_DATE_STRING") }} py_version: ${{ env.get("RAPIDS_PY_VERSION") | version_to_buildstring }} head_rev: ${{ git.head_rev(".")[:8] }} @@ -61,7 +61,7 @@ requirements: then: cuda-cudart-dev run: - python - - ${{ pin_compatible('cuda-version', upper_bound='x', lower_bound='x') }} + - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - libcudf_kafka =${{ version }} - pylibcudf =${{ version }} - if: cuda_major != "11" diff --git a/conda/recipes/custreamz/recipe.yaml b/conda/recipes/custreamz/recipe.yaml index a12b4d41f87..947ec0f6d47 100644 --- a/conda/recipes/custreamz/recipe.yaml +++ b/conda/recipes/custreamz/recipe.yaml @@ -4,9 +4,9 @@ schema_version: 1 context: version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} - minor_version: ${{ (version | split('.'))[:2] | join('.') }} - cuda_version: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[:2] | join(".") }} - cuda_major: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[0] }} + minor_version: ${{ (version | split("."))[:2] | join(".") }} + cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} + cuda_major: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }} date_string: ${{ env.get("RAPIDS_DATE_STRING") }} head_rev: ${{ git.head_rev(".")[:8] }} @@ -55,7 +55,7 @@ requirements: - cudf_kafka =${{ version }} - rapids-dask-dependency =${{ minor_version }} - python-confluent-kafka >=2.5.0,<2.6.0a0 - - ${{ pin_compatible('cuda-version', upper_bound='x', lower_bound='x') }} + - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} ignore_run_exports: by_name: - cuda-version diff --git a/conda/recipes/dask-cudf/recipe.yaml b/conda/recipes/dask-cudf/recipe.yaml index 78338ee3aa5..abcfc309f61 100644 --- a/conda/recipes/dask-cudf/recipe.yaml +++ b/conda/recipes/dask-cudf/recipe.yaml @@ -4,9 +4,9 @@ schema_version: 1 context: version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} - minor_version: ${{ (version | split('.'))[:2] | join('.') }} - cuda_version: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[:2] | join(".") }} - cuda_major: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[0] }} + minor_version: ${{ (version | split("."))[:2] | join(".") }} + cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} + cuda_major: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }} date_string: ${{ env.get("RAPIDS_DATE_STRING") }} py_version: ${{ env.get("RAPIDS_PY_VERSION") | version_to_buildstring }} head_rev: ${{ git.head_rev(".")[:8] }} @@ -51,7 +51,7 @@ requirements: - cudf =${{ version }} - pynvml >=12.0.0,<13.0.0a0 - rapids-dask-dependency =${{ minor_version }} - - ${{ pin_compatible('cuda-version', upper_bound='x', lower_bound='x') }} + - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} ignore_run_exports: by_name: - cuda-version diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index f09871853aa..f4267e16395 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -4,9 +4,9 @@ schema_version: 1 context: version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} - minor_version: ${{ (version | split('.'))[:2] | join('.') }} - cuda_version: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[:2] | join(".") }} - cuda_major: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[0] }} + minor_version: ${{ (version | split("."))[:2] | join(".") }} + cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} + cuda_major: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }} date_string: ${{ env.get("RAPIDS_DATE_STRING") }} head_rev: ${{ git.head_rev(".")[:8] }} @@ -47,9 +47,9 @@ cache: requirements: build: - - ${{ compiler('c') }} - - ${{ compiler('cxx') }} - - ${{ compiler('cuda') }} + - ${{ compiler("c") }} + - ${{ compiler("cxx") }} + - ${{ compiler("cuda") }} - cuda-version =${{ cuda_version }} - cmake ${{ cmake_version }} - ninja @@ -94,7 +94,7 @@ outputs: host: - cuda-version =${{ cuda_version }} run: - - ${{ pin_compatible('cuda-version', upper_bound='x', lower_bound='x') }} + - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - if: cuda_major == "11" then: - cudatoolkit @@ -135,10 +135,10 @@ outputs: requirements: host: - librdkafka ${{ librdkafka_version }} - - ${{ pin_subpackage('libcudf', exact=True) }} + - ${{ pin_subpackage("libcudf", exact=True) }} run: - librdkafka ${{ librdkafka_version }} - - ${{ pin_subpackage('libcudf', exact=True) }} + - ${{ pin_subpackage("libcudf", exact=True) }} ignore_run_exports: by_name: - cuda-version @@ -160,14 +160,14 @@ outputs: requirements: build: - cmake ${{ cmake_version }} - - ${{ compiler('c') }} - - ${{ compiler('cxx') }} - - ${{ compiler('cuda') }} =${{ cuda_version }} + - ${{ compiler("c") }} + - ${{ compiler("cxx") }} + - ${{ compiler("cuda") }} =${{ cuda_version }} - cuda-version =${{ cuda_version }} - ninja - ${{ stdlib("c") }} host: - - ${{ pin_subpackage('libcudf', exact=True) }} + - ${{ pin_subpackage("libcudf", exact=True) }} - if: cuda_major == "11" then: - cuda-nvtx =${{ cuda_version }} @@ -175,8 +175,8 @@ outputs: - cuda-nvtx-dev - cuda-version =${{ cuda_version }} run: - - ${{ pin_subpackage('libcudf', exact=True) }} - - ${{ pin_compatible('cuda-version', upper_bound='x', lower_bound='x') }} + - ${{ pin_subpackage("libcudf", exact=True) }} + - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - if: cuda_major != "11" then: - cuda-nvtx @@ -202,17 +202,17 @@ outputs: string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} requirements: host: - - ${{ pin_subpackage('libcudf', exact=True) }} - - ${{ pin_subpackage('libcudf_kafka', exact=True) }} + - ${{ pin_subpackage("libcudf", exact=True) }} + - ${{ pin_subpackage("libcudf_kafka", exact=True) }} - cuda-version =${{ cuda_version }} - if: cuda_major == "11" then: - libcurand ${{ cuda11_libcurand_run_version }} else: run: - - ${{ pin_compatible('cuda-version', upper_bound='x', lower_bound='x') }} - - ${{ pin_subpackage('libcudf', exact=True) }} - - ${{ pin_subpackage('libcudf_kafka', exact=True) }} + - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} + - ${{ pin_subpackage("libcudf", exact=True) }} + - ${{ pin_subpackage("libcudf_kafka", exact=True) }} - if: cuda_major == "11" then: - libcurand ${{ cuda11_libcurand_run_version }} diff --git a/conda/recipes/pylibcudf/recipe.yaml b/conda/recipes/pylibcudf/recipe.yaml index f6bd76b2697..d3f8b0d007f 100644 --- a/conda/recipes/pylibcudf/recipe.yaml +++ b/conda/recipes/pylibcudf/recipe.yaml @@ -4,9 +4,9 @@ schema_version: 1 context: version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} - minor_version: ${{ (version | split('.'))[:2] | join('.') }} - cuda_version: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[:2] | join(".") }} - cuda_major: ${{ (env.get('RAPIDS_CUDA_VERSION') | split('.'))[0] }} + minor_version: ${{ (version | split("."))[:2] | join(".") }} + cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} + cuda_major: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }} date_string: ${{ env.get("RAPIDS_DATE_STRING") }} py_version: ${{ env.get("RAPIDS_PY_VERSION") | version_to_buildstring }} head_rev: ${{ git.head_rev(".")[:8] }} @@ -74,7 +74,7 @@ requirements: - numpy >=1.23,<3.0a0 - pyarrow>=14.0.0,<20.0.0a0 - libcudf =${{ version }} - - ${{ pin_compatible('rmm', upper_bound='x.x') }} + - ${{ pin_compatible("rmm", upper_bound="x.x") }} - fsspec >=0.6.0 - if: cuda_major == "11" then: From e78f088430a4ed407689fd08dd64cebee250df1d Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 21 Feb 2025 09:37:00 -0500 Subject: [PATCH 16/51] fix(libcudf-tests): add missing else package --- conda/recipes/libcudf/recipe.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index f4267e16395..17253574b89 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -209,6 +209,7 @@ outputs: then: - libcurand ${{ cuda11_libcurand_run_version }} else: + - libcurand-dev run: - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - ${{ pin_subpackage("libcudf", exact=True) }} From 892cb8d915eb6a3343f4018428c7eb1d62fa6fb6 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 21 Feb 2025 09:40:08 -0500 Subject: [PATCH 17/51] make whitespace and newlines consistent --- conda/recipes/cudf-polars/recipe.yaml | 1 - conda/recipes/cudf/recipe.yaml | 1 - conda/recipes/cudf_kafka/recipe.yaml | 2 -- conda/recipes/custreamz/recipe.yaml | 2 -- conda/recipes/dask-cudf/recipe.yaml | 2 -- conda/recipes/pylibcudf/recipe.yaml | 1 - 6 files changed, 9 deletions(-) diff --git a/conda/recipes/cudf-polars/recipe.yaml b/conda/recipes/cudf-polars/recipe.yaml index 45cab112402..07bcc492868 100644 --- a/conda/recipes/cudf-polars/recipe.yaml +++ b/conda/recipes/cudf-polars/recipe.yaml @@ -60,7 +60,6 @@ tests: - cudf_polars pip_check: false - about: homepage: ${{ load_from_file("python/cudf_polars/pyproject.toml").project.urls.Homepage }} license: ${{ load_from_file("python/cudf_polars/pyproject.toml").project.license.text }} diff --git a/conda/recipes/cudf/recipe.yaml b/conda/recipes/cudf/recipe.yaml index 9c5d856b6b5..da95fa9a5f3 100644 --- a/conda/recipes/cudf/recipe.yaml +++ b/conda/recipes/cudf/recipe.yaml @@ -112,7 +112,6 @@ requirements: - if: linux64 then: libcufile-dev - tests: - python: imports: diff --git a/conda/recipes/cudf_kafka/recipe.yaml b/conda/recipes/cudf_kafka/recipe.yaml index bd86900fc25..37cf4fa9678 100644 --- a/conda/recipes/cudf_kafka/recipe.yaml +++ b/conda/recipes/cudf_kafka/recipe.yaml @@ -11,7 +11,6 @@ context: py_version: ${{ env.get("RAPIDS_PY_VERSION") | version_to_buildstring }} head_rev: ${{ git.head_rev(".")[:8] }} - package: name: cudf_kafka version: ${{ version }} @@ -73,7 +72,6 @@ requirements: by_name: - cuda-version - tests: - python: imports: diff --git a/conda/recipes/custreamz/recipe.yaml b/conda/recipes/custreamz/recipe.yaml index 947ec0f6d47..a60db49f736 100644 --- a/conda/recipes/custreamz/recipe.yaml +++ b/conda/recipes/custreamz/recipe.yaml @@ -10,7 +10,6 @@ context: date_string: ${{ env.get("RAPIDS_DATE_STRING") }} head_rev: ${{ git.head_rev(".")[:8] }} - package: name: custreamz version: ${{ version }} @@ -27,7 +26,6 @@ build: - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY - AWS_SESSION_TOKEN - env: CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} diff --git a/conda/recipes/dask-cudf/recipe.yaml b/conda/recipes/dask-cudf/recipe.yaml index abcfc309f61..ce814dd550d 100644 --- a/conda/recipes/dask-cudf/recipe.yaml +++ b/conda/recipes/dask-cudf/recipe.yaml @@ -27,7 +27,6 @@ build: - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY - AWS_SESSION_TOKEN - env: CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} @@ -56,7 +55,6 @@ requirements: by_name: - cuda-version - tests: - python: imports: diff --git a/conda/recipes/pylibcudf/recipe.yaml b/conda/recipes/pylibcudf/recipe.yaml index d3f8b0d007f..9a5787faeda 100644 --- a/conda/recipes/pylibcudf/recipe.yaml +++ b/conda/recipes/pylibcudf/recipe.yaml @@ -27,7 +27,6 @@ build: - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY - AWS_SESSION_TOKEN - env: CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} From b88ac255fa661f24bea37b84a8dabd563718199d Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 21 Feb 2025 09:52:22 -0500 Subject: [PATCH 18/51] ci(build_python): longname for cli args --- ci/build_python.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ci/build_python.sh b/ci/build_python.sh index 16393f74107..ad6afaecbef 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -19,8 +19,6 @@ CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) sccache --zero-stats -# TODO: Remove `--test skip` flag once importing on a CPU -# node works correctly RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) export RAPIDS_PACKAGE_VERSION @@ -28,6 +26,8 @@ export RAPIDS_PACKAGE_VERSION # populates `RATTLER_CHANNELS` array source rapids-rattler-channel-string +# TODO: Remove `--test skip` flag once importing on a CPU node works correctly + # --no-build-id allows for caching with `sccache` # more info is available at # https://rattler.build/latest/tips_and_tricks/#using-sccache-or-ccache-with-rattler-build @@ -37,7 +37,7 @@ rattler-build build --recipe conda/recipes/pylibcudf \ --channel-priority disabled \ --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ --test skip \ - -c "${CPP_CHANNEL}" \ + --channel "${CPP_CHANNEL}" \ "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats @@ -49,7 +49,7 @@ rattler-build build --recipe conda/recipes/cudf \ --channel-priority disabled \ --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ --test skip \ - -c "${CPP_CHANNEL}" \ + --channel "${CPP_CHANNEL}" \ "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats @@ -61,7 +61,7 @@ rattler-build build --recipe conda/recipes/dask-cudf \ --channel-priority disabled \ --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ --test skip \ - -c "${CPP_CHANNEL}" \ + --channel "${CPP_CHANNEL}" \ "${RATTLER_CHANNELS[@]}" rattler-build build --recipe conda/recipes/cudf_kafka \ @@ -70,7 +70,7 @@ rattler-build build --recipe conda/recipes/cudf_kafka \ --channel-priority disabled \ --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ --test skip \ - -c "${CPP_CHANNEL}" \ + --channel "${CPP_CHANNEL}" \ "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats @@ -81,7 +81,7 @@ rattler-build build --recipe conda/recipes/custreamz \ --channel-priority disabled \ --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ --test skip \ - -c "${CPP_CHANNEL}" \ + --channel "${CPP_CHANNEL}" \ "${RATTLER_CHANNELS[@]}" rattler-build build --recipe conda/recipes/cudf-polars \ @@ -90,7 +90,7 @@ rattler-build build --recipe conda/recipes/cudf-polars \ --channel-priority disabled \ --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ --test skip \ - -c "${CPP_CHANNEL}" \ + --channel "${CPP_CHANNEL}" \ "${RATTLER_CHANNELS[@]}" # remove build_cache directory From 5a3e0784a501a6da1ce446d2c648ba81ad5adb7b Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 21 Feb 2025 09:53:11 -0500 Subject: [PATCH 19/51] fix(build_python): consistently show and zero sccache stats --- ci/build_python.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ci/build_python.sh b/ci/build_python.sh index ad6afaecbef..03c415f3ffe 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -64,6 +64,9 @@ rattler-build build --recipe conda/recipes/dask-cudf \ --channel "${CPP_CHANNEL}" \ "${RATTLER_CHANNELS[@]}" +sccache --show-adv-stats +sccache --zero-stats + rattler-build build --recipe conda/recipes/cudf_kafka \ --experimental \ --no-build-id \ @@ -74,6 +77,7 @@ rattler-build build --recipe conda/recipes/cudf_kafka \ "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats +sccache --zero-stats rattler-build build --recipe conda/recipes/custreamz \ --experimental \ @@ -84,6 +88,9 @@ rattler-build build --recipe conda/recipes/custreamz \ --channel "${CPP_CHANNEL}" \ "${RATTLER_CHANNELS[@]}" +sccache --show-adv-stats +sccache --zero-stats + rattler-build build --recipe conda/recipes/cudf-polars \ --experimental \ --no-build-id \ @@ -93,6 +100,8 @@ rattler-build build --recipe conda/recipes/cudf-polars \ --channel "${CPP_CHANNEL}" \ "${RATTLER_CHANNELS[@]}" +sccache --show-adv-stats + # remove build_cache directory rm -rf "$RAPIDS_CONDA_BLD_OUTPUT_DIR"/build_cache From bd084f6a58938b14832dd66d39de3459e9119113 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 21 Feb 2025 10:14:44 -0500 Subject: [PATCH 20/51] fix(libcudf): disambiguate cuda11 versions for libcufile and libcurand --- conda/recipes/libcudf/recipe.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index 17253574b89..b50eea50435 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -60,10 +60,10 @@ cache: - if: cuda_major == "11" then: - cudatoolkit - - libcufile ${{ cuda11_libcufile_host_version }} - - libcufile-dev ${{ cuda11_libcufile_host_version }} - - libcurand ${{ cuda11_libcurand_host_version }} - - libcurand-dev ${{ cuda11_libcurand_host_version }} + - libcufile =${{ cuda11_libcufile_host_version }} + - libcufile-dev =${{ cuda11_libcufile_host_version }} + - libcurand =${{ cuda11_libcurand_host_version }} + - libcurand-dev =${{ cuda11_libcurand_host_version }} - cuda-nvrtc =${{ cuda_version }} - cuda-nvrtc-dev =${{ cuda_version }} - cuda-nvtx =${{ cuda_version }} From 11ca19175e82e40e5d3947a7b5937e45fdbaa846 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 21 Feb 2025 10:15:08 -0500 Subject: [PATCH 21/51] debug: render-only for cpp build because context seems wrong --- ci/build_cpp.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 78a15bc8092..202c573d204 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -30,6 +30,7 @@ rattler-build build --recipe conda/recipes/libcudf \ --no-build-id \ --channel-priority disabled \ --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ + --render-only \ "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats From 6f944c9874f3843bbf4f1044c443afa8428d50ef Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 21 Feb 2025 10:40:48 -0500 Subject: [PATCH 22/51] fix(libcudf-examples): cuda-nvtx-dev for cuda12 --- conda/recipes/libcudf/recipe.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index b50eea50435..9a200fa6bcb 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -179,12 +179,12 @@ outputs: - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - if: cuda_major != "11" then: - - cuda-nvtx + - cuda-nvtx-dev ignore_run_exports: from_package: - if: cuda_major != "11" then: - - cuda-nvtx + - cuda-nvtx-dev by_name: - cuda-version about: From 2e16236c4b96946b757abaa1a1a9d8e7babb7295 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 21 Feb 2025 11:03:29 -0500 Subject: [PATCH 23/51] fix(all): wrap cuda_major and date_string in quotes --- conda/recipes/cudf-polars/recipe.yaml | 4 ++-- conda/recipes/cudf/recipe.yaml | 4 ++-- conda/recipes/cudf_kafka/recipe.yaml | 4 ++-- conda/recipes/custreamz/recipe.yaml | 4 ++-- conda/recipes/dask-cudf/recipe.yaml | 4 ++-- conda/recipes/libcudf/recipe.yaml | 4 ++-- conda/recipes/pylibcudf/recipe.yaml | 4 ++-- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/conda/recipes/cudf-polars/recipe.yaml b/conda/recipes/cudf-polars/recipe.yaml index 07bcc492868..042e9555cd5 100644 --- a/conda/recipes/cudf-polars/recipe.yaml +++ b/conda/recipes/cudf-polars/recipe.yaml @@ -6,8 +6,8 @@ context: version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} minor_version: ${{ (version | split("."))[:2] | join(".") }} cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} - cuda_major: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }} - date_string: ${{ env.get("RAPIDS_DATE_STRING") }} + cuda_major: '${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }}' + date_string: '${{ env.get("RAPIDS_DATE_STRING") }}' py_version: ${{ env.get("RAPIDS_PY_VERSION") | version_to_buildstring }} head_rev: ${{ git.head_rev(".")[:8] }} diff --git a/conda/recipes/cudf/recipe.yaml b/conda/recipes/cudf/recipe.yaml index da95fa9a5f3..17dd1529ec8 100644 --- a/conda/recipes/cudf/recipe.yaml +++ b/conda/recipes/cudf/recipe.yaml @@ -6,8 +6,8 @@ context: version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} minor_version: ${{ (version | split("."))[:2] | join(".") }} cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} - cuda_major: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }} - date_string: ${{ env.get("RAPIDS_DATE_STRING") }} + cuda_major: '${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }}' + date_string: '${{ env.get("RAPIDS_DATE_STRING") }}' py_version: ${{ env.get("RAPIDS_PY_VERSION") | version_to_buildstring }} head_rev: ${{ git.head_rev(".")[:8] }} diff --git a/conda/recipes/cudf_kafka/recipe.yaml b/conda/recipes/cudf_kafka/recipe.yaml index 37cf4fa9678..3993a1bf1a5 100644 --- a/conda/recipes/cudf_kafka/recipe.yaml +++ b/conda/recipes/cudf_kafka/recipe.yaml @@ -6,8 +6,8 @@ context: version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} minor_version: ${{ (version | split("."))[:2] | join(".") }} cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} - cuda_major: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }} - date_string: ${{ env.get("RAPIDS_DATE_STRING") }} + cuda_major: '${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }}' + date_string: '${{ env.get("RAPIDS_DATE_STRING") }}' py_version: ${{ env.get("RAPIDS_PY_VERSION") | version_to_buildstring }} head_rev: ${{ git.head_rev(".")[:8] }} diff --git a/conda/recipes/custreamz/recipe.yaml b/conda/recipes/custreamz/recipe.yaml index a60db49f736..1f122e2fea5 100644 --- a/conda/recipes/custreamz/recipe.yaml +++ b/conda/recipes/custreamz/recipe.yaml @@ -6,8 +6,8 @@ context: version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} minor_version: ${{ (version | split("."))[:2] | join(".") }} cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} - cuda_major: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }} - date_string: ${{ env.get("RAPIDS_DATE_STRING") }} + cuda_major: '${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }}' + date_string: '${{ env.get("RAPIDS_DATE_STRING") }}' head_rev: ${{ git.head_rev(".")[:8] }} package: diff --git a/conda/recipes/dask-cudf/recipe.yaml b/conda/recipes/dask-cudf/recipe.yaml index ce814dd550d..6c88a3980ec 100644 --- a/conda/recipes/dask-cudf/recipe.yaml +++ b/conda/recipes/dask-cudf/recipe.yaml @@ -6,8 +6,8 @@ context: version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} minor_version: ${{ (version | split("."))[:2] | join(".") }} cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} - cuda_major: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }} - date_string: ${{ env.get("RAPIDS_DATE_STRING") }} + cuda_major: '${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }}' + date_string: '${{ env.get("RAPIDS_DATE_STRING") }}' py_version: ${{ env.get("RAPIDS_PY_VERSION") | version_to_buildstring }} head_rev: ${{ git.head_rev(".")[:8] }} diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index 9a200fa6bcb..78796174120 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -6,8 +6,8 @@ context: version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} minor_version: ${{ (version | split("."))[:2] | join(".") }} cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} - cuda_major: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }} - date_string: ${{ env.get("RAPIDS_DATE_STRING") }} + cuda_major: '${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }}' + date_string: '${{ env.get("RAPIDS_DATE_STRING") }}' head_rev: ${{ git.head_rev(".")[:8] }} recipe: diff --git a/conda/recipes/pylibcudf/recipe.yaml b/conda/recipes/pylibcudf/recipe.yaml index 9a5787faeda..14b79d5b341 100644 --- a/conda/recipes/pylibcudf/recipe.yaml +++ b/conda/recipes/pylibcudf/recipe.yaml @@ -6,8 +6,8 @@ context: version: ${{ env.get("RAPIDS_PACKAGE_VERSION") }} minor_version: ${{ (version | split("."))[:2] | join(".") }} cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} - cuda_major: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }} - date_string: ${{ env.get("RAPIDS_DATE_STRING") }} + cuda_major: '${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }}' + date_string: '${{ env.get("RAPIDS_DATE_STRING") }}' py_version: ${{ env.get("RAPIDS_PY_VERSION") | version_to_buildstring }} head_rev: ${{ git.head_rev(".")[:8] }} From 075c28718c66805c98b202444c14f6d25ba31c27 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 21 Feb 2025 11:18:10 -0500 Subject: [PATCH 24/51] (debug): turn off render-only --- ci/build_cpp.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 202c573d204..78a15bc8092 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -30,7 +30,6 @@ rattler-build build --recipe conda/recipes/libcudf \ --no-build-id \ --channel-priority disabled \ --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ - --render-only \ "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats From e450344375d558a339ae406c944b6e04f80a04cd Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 21 Feb 2025 13:00:25 -0500 Subject: [PATCH 25/51] fix(libcudf): make sure `cmake` is in `host` for `--install` --- conda/recipes/libcudf/recipe.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index 78796174120..dfc26f120da 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -92,6 +92,7 @@ outputs: string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} requirements: host: + - cmake ${{ cmake_version }} - cuda-version =${{ cuda_version }} run: - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} @@ -134,6 +135,7 @@ outputs: string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} requirements: host: + - cmake ${{ cmake_version }} - librdkafka ${{ librdkafka_version }} - ${{ pin_subpackage("libcudf", exact=True) }} run: @@ -202,6 +204,7 @@ outputs: string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} requirements: host: + - cmake ${{ cmake_version }} - ${{ pin_subpackage("libcudf", exact=True) }} - ${{ pin_subpackage("libcudf_kafka", exact=True) }} - cuda-version =${{ cuda_version }} From a79e92291eaa9e94fb86a8799491507a2aef0346 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 21 Feb 2025 15:17:45 -0500 Subject: [PATCH 26/51] fix(libcudf): only use libcufile on linux64 --- conda/recipes/libcudf/recipe.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index dfc26f120da..82f0f9c2100 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -60,13 +60,15 @@ cache: - if: cuda_major == "11" then: - cudatoolkit - - libcufile =${{ cuda11_libcufile_host_version }} - - libcufile-dev =${{ cuda11_libcufile_host_version }} - libcurand =${{ cuda11_libcurand_host_version }} - libcurand-dev =${{ cuda11_libcurand_host_version }} - cuda-nvrtc =${{ cuda_version }} - cuda-nvrtc-dev =${{ cuda_version }} - cuda-nvtx =${{ cuda_version }} + - if: linux64 + then: + - libcufile =${{ cuda11_libcufile_host_version }} + - libcufile-dev =${{ cuda11_libcufile_host_version }} else: - cuda-nvrtc-dev - cuda-nvtx-dev From 0420e2a7290cf99734fa1a05df0fd9d9ed90becf Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 21 Feb 2025 16:50:19 -0500 Subject: [PATCH 27/51] fix(libcudf): add `make` (`CMAKE_BUILD_PROGRAM` not set) --- conda/recipes/libcudf/recipe.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index 82f0f9c2100..bc8a5936d34 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -170,6 +170,7 @@ outputs: - cuda-version =${{ cuda_version }} - ninja - ${{ stdlib("c") }} + - make host: - ${{ pin_subpackage("libcudf", exact=True) }} - if: cuda_major == "11" From 23a65b799c3570bf78887edc476c81d5be515ad6 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Mon, 24 Feb 2025 09:57:06 -0500 Subject: [PATCH 28/51] fix(pypkgs): add `pip` to host --- conda/recipes/cudf-polars/recipe.yaml | 1 + conda/recipes/cudf/recipe.yaml | 1 + conda/recipes/cudf_kafka/recipe.yaml | 1 + conda/recipes/custreamz/recipe.yaml | 1 + conda/recipes/dask-cudf/recipe.yaml | 1 + conda/recipes/pylibcudf/recipe.yaml | 1 + 6 files changed, 6 insertions(+) diff --git a/conda/recipes/cudf-polars/recipe.yaml b/conda/recipes/cudf-polars/recipe.yaml index 042e9555cd5..8461cb1f8d8 100644 --- a/conda/recipes/cudf-polars/recipe.yaml +++ b/conda/recipes/cudf-polars/recipe.yaml @@ -42,6 +42,7 @@ build: requirements: host: - python + - pip - rapids-build-backend >=0.3.0,<0.4.0.dev0 - setuptools - cuda-version =${{ cuda_version }} diff --git a/conda/recipes/cudf/recipe.yaml b/conda/recipes/cudf/recipe.yaml index 17dd1529ec8..76654d02345 100644 --- a/conda/recipes/cudf/recipe.yaml +++ b/conda/recipes/cudf/recipe.yaml @@ -50,6 +50,7 @@ requirements: - ${{ stdlib("c") }} host: - python + - pip - cython >=3.0.3 - rapids-build-backend >=0.3.0,<0.4.0.dev0 - scikit-build-core >=0.10.0 diff --git a/conda/recipes/cudf_kafka/recipe.yaml b/conda/recipes/cudf_kafka/recipe.yaml index 3993a1bf1a5..e78fb007416 100644 --- a/conda/recipes/cudf_kafka/recipe.yaml +++ b/conda/recipes/cudf_kafka/recipe.yaml @@ -50,6 +50,7 @@ requirements: - ${{ stdlib("c") }} host: - python + - pip - cython >=3.0.3 - cuda-version =${{ cuda_version }} - pylibcudf =${{ version }} diff --git a/conda/recipes/custreamz/recipe.yaml b/conda/recipes/custreamz/recipe.yaml index 1f122e2fea5..7b9bf725921 100644 --- a/conda/recipes/custreamz/recipe.yaml +++ b/conda/recipes/custreamz/recipe.yaml @@ -41,6 +41,7 @@ build: requirements: host: - python + - pip - rapids-build-backend >=0.3.0,<0.4.0.dev0 - setuptools - python-confluent-kafka >=2.5.0,<2.6.0a0 diff --git a/conda/recipes/dask-cudf/recipe.yaml b/conda/recipes/dask-cudf/recipe.yaml index 6c88a3980ec..7260c70752c 100644 --- a/conda/recipes/dask-cudf/recipe.yaml +++ b/conda/recipes/dask-cudf/recipe.yaml @@ -42,6 +42,7 @@ build: requirements: host: - python + - pip - rapids-build-backend >=0.3.0,<0.4.0.dev0 - setuptools - cuda-version =${{ cuda_version }} diff --git a/conda/recipes/pylibcudf/recipe.yaml b/conda/recipes/pylibcudf/recipe.yaml index 14b79d5b341..ff77eb77fdb 100644 --- a/conda/recipes/pylibcudf/recipe.yaml +++ b/conda/recipes/pylibcudf/recipe.yaml @@ -50,6 +50,7 @@ requirements: - ${{ stdlib("c") }} host: - python + - pip - cython >=3.0.3 - rapids-build-backend >=0.3.0,<0.4.0.dev0 - scikit-build-core >=0.10.0 From aafe473479b2b2c90648a2651352783d6536b607 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Mon, 24 Feb 2025 10:34:36 -0500 Subject: [PATCH 29/51] fix(libcudf): move `cmake` to `build` --- conda/recipes/libcudf/recipe.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index bc8a5936d34..bbc188d3130 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -93,8 +93,9 @@ outputs: - cmake --install cpp/build string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} requirements: - host: + build: - cmake ${{ cmake_version }} + host: - cuda-version =${{ cuda_version }} run: - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} @@ -136,8 +137,9 @@ outputs: - cmake --install cpp/libcudf_kafka/build string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} requirements: - host: + build: - cmake ${{ cmake_version }} + host: - librdkafka ${{ librdkafka_version }} - ${{ pin_subpackage("libcudf", exact=True) }} run: @@ -206,8 +208,9 @@ outputs: - cmake --install cpp/libcudf_kafka/build --component testing string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} requirements: - host: + build: - cmake ${{ cmake_version }} + host: - ${{ pin_subpackage("libcudf", exact=True) }} - ${{ pin_subpackage("libcudf_kafka", exact=True) }} - cuda-version =${{ cuda_version }} From b1e3c9214e8b30247def175a580d2df9c3f4b6df Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Mon, 24 Feb 2025 12:58:06 -0500 Subject: [PATCH 30/51] fix(custreamz): update license, remove double $ --- conda/recipes/custreamz/recipe.yaml | 6 +++--- python/custreamz/pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conda/recipes/custreamz/recipe.yaml b/conda/recipes/custreamz/recipe.yaml index 7b9bf725921..973eed265ae 100644 --- a/conda/recipes/custreamz/recipe.yaml +++ b/conda/recipes/custreamz/recipe.yaml @@ -66,6 +66,6 @@ tests: pip_check: false about: - homepage: $${{ load_from_file("python/custreamz/pyproject.toml").project.urls.Homepage }} - license: $${{ load_from_file("python/custreamz/pyproject.toml").project.license.text }} - summary: $${{ load_from_file("python/custreamz/pyproject.toml").project.description }} + homepage: ${{ load_from_file("python/custreamz/pyproject.toml").project.urls.Homepage }} + license: ${{ load_from_file("python/custreamz/pyproject.toml").project.license.text }} + summary: ${{ load_from_file("python/custreamz/pyproject.toml").project.description }} diff --git a/python/custreamz/pyproject.toml b/python/custreamz/pyproject.toml index 665b0a76ecf..b1fbe901189 100644 --- a/python/custreamz/pyproject.toml +++ b/python/custreamz/pyproject.toml @@ -16,7 +16,7 @@ readme = { file = "README.md", content-type = "text/markdown" } authors = [ { name = "NVIDIA Corporation" }, ] -license = { text = "Apache 2.0" } +license = { text = "Apache-2.0" } requires-python = ">=3.10" dependencies = [ "confluent-kafka>=2.5.0,<2.6.0a0", From 16d67285ffff62da5be8c1d28b5df9c98bb6ce14 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Wed, 26 Feb 2025 11:35:46 -0500 Subject: [PATCH 31/51] fix(libcudf): fix overlinking --- conda/recipes/libcudf/recipe.yaml | 69 ++++++++++++++++++++++++++++--- 1 file changed, 63 insertions(+), 6 deletions(-) diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index bbc188d3130..637e597c459 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -92,11 +92,20 @@ outputs: script: - cmake --install cpp/build string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} + dynamic_linking: + overlinking_behavior: "error" requirements: build: - cmake ${{ cmake_version }} + - ${{ compiler("c") }} host: - cuda-version =${{ cuda_version }} + - libkvikio =${{ minor_version }} + - rapids-logger =0.1 + - zlib ${{ zlib_version }} + - if: cuda_major == "11" + then: cudatoolkit + else: cuda-cudart-dev run: - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - if: cuda_major == "11" @@ -119,7 +128,16 @@ outputs: - ${{ pin_subpackage("libcudf", upper_bound="x.x") }} ignore_run_exports: by_name: + - cuda-cudart + - cuda-nvrtc + - cuda-nvtx - cuda-version + - flatbuffers + - libcurand + - libkvikio + - librdkafka + - librmm + - nvcomp tests: - script: - test -f $PREFIX/lib/libcudf.so @@ -136,9 +154,12 @@ outputs: script: - cmake --install cpp/libcudf_kafka/build string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} + dynamic_linking: + overlinking_behavior: "error" requirements: build: - cmake ${{ cmake_version }} + - ${{ stdlib("c") }} host: - librdkafka ${{ librdkafka_version }} - ${{ pin_subpackage("libcudf", exact=True) }} @@ -147,7 +168,16 @@ outputs: - ${{ pin_subpackage("libcudf", exact=True) }} ignore_run_exports: by_name: + - cuda-cudart + - cuda-nvrtc + - cuda-nvtx - cuda-version + - flatbuffers + - libcurand + - libkvikio + - librdkafka + - librmm + - nvcomp tests: - script: - test -f $PREFIX/lib/libcudf_kafka.so @@ -163,37 +193,50 @@ outputs: script: - ./cpp/examples/build.sh --install string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} + dynamic_linking: + overlinking_behavior: "error" requirements: build: - - cmake ${{ cmake_version }} - ${{ compiler("c") }} + - ${{ compiler("cuda") }} - ${{ compiler("cxx") }} - - ${{ compiler("cuda") }} =${{ cuda_version }} - - cuda-version =${{ cuda_version }} - - ninja - ${{ stdlib("c") }} + - cmake ${{ cmake_version }} + - cuda-version =${{ cuda_version }} - make + - ninja host: - ${{ pin_subpackage("libcudf", exact=True) }} + - cuda-version =${{ cuda_version }} - if: cuda_major == "11" then: - cuda-nvtx =${{ cuda_version }} + - cudatoolkit else: - cuda-nvtx-dev - - cuda-version =${{ cuda_version }} + - cuda-cudart-dev run: - ${{ pin_subpackage("libcudf", exact=True) }} - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - if: cuda_major != "11" then: - - cuda-nvtx-dev + - cuda-nvtx ignore_run_exports: from_package: - if: cuda_major != "11" then: - cuda-nvtx-dev by_name: + - cuda-cudart + - cuda-nvrtc + - cuda-nvtx - cuda-version + - flatbuffers + - libcurand + - libkvikio + - librdkafka + - librmm + - nvcomp about: homepage: ${{ load_from_file("python/libcudf/pyproject.toml").project.urls.Homepage }} license: ${{ load_from_file("python/libcudf/pyproject.toml").project.license.text }} @@ -207,9 +250,12 @@ outputs: - cmake --install cpp/build --component testing - cmake --install cpp/libcudf_kafka/build --component testing string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} + dynamic_linking: + overlinking_behavior: "error" requirements: build: - cmake ${{ cmake_version }} + - ${{ stdlib("c") }} host: - ${{ pin_subpackage("libcudf", exact=True) }} - ${{ pin_subpackage("libcudf_kafka", exact=True) }} @@ -217,8 +263,10 @@ outputs: - if: cuda_major == "11" then: - libcurand ${{ cuda11_libcurand_run_version }} + - cudatoolkit else: - libcurand-dev + - cuda-cudart-dev run: - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - ${{ pin_subpackage("libcudf", exact=True) }} @@ -234,7 +282,16 @@ outputs: then: - libcurand-dev by_name: + - cuda-cudart + - cuda-nvrtc + - cuda-nvtx - cuda-version + - flatbuffers + - libcurand + - libkvikio + - librdkafka + - librmm + - nvcomp about: homepage: ${{ load_from_file("python/libcudf/pyproject.toml").project.urls.Homepage }} license: ${{ load_from_file("python/libcudf/pyproject.toml").project.license.text }} From f049b9567fa9bfc421d67d3d4fe479e8ad61b346 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Wed, 26 Feb 2025 11:36:10 -0500 Subject: [PATCH 32/51] fix(cudf): don't overconstrain cuda-version --- conda/recipes/cudf/recipe.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/conda/recipes/cudf/recipe.yaml b/conda/recipes/cudf/recipe.yaml index 76654d02345..992428b46f2 100644 --- a/conda/recipes/cudf/recipe.yaml +++ b/conda/recipes/cudf/recipe.yaml @@ -81,7 +81,7 @@ requirements: - pylibcudf =${{ version }} - ${{ pin_compatible("rmm", upper_bound="x.x") }} - fsspec >=0.6.0 - - if: cuda_major == "11" + - if: cuda_major == "11" then: - cudatoolkit - ptxcompiler >=0.7.0 @@ -104,7 +104,6 @@ requirements: - packaging - cachetools - rich - ignore_run_exports: from_package: - if: cuda_major != "11" @@ -112,6 +111,8 @@ requirements: - cuda-cudart-dev - if: linux64 then: libcufile-dev + by_name: + - cuda-version tests: - python: From 258bbe550c0bdf464b882e233429b5114351692b Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Wed, 26 Feb 2025 11:36:15 -0500 Subject: [PATCH 33/51] fix(custreamz): add `py_version` --- conda/recipes/custreamz/recipe.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda/recipes/custreamz/recipe.yaml b/conda/recipes/custreamz/recipe.yaml index 973eed265ae..7a7ee63ff3d 100644 --- a/conda/recipes/custreamz/recipe.yaml +++ b/conda/recipes/custreamz/recipe.yaml @@ -8,6 +8,7 @@ context: cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} cuda_major: '${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }}' date_string: '${{ env.get("RAPIDS_DATE_STRING") }}' + py_version: ${{ env.get("RAPIDS_PY_VERSION") | version_to_buildstring }} head_rev: ${{ git.head_rev(".")[:8] }} package: From a13d1f3c3f4473c540dbef499958d3e12aa759bd Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Wed, 26 Feb 2025 11:36:20 -0500 Subject: [PATCH 34/51] fix(pylibcudf): simplify neq syntax --- conda/recipes/pylibcudf/recipe.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/recipes/pylibcudf/recipe.yaml b/conda/recipes/pylibcudf/recipe.yaml index ff77eb77fdb..c955914904e 100644 --- a/conda/recipes/pylibcudf/recipe.yaml +++ b/conda/recipes/pylibcudf/recipe.yaml @@ -85,7 +85,7 @@ requirements: - packaging ignore_run_exports: from_package: - - if: not (cuda_major == "11") + - if: cuda_major != "11" then: - cuda-cudart-dev - if: linux64 From bb41ea352bc3d31088be20e709761f7be5fb15bc Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Wed, 26 Feb 2025 11:36:25 -0500 Subject: [PATCH 35/51] fix(custreamz): remove unnecessary ignore_run_exports --- conda/recipes/custreamz/recipe.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/conda/recipes/custreamz/recipe.yaml b/conda/recipes/custreamz/recipe.yaml index 7a7ee63ff3d..5150ce8c707 100644 --- a/conda/recipes/custreamz/recipe.yaml +++ b/conda/recipes/custreamz/recipe.yaml @@ -56,9 +56,6 @@ requirements: - rapids-dask-dependency =${{ minor_version }} - python-confluent-kafka >=2.5.0,<2.6.0a0 - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - ignore_run_exports: - by_name: - - cuda-version tests: - python: From a4c06fe7e1cf11c3f84a46b33c77eb3f61a7b1b6 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Wed, 26 Feb 2025 11:36:29 -0500 Subject: [PATCH 36/51] refactor(libcudf): multiline yaml block for build script --- conda/recipes/libcudf/recipe.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index 637e597c459..342babb43bc 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -19,11 +19,10 @@ cache: build: script: - content: - - cudf_ROOT="$(realpath ./cpp/build)" - - export cudf_ROOT - - > - ./build.sh -n -v \ + content: | + cudf_ROOT="$(realpath ./cpp/build)" + export cudf_ROOT + ./build.sh -n -v \ libcudf libcudf_kafka benchmarks tests \ --build_metrics --incl_cache_stats --allgpuarch \ --cmake-args=\"-DCMAKE_INSTALL_LIBDIR=lib -DCUDF_ENABLE_ARROW_S3=ON\" From ba1eae29f3135fff4af81347bb519400e4d875dc Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Wed, 26 Feb 2025 11:36:33 -0500 Subject: [PATCH 37/51] fix(dask-cudf): remove extraneous ignore_run_export --- conda/recipes/dask-cudf/recipe.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/conda/recipes/dask-cudf/recipe.yaml b/conda/recipes/dask-cudf/recipe.yaml index 7260c70752c..54050fded4e 100644 --- a/conda/recipes/dask-cudf/recipe.yaml +++ b/conda/recipes/dask-cudf/recipe.yaml @@ -52,9 +52,6 @@ requirements: - pynvml >=12.0.0,<13.0.0a0 - rapids-dask-dependency =${{ minor_version }} - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} - ignore_run_exports: - by_name: - - cuda-version tests: - python: From eb84ed51086886700ee6b1c572d8994cb42ee51d Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Wed, 26 Feb 2025 11:36:38 -0500 Subject: [PATCH 38/51] fix(libcudf): remove CMAKE_INSTALL_LIBDIR arg --- conda/recipes/libcudf/recipe.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index 342babb43bc..50bfc1100e6 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -25,7 +25,7 @@ cache: ./build.sh -n -v \ libcudf libcudf_kafka benchmarks tests \ --build_metrics --incl_cache_stats --allgpuarch \ - --cmake-args=\"-DCMAKE_INSTALL_LIBDIR=lib -DCUDF_ENABLE_ARROW_S3=ON\" + --cmake-args=\"-DCUDF_ENABLE_ARROW_S3=ON\" secrets: - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY From f7fb7136029add87cb06994f5af4a1a93c758f9c Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Wed, 26 Feb 2025 11:36:42 -0500 Subject: [PATCH 39/51] refactor(all): use multiline yaml block for all script contents --- conda/recipes/cudf-polars/recipe.yaml | 4 ++-- conda/recipes/cudf/recipe.yaml | 4 ++-- conda/recipes/cudf_kafka/recipe.yaml | 4 ++-- conda/recipes/custreamz/recipe.yaml | 6 +++--- conda/recipes/dask-cudf/recipe.yaml | 4 ++-- conda/recipes/libcudf/recipe.yaml | 21 +++++++++++++++++++-- conda/recipes/pylibcudf/recipe.yaml | 4 ++-- 7 files changed, 32 insertions(+), 15 deletions(-) diff --git a/conda/recipes/cudf-polars/recipe.yaml b/conda/recipes/cudf-polars/recipe.yaml index 8461cb1f8d8..fc8b6343cac 100644 --- a/conda/recipes/cudf-polars/recipe.yaml +++ b/conda/recipes/cudf-polars/recipe.yaml @@ -21,8 +21,8 @@ source: build: string: cuda${{ cuda_major }}_py${{ py_version }}_${{ date_string }}_${{ head_rev }} script: - content: - - ./build.sh cudf_polars + content: | + ./build.sh cudf_polars secrets: - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY diff --git a/conda/recipes/cudf/recipe.yaml b/conda/recipes/cudf/recipe.yaml index 992428b46f2..749a86714db 100644 --- a/conda/recipes/cudf/recipe.yaml +++ b/conda/recipes/cudf/recipe.yaml @@ -21,8 +21,8 @@ source: build: string: cuda${{ cuda_major }}_py${{ py_version }}_${{ date_string }}_${{ head_rev }} script: - content: - - ./build.sh cudf + content: | + ./build.sh cudf secrets: - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY diff --git a/conda/recipes/cudf_kafka/recipe.yaml b/conda/recipes/cudf_kafka/recipe.yaml index e78fb007416..130f4efa8c0 100644 --- a/conda/recipes/cudf_kafka/recipe.yaml +++ b/conda/recipes/cudf_kafka/recipe.yaml @@ -21,8 +21,8 @@ source: build: string: cuda${{ cuda_major }}_py${{ py_version }}_${{ date_string }}_${{ head_rev }} script: - content: - - ./build.sh -v cudf_kafka + content: | + ./build.sh -v cudf_kafka secrets: - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY diff --git a/conda/recipes/custreamz/recipe.yaml b/conda/recipes/custreamz/recipe.yaml index 5150ce8c707..a879045d6a4 100644 --- a/conda/recipes/custreamz/recipe.yaml +++ b/conda/recipes/custreamz/recipe.yaml @@ -21,8 +21,8 @@ source: build: string: cuda${{ cuda_major }}_py${{ py_version }}_${{ date_string }}_${{ head_rev }} script: - content: - - ./build.sh -v custreamz + content: | + ./build.sh -v custreamz secrets: - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY @@ -37,7 +37,7 @@ build: SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }} SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} - SCCACHE_S3_KEY_PREFIX: pylibcudf-${{ env.get("RAPIDS_CONDA_ARCH") }} + SCCACHE_S3_KEY_PREFIX: custreamz-${{ env.get("RAPIDS_CONDA_ARCH") }} requirements: host: diff --git a/conda/recipes/dask-cudf/recipe.yaml b/conda/recipes/dask-cudf/recipe.yaml index 54050fded4e..060520d4e7b 100644 --- a/conda/recipes/dask-cudf/recipe.yaml +++ b/conda/recipes/dask-cudf/recipe.yaml @@ -21,8 +21,8 @@ source: build: string: cuda${{ cuda_major }}_py${{ py_version }}_${{ date_string }}_${{ head_rev }} script: - content: - - ./build.sh dask_cudf + content: | + ./build.sh dask_cudf secrets: - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index 50bfc1100e6..3458201d8e2 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -190,7 +190,25 @@ outputs: version: ${{ version }} build: script: - - ./cpp/examples/build.sh --install + content: | + ./cpp/examples/build.sh --install + secrets: + - AWS_ACCESS_KEY_ID + - AWS_SECRET_ACCESS_KEY + - AWS_SESSION_TOKEN + env: + CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} + CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} + CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} + CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} + PARALLEL_LEVEL: ${{ env.get("PARALLEL_LEVEL") }} + RAPIDS_ARTIFACTS_DIR: ${{ env.get("RAPIDS_ARTIFACTS_DIR") }} + SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }} + SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }} + SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }} + SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} + SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} + SCCACHE_S3_KEY_PREFIX: libcudf-${{ env.get("RAPIDS_CONDA_ARCH") }} string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} dynamic_linking: overlinking_behavior: "error" @@ -202,7 +220,6 @@ outputs: - ${{ stdlib("c") }} - cmake ${{ cmake_version }} - cuda-version =${{ cuda_version }} - - make - ninja host: - ${{ pin_subpackage("libcudf", exact=True) }} diff --git a/conda/recipes/pylibcudf/recipe.yaml b/conda/recipes/pylibcudf/recipe.yaml index c955914904e..9617bf6e367 100644 --- a/conda/recipes/pylibcudf/recipe.yaml +++ b/conda/recipes/pylibcudf/recipe.yaml @@ -21,8 +21,8 @@ source: build: string: cuda${{ cuda_major }}_py${{ py_version }}_${{ date_string }}_${{ head_rev }} script: - content: - - ./build.sh pylibcudf + content: | + ./build.sh pylibcudf secrets: - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY From 90f33d40148ed897ce2cd53c9f8b0426e28debed Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Wed, 26 Feb 2025 11:36:48 -0500 Subject: [PATCH 40/51] chore(cudf): update numpy and numba pins --- conda/recipes/cudf/recipe.yaml | 6 +++--- conda/recipes/pylibcudf/recipe.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conda/recipes/cudf/recipe.yaml b/conda/recipes/cudf/recipe.yaml index 749a86714db..611310fd76a 100644 --- a/conda/recipes/cudf/recipe.yaml +++ b/conda/recipes/cudf/recipe.yaml @@ -73,9 +73,9 @@ requirements: - typing_extensions >=4.0.0 - pandas >=2.0,<2.2.4dev0 - cupy >=12.0.0 - - numba-cuda >=0.2.0,<0.3.0a0 - - numba >=0.59.1,<0.61.0a0 - - numpy >=1.23,<3.0a0 + - numba-cuda >=0.4.0,<0.5.0a0 + - numba >=0.59.1,<0.62.0a0 + - numpy >=1.23,<2.1 - pyarrow>=14.0.0,<20.0.0a0 - libcudf =${{ version }} - pylibcudf =${{ version }} diff --git a/conda/recipes/pylibcudf/recipe.yaml b/conda/recipes/pylibcudf/recipe.yaml index 9617bf6e367..40d9ee547a1 100644 --- a/conda/recipes/pylibcudf/recipe.yaml +++ b/conda/recipes/pylibcudf/recipe.yaml @@ -71,7 +71,7 @@ requirements: - python - typing_extensions >=4.0.0 - pandas >=2.0,<2.2.4dev0 - - numpy >=1.23,<3.0a0 + - numpy >=1.23,<2.1 - pyarrow>=14.0.0,<20.0.0a0 - libcudf =${{ version }} - ${{ pin_compatible("rmm", upper_bound="x.x") }} From 058c46697af41d1420522111beebbd7f0c80b1ed Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Wed, 26 Feb 2025 13:06:36 -0800 Subject: [PATCH 41/51] fix(py): set explicit py_version for python outputs --- conda/recipes/cudf-polars/recipe.yaml | 7 ++++--- conda/recipes/cudf/recipe.yaml | 7 ++++--- conda/recipes/cudf_kafka/recipe.yaml | 7 ++++--- conda/recipes/custreamz/recipe.yaml | 7 ++++--- conda/recipes/dask-cudf/recipe.yaml | 7 ++++--- conda/recipes/pylibcudf/recipe.yaml | 7 ++++--- 6 files changed, 24 insertions(+), 18 deletions(-) diff --git a/conda/recipes/cudf-polars/recipe.yaml b/conda/recipes/cudf-polars/recipe.yaml index fc8b6343cac..f17772a0cfc 100644 --- a/conda/recipes/cudf-polars/recipe.yaml +++ b/conda/recipes/cudf-polars/recipe.yaml @@ -8,7 +8,8 @@ context: cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} cuda_major: '${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }}' date_string: '${{ env.get("RAPIDS_DATE_STRING") }}' - py_version: ${{ env.get("RAPIDS_PY_VERSION") | version_to_buildstring }} + py_version: ${{ env.get("RAPIDS_PY_VERSION") }} + py_buildstring: ${{ py_version | version_to_buildstring }} head_rev: ${{ git.head_rev(".")[:8] }} package: @@ -19,7 +20,7 @@ source: path: ../../.. build: - string: cuda${{ cuda_major }}_py${{ py_version }}_${{ date_string }}_${{ head_rev }} + string: cuda${{ cuda_major }}_py${{ py_buildstring }}_${{ date_string }}_${{ head_rev }} script: content: | ./build.sh cudf_polars @@ -41,7 +42,7 @@ build: requirements: host: - - python + - python =${{ py_version }} - pip - rapids-build-backend >=0.3.0,<0.4.0.dev0 - setuptools diff --git a/conda/recipes/cudf/recipe.yaml b/conda/recipes/cudf/recipe.yaml index 611310fd76a..fd4aba3c992 100644 --- a/conda/recipes/cudf/recipe.yaml +++ b/conda/recipes/cudf/recipe.yaml @@ -8,7 +8,8 @@ context: cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} cuda_major: '${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }}' date_string: '${{ env.get("RAPIDS_DATE_STRING") }}' - py_version: ${{ env.get("RAPIDS_PY_VERSION") | version_to_buildstring }} + py_version: ${{ env.get("RAPIDS_PY_VERSION") }} + py_buildstring: ${{ py_version | version_to_buildstring }} head_rev: ${{ git.head_rev(".")[:8] }} package: @@ -19,7 +20,7 @@ source: path: ../../.. build: - string: cuda${{ cuda_major }}_py${{ py_version }}_${{ date_string }}_${{ head_rev }} + string: cuda${{ cuda_major }}_py${{ py_buildstring }}_${{ date_string }}_${{ head_rev }} script: content: | ./build.sh cudf @@ -49,7 +50,7 @@ requirements: - cuda-version =${{ cuda_version }} - ${{ stdlib("c") }} host: - - python + - python =${{ py_version }} - pip - cython >=3.0.3 - rapids-build-backend >=0.3.0,<0.4.0.dev0 diff --git a/conda/recipes/cudf_kafka/recipe.yaml b/conda/recipes/cudf_kafka/recipe.yaml index 130f4efa8c0..11eb0289169 100644 --- a/conda/recipes/cudf_kafka/recipe.yaml +++ b/conda/recipes/cudf_kafka/recipe.yaml @@ -8,7 +8,8 @@ context: cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} cuda_major: '${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }}' date_string: '${{ env.get("RAPIDS_DATE_STRING") }}' - py_version: ${{ env.get("RAPIDS_PY_VERSION") | version_to_buildstring }} + py_version: ${{ env.get("RAPIDS_PY_VERSION") }} + py_buildstring: ${{ py_version | version_to_buildstring }} head_rev: ${{ git.head_rev(".")[:8] }} package: @@ -19,7 +20,7 @@ source: path: ../../.. build: - string: cuda${{ cuda_major }}_py${{ py_version }}_${{ date_string }}_${{ head_rev }} + string: cuda${{ cuda_major }}_py${{ py_buildstring }}_${{ date_string }}_${{ head_rev }} script: content: | ./build.sh -v cudf_kafka @@ -49,7 +50,7 @@ requirements: - cuda-version =${{ cuda_version }} - ${{ stdlib("c") }} host: - - python + - python =${{ py_version }} - pip - cython >=3.0.3 - cuda-version =${{ cuda_version }} diff --git a/conda/recipes/custreamz/recipe.yaml b/conda/recipes/custreamz/recipe.yaml index a879045d6a4..82f34f03856 100644 --- a/conda/recipes/custreamz/recipe.yaml +++ b/conda/recipes/custreamz/recipe.yaml @@ -8,7 +8,8 @@ context: cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} cuda_major: '${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }}' date_string: '${{ env.get("RAPIDS_DATE_STRING") }}' - py_version: ${{ env.get("RAPIDS_PY_VERSION") | version_to_buildstring }} + py_version: ${{ env.get("RAPIDS_PY_VERSION") }} + py_buildstring: ${{ py_version | version_to_buildstring }} head_rev: ${{ git.head_rev(".")[:8] }} package: @@ -19,7 +20,7 @@ source: path: ../../.. build: - string: cuda${{ cuda_major }}_py${{ py_version }}_${{ date_string }}_${{ head_rev }} + string: cuda${{ cuda_major }}_py${{ py_buildstring }}_${{ date_string }}_${{ head_rev }} script: content: | ./build.sh -v custreamz @@ -41,7 +42,7 @@ build: requirements: host: - - python + - python =${{ py_version }} - pip - rapids-build-backend >=0.3.0,<0.4.0.dev0 - setuptools diff --git a/conda/recipes/dask-cudf/recipe.yaml b/conda/recipes/dask-cudf/recipe.yaml index 060520d4e7b..e2cac732f7c 100644 --- a/conda/recipes/dask-cudf/recipe.yaml +++ b/conda/recipes/dask-cudf/recipe.yaml @@ -8,7 +8,8 @@ context: cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} cuda_major: '${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }}' date_string: '${{ env.get("RAPIDS_DATE_STRING") }}' - py_version: ${{ env.get("RAPIDS_PY_VERSION") | version_to_buildstring }} + py_version: ${{ env.get("RAPIDS_PY_VERSION") }} + py_buildstring: ${{ py_version | version_to_buildstring }} head_rev: ${{ git.head_rev(".")[:8] }} package: @@ -19,7 +20,7 @@ source: path: ../../.. build: - string: cuda${{ cuda_major }}_py${{ py_version }}_${{ date_string }}_${{ head_rev }} + string: cuda${{ cuda_major }}_py${{ py_buildstring }}_${{ date_string }}_${{ head_rev }} script: content: | ./build.sh dask_cudf @@ -41,7 +42,7 @@ build: requirements: host: - - python + - python =${{ py_version }} - pip - rapids-build-backend >=0.3.0,<0.4.0.dev0 - setuptools diff --git a/conda/recipes/pylibcudf/recipe.yaml b/conda/recipes/pylibcudf/recipe.yaml index 40d9ee547a1..29942f181b3 100644 --- a/conda/recipes/pylibcudf/recipe.yaml +++ b/conda/recipes/pylibcudf/recipe.yaml @@ -8,7 +8,8 @@ context: cuda_version: ${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[:2] | join(".") }} cuda_major: '${{ (env.get("RAPIDS_CUDA_VERSION") | split("."))[0] }}' date_string: '${{ env.get("RAPIDS_DATE_STRING") }}' - py_version: ${{ env.get("RAPIDS_PY_VERSION") | version_to_buildstring }} + py_version: ${{ env.get("RAPIDS_PY_VERSION") }} + py_buildstring: ${{ py_version | version_to_buildstring }} head_rev: ${{ git.head_rev(".")[:8] }} package: @@ -19,7 +20,7 @@ source: path: ../../.. build: - string: cuda${{ cuda_major }}_py${{ py_version }}_${{ date_string }}_${{ head_rev }} + string: cuda${{ cuda_major }}_py${{ py_buildstring }}_${{ date_string }}_${{ head_rev }} script: content: | ./build.sh pylibcudf @@ -49,7 +50,7 @@ requirements: - cuda-version =${{ cuda_version }} - ${{ stdlib("c") }} host: - - python + - python =${{ py_version }} - pip - cython >=3.0.3 - rapids-build-backend >=0.3.0,<0.4.0.dev0 From cfcf8347f7c94dde947c407a7a791f475ee6c1a8 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Wed, 26 Feb 2025 13:54:46 -0800 Subject: [PATCH 42/51] fix(libcudf): allowlist `libnvidia-ml.so.1` --- conda/recipes/libcudf/recipe.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index 3458201d8e2..ddd9c8ddae2 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -268,6 +268,8 @@ outputs: string: cuda${{ cuda_major }}_${{ date_string }}_${{ head_rev }} dynamic_linking: overlinking_behavior: "error" + missing_dso_allowlist: + - "libnvidia-ml.so.1" requirements: build: - cmake ${{ cmake_version }} From 42ece0b46d38ed2213e8822cf074cc92bc97cdb5 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Thu, 27 Feb 2025 11:57:40 -0500 Subject: [PATCH 43/51] chore: restore additional commented-out workflow jobs --- .github/workflows/pr.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index fc7728964f0..2c583598f54 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -17,7 +17,7 @@ jobs: - changed-files - checks - conda-cpp-build - # - cpp-linters + - cpp-linters - conda-cpp-checks - conda-cpp-tests - conda-python-build @@ -41,7 +41,7 @@ jobs: - pandas-tests-diff - narwhals-tests - telemetry-setup - # - third-party-integration-tests-cudf-pandas + - third-party-integration-tests-cudf-pandas secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-25.04 if: always() @@ -135,14 +135,14 @@ jobs: with: build_type: pull-request node_type: "cpu16" - # cpp-linters: - # secrets: inherit - # needs: checks - # uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.04 - # with: - # build_type: pull-request - # run_script: "ci/cpp_linters.sh" - # node_type: "cpu16" + cpp-linters: + secrets: inherit + needs: checks + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.04 + with: + build_type: pull-request + run_script: "ci/cpp_linters.sh" + node_type: "cpu16" conda-cpp-checks: needs: conda-cpp-build secrets: inherit From 2d7b95cec956ea654aec977f88454e72f567dcd5 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Thu, 27 Feb 2025 11:46:42 -0800 Subject: [PATCH 44/51] fix(libcudf): resolve overlinking error --- conda/recipes/libcudf/recipe.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index ddd9c8ddae2..dd270e50d2c 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -100,6 +100,7 @@ outputs: host: - cuda-version =${{ cuda_version }} - libkvikio =${{ minor_version }} + - nvcomp ${{ nvcomp_version }} - rapids-logger =0.1 - zlib ${{ zlib_version }} - if: cuda_major == "11" From e3972e903e35270d5769c1d8beea5f7324864fa5 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Thu, 27 Feb 2025 16:47:56 -0500 Subject: [PATCH 45/51] fix(cudf-polars): bump to <1.24 --- conda/recipes/cudf-polars/recipe.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/recipes/cudf-polars/recipe.yaml b/conda/recipes/cudf-polars/recipe.yaml index f17772a0cfc..c3fc2be1575 100644 --- a/conda/recipes/cudf-polars/recipe.yaml +++ b/conda/recipes/cudf-polars/recipe.yaml @@ -50,7 +50,7 @@ requirements: run: - python - pylibcudf =${{ version }} - - polars >=1.20,<1.23 + - polars >=1.20,<1.24 - ${{ pin_compatible("cuda-version", upper_bound="x", lower_bound="x") }} ignore_run_exports: by_name: From 167b3102a0b1643d032b6b346aa499d71912ac75 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 28 Feb 2025 09:37:40 -0500 Subject: [PATCH 46/51] fix(python): prepend $CPP_CHANNEL to $RATTLER_CHANNELS --- ci/build_python.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ci/build_python.sh b/ci/build_python.sh index 03c415f3ffe..c03e34beed6 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -26,6 +26,10 @@ export RAPIDS_PACKAGE_VERSION # populates `RATTLER_CHANNELS` array source rapids-rattler-channel-string +rapids-logger "Prepending channel ${CPP_CHANNEL} to RATTLER_CHANNELS" + +RATTLER_CHANNELS=("--channel ${CPP_CHANNEL}" "${RATTLER_CHANNELS[@]}") + # TODO: Remove `--test skip` flag once importing on a CPU node works correctly # --no-build-id allows for caching with `sccache` @@ -37,7 +41,6 @@ rattler-build build --recipe conda/recipes/pylibcudf \ --channel-priority disabled \ --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ --test skip \ - --channel "${CPP_CHANNEL}" \ "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats @@ -49,7 +52,6 @@ rattler-build build --recipe conda/recipes/cudf \ --channel-priority disabled \ --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ --test skip \ - --channel "${CPP_CHANNEL}" \ "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats @@ -61,7 +63,6 @@ rattler-build build --recipe conda/recipes/dask-cudf \ --channel-priority disabled \ --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ --test skip \ - --channel "${CPP_CHANNEL}" \ "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats @@ -73,7 +74,6 @@ rattler-build build --recipe conda/recipes/cudf_kafka \ --channel-priority disabled \ --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ --test skip \ - --channel "${CPP_CHANNEL}" \ "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats @@ -85,7 +85,6 @@ rattler-build build --recipe conda/recipes/custreamz \ --channel-priority disabled \ --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ --test skip \ - --channel "${CPP_CHANNEL}" \ "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats @@ -97,7 +96,6 @@ rattler-build build --recipe conda/recipes/cudf-polars \ --channel-priority disabled \ --output-dir "$RAPIDS_CONDA_BLD_OUTPUT_DIR" \ --test skip \ - --channel "${CPP_CHANNEL}" \ "${RATTLER_CHANNELS[@]}" sccache --show-adv-stats From 29bcce7655151d0e861c5ae8323ebad75dbe9dc6 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 28 Feb 2025 09:42:52 -0500 Subject: [PATCH 47/51] chore: remove verbose flags --- conda/recipes/cudf_kafka/recipe.yaml | 2 +- conda/recipes/custreamz/recipe.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/recipes/cudf_kafka/recipe.yaml b/conda/recipes/cudf_kafka/recipe.yaml index 11eb0289169..cbb295b0e7b 100644 --- a/conda/recipes/cudf_kafka/recipe.yaml +++ b/conda/recipes/cudf_kafka/recipe.yaml @@ -23,7 +23,7 @@ build: string: cuda${{ cuda_major }}_py${{ py_buildstring }}_${{ date_string }}_${{ head_rev }} script: content: | - ./build.sh -v cudf_kafka + ./build.sh cudf_kafka secrets: - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY diff --git a/conda/recipes/custreamz/recipe.yaml b/conda/recipes/custreamz/recipe.yaml index 82f34f03856..5f879a3f181 100644 --- a/conda/recipes/custreamz/recipe.yaml +++ b/conda/recipes/custreamz/recipe.yaml @@ -23,7 +23,7 @@ build: string: cuda${{ cuda_major }}_py${{ py_buildstring }}_${{ date_string }}_${{ head_rev }} script: content: | - ./build.sh -v custreamz + ./build.sh custreamz secrets: - AWS_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY From 5c0cafffade0349b66cfb2e4bfef14b478395edd Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 28 Feb 2025 09:43:50 -0500 Subject: [PATCH 48/51] chore: remove secrets and env from pure python packages --- conda/recipes/custreamz/recipe.yaml | 15 --------------- conda/recipes/dask-cudf/recipe.yaml | 15 --------------- 2 files changed, 30 deletions(-) diff --git a/conda/recipes/custreamz/recipe.yaml b/conda/recipes/custreamz/recipe.yaml index 5f879a3f181..9048490afb7 100644 --- a/conda/recipes/custreamz/recipe.yaml +++ b/conda/recipes/custreamz/recipe.yaml @@ -24,21 +24,6 @@ build: script: content: | ./build.sh custreamz - secrets: - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN - env: - CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} - CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} - CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} - CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} - SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }} - SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }} - SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }} - SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} - SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} - SCCACHE_S3_KEY_PREFIX: custreamz-${{ env.get("RAPIDS_CONDA_ARCH") }} requirements: host: diff --git a/conda/recipes/dask-cudf/recipe.yaml b/conda/recipes/dask-cudf/recipe.yaml index e2cac732f7c..e2ec8ccce74 100644 --- a/conda/recipes/dask-cudf/recipe.yaml +++ b/conda/recipes/dask-cudf/recipe.yaml @@ -24,21 +24,6 @@ build: script: content: | ./build.sh dask_cudf - secrets: - - AWS_ACCESS_KEY_ID - - AWS_SECRET_ACCESS_KEY - - AWS_SESSION_TOKEN - env: - CMAKE_C_COMPILER_LAUNCHER: ${{ env.get("CMAKE_C_COMPILER_LAUNCHER") }} - CMAKE_CUDA_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CUDA_COMPILER_LAUNCHER") }} - CMAKE_CXX_COMPILER_LAUNCHER: ${{ env.get("CMAKE_CXX_COMPILER_LAUNCHER") }} - CMAKE_GENERATOR: ${{ env.get("CMAKE_GENERATOR") }} - SCCACHE_BUCKET: ${{ env.get("SCCACHE_BUCKET") }} - SCCACHE_IDLE_TIMEOUT: ${{ env.get("SCCACHE_IDLE_TIMEOUT") }} - SCCACHE_REGION: ${{ env.get("SCCACHE_REGION") }} - SCCACHE_S3_USE_SSL: ${{ env.get("SCCACHE_S3_USE_SSL") }} - SCCACHE_S3_NO_CREDENTIALS: ${{ env.get("SCCACHE_S3_NO_CREDENTIALS") }} - SCCACHE_S3_KEY_PREFIX: dask-cudf-${{ env.get("RAPIDS_CONDA_ARCH") }} requirements: host: From 8a6e1c4126b67070008bdb977192b546673082ac Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 28 Feb 2025 09:44:42 -0500 Subject: [PATCH 49/51] fix(libcudf): remove cudf_ROOT argument --- conda/recipes/libcudf/recipe.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index dd270e50d2c..d594947492f 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -20,8 +20,6 @@ cache: build: script: content: | - cudf_ROOT="$(realpath ./cpp/build)" - export cudf_ROOT ./build.sh -n -v \ libcudf libcudf_kafka benchmarks tests \ --build_metrics --incl_cache_stats --allgpuarch \ From 8413cbe87c10f56d5fa06a9f8bbe4a0a5ae6ad64 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 28 Feb 2025 11:55:28 -0500 Subject: [PATCH 50/51] fix(build_python): channel flag is a separate array element --- ci/build_python.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_python.sh b/ci/build_python.sh index c03e34beed6..ee114b63631 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -28,7 +28,7 @@ source rapids-rattler-channel-string rapids-logger "Prepending channel ${CPP_CHANNEL} to RATTLER_CHANNELS" -RATTLER_CHANNELS=("--channel ${CPP_CHANNEL}" "${RATTLER_CHANNELS[@]}") +RATTLER_CHANNELS=("--channel" "${CPP_CHANNEL}" "${RATTLER_CHANNELS[@]}") # TODO: Remove `--test skip` flag once importing on a CPU node works correctly From 76ce12a2cbe9cd0b7a7549d4daed8c28b988772d Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 28 Feb 2025 16:25:49 -0500 Subject: [PATCH 51/51] fix(libcudf): remove cache-busting debug line from CFLAGS and CXXFLAGS --- conda/recipes/libcudf/recipe.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/conda/recipes/libcudf/recipe.yaml b/conda/recipes/libcudf/recipe.yaml index d594947492f..90f5f13845e 100644 --- a/conda/recipes/libcudf/recipe.yaml +++ b/conda/recipes/libcudf/recipe.yaml @@ -20,6 +20,15 @@ cache: build: script: content: | + + # Remove `-fdebug-prefix-map` line from CFLAGS and CXXFLAGS so the + # incrementing version number in the compile line doesn't break the + # cache + set -x + export CFLAGS=$(echo $CFLAGS | sed -E 's@\-fdebug\-prefix\-map[^ ]*@@g') + export CXXFLAGS=$(echo $CXXFLAGS | sed -E 's@\-fdebug\-prefix\-map[^ ]*@@g') + set +x + ./build.sh -n -v \ libcudf libcudf_kafka benchmarks tests \ --build_metrics --incl_cache_stats --allgpuarch \